Terminal jobs (DONE/EXIT/PERR) were only filtered from checkpoints and WAL replay (job_should_retain, finished_retain_s=3600) — the live jobmgr never dropped them, so joblist and the scheduler's core->jobs view grew without bound across a controller's uptime (N4). Three pieces: 1. kvmgr delete path (src/lib/kvmgr.c): xkvmgr_type had .del_key=NULL and .vclear=NULL, so key_del()/vclear() on a kvmgr only returned -EINVAL. New xkvmgr_del_key (key_find to locate the slot idx -> key_del — kvtab's val_killer is NULL so the idx scalar is untouched -> vclear(data, idx)) and xkvmgr_vclear forwarder, wired into the typeinfo. A missing key propagates -ENOENT (same contract as xkvmgr_find). 2. Controller periodic sweep (job.c): job_live_gc_sweep — primary-only (standbys rebuild their live table from WAL replay; deleting there diverges), collects !job_should_retain jobs under the jobmgr READ lock with reference() (check_workers_timeout collect pattern), then per-job WRITE-locked key_del. TWO dereferences per purge: the collection reference and the table's own makeJob reference — the jobmgr's datakiller (job_obj_killer) is a deliberate no-op, so key_del transfers the table's ownership duty to the caller; one dereference would leak at refcount 1 forever (the very accumulation this sweep drains). Registered by cluster_start_job_live_gc with ha_start_primary_heartbeat's idempotent static-arm pattern + delaymgr.running guard (60s, runcount=-1). 3. Registration on both promotion tails (ha.c): raft road after ha_start_primary_heartbeat and the non-raft ha_promote_to_primary. NOT cluster_start — it is orphaned on the HA myapp_run path. Safety: apply_committed_entry's jobstate group and WAL_JOB_UPDATE already treat a key_find miss as a benign skip (bug-1970, which explicitly anticipates post-purge facts); persist_verify_apply_correctness counts only within the boot replay window. The TRANSIENT TTL now takes live effect too (previously checkpoint-exclusion only). UTs: kvmgr del INT+STRING (miss propagation, sibling intact, freed-slot reuse, killer invocation), controller test_job_live_gc.c (expired DONE purged + refcount 0, fresh/RUN retained, idempotent re-run, EXIT/PERR, TRANSIENT TTL + transient_since==0 guard, standby no-op, NULL-safe, arming guard). xcelib 761/761, controller 1354/1354. E2E (t1): promote arms the sweep; the pre-restart DONE array jobs were purged across five sweeps (33 jobs) as their 3600s windows elapsed; submit→RUN→END flows and all daemons healthy throughout. Co-Authored-By: Claude Code <noreply@anthropic.com>
230 lines
6.3 KiB
C
230 lines
6.3 KiB
C
#include <rtti.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <unity/unity.h>
|
|
|
|
#include "kv.h"
|
|
#include "kvmgr.h"
|
|
#include "utils.h"
|
|
|
|
typedef struct {
|
|
char* key;
|
|
int data;
|
|
} kv_data_t;
|
|
|
|
static void test_kvmgr_create(void)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 4,
|
|
.delta = 4,
|
|
.datakiller = NULL };
|
|
var kvmgr = makeInstance(&xkvmgr_type, ¶ms);
|
|
|
|
TEST_ASSERT_TRUE(kvmgr != NULL);
|
|
|
|
destroy(kvmgr);
|
|
}
|
|
|
|
typedef struct {
|
|
char* name;
|
|
uint16_t age;
|
|
} user_t;
|
|
|
|
static void my_user_killer(void* data)
|
|
{
|
|
user_t* u = (user_t*)data;
|
|
|
|
if (u == NULL) {
|
|
return;
|
|
}
|
|
|
|
FREEUP(u->name);
|
|
FREEUP(u);
|
|
}
|
|
|
|
static void test_kvmgr_add(void)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 4,
|
|
.delta = 4,
|
|
.datakiller = my_user_killer };
|
|
var kvmgr = makeInstance(&xkvmgr_type, ¶ms);
|
|
user_t* u = NULL;
|
|
int rc = 0;
|
|
void* data = NULL;
|
|
|
|
TEST_ASSERT_TRUE(kvmgr != NULL);
|
|
|
|
u = calloc(1, sizeof(*u));
|
|
TEST_ASSERT_TRUE(u != NULL);
|
|
|
|
u->name = strdup("tester1");
|
|
TEST_ASSERT_TRUE(u->name != NULL);
|
|
|
|
u->age = 32;
|
|
|
|
rc = key_add(kvmgr, (void*)u->name, (void*)u);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
rc = key_find(kvmgr, (void*)u->name, &data);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
TEST_ASSERT_TRUE(data == u);
|
|
|
|
destroy(kvmgr);
|
|
}
|
|
|
|
static void test_kvmgr_iter(void)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_INT,
|
|
.nslots = 4,
|
|
.delta = 4,
|
|
.datakiller = NULL };
|
|
var kvmgr = makeInstance(&xkvmgr_type, ¶ms);
|
|
int rc = 0;
|
|
int i;
|
|
iterator_t it;
|
|
|
|
TEST_ASSERT_TRUE(kvmgr != NULL);
|
|
|
|
for (i = 0; i < 10; i++) {
|
|
rc = key_add(kvmgr, (void*)(long)i, (void*)(long)(i * i));
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
}
|
|
|
|
rc = iter_make(&it, kvmgr);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
i = 0;
|
|
for (;;) {
|
|
void* data;
|
|
|
|
rc = iter_next(&it, &data);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
break;
|
|
}
|
|
|
|
TEST_ASSERT_TRUE((int)(long)data == i * i);
|
|
i++;
|
|
}
|
|
|
|
TEST_ASSERT_TRUE(i == 10);
|
|
|
|
iter_destroy(&it);
|
|
|
|
destroy(kvmgr);
|
|
}
|
|
|
|
/* admin-plane-gaps 断点③: xkvmgr del_key / vclear wiring. key_del unlinks
|
|
* the kvtab entry AND clears the backing varray slot; a subsequent find
|
|
* misses, sibling entries stay intact, and a double delete propagates the
|
|
* miss. With a datakiller wired, clearing the slot frees the value (the
|
|
* same path the controller's jobmgr purge relies on, except there the
|
|
* killer is a no-op and the caller dereferences the job itself). */
|
|
static void test_kvmgr_del_key(void)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 4,
|
|
.delta = 4,
|
|
.datakiller = my_user_killer };
|
|
var kvmgr = makeInstance(&xkvmgr_type, ¶ms);
|
|
char key1[] = "user1";
|
|
char key2[] = "user2";
|
|
user_t* u1 = NULL;
|
|
user_t* u2 = NULL;
|
|
int rc = 0;
|
|
void* data = NULL;
|
|
|
|
TEST_ASSERT_TRUE(kvmgr != NULL);
|
|
|
|
u1 = calloc(1, sizeof(*u1));
|
|
u2 = calloc(1, sizeof(*u2));
|
|
TEST_ASSERT_TRUE(u1 != NULL && u2 != NULL);
|
|
u1->name = strdup(key1);
|
|
u2->name = strdup(key2);
|
|
TEST_ASSERT_TRUE(u1->name != NULL && u2->name != NULL);
|
|
u1->age = 30;
|
|
u2->age = 40;
|
|
|
|
rc = key_add(kvmgr, (void*)key1, (void*)u1);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
rc = key_add(kvmgr, (void*)key2, (void*)u2);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
/* Delete the first key. With a datakiller wired, the cleared slot
|
|
* frees u1 — so only the stack key buffers are used from here on. */
|
|
rc = key_del(kvmgr, (void*)key1);
|
|
TEST_ASSERT_EQUAL_INT(1, rc);
|
|
|
|
/* The deleted key no longer resolves. */
|
|
rc = key_find(kvmgr, (void*)key1, &data);
|
|
TEST_ASSERT_TRUE(rc < 0);
|
|
|
|
/* The sibling entry is untouched. */
|
|
rc = key_find(kvmgr, (void*)key2, &data);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
TEST_ASSERT_TRUE(data == u2);
|
|
|
|
/* Double delete misses — xkvmgr_del_key propagates key_find's
|
|
* -ENOENT for an absent key (same contract as xkvmgr_find). */
|
|
rc = key_del(kvmgr, (void*)key1);
|
|
TEST_ASSERT_EQUAL_INT(-ENOENT, rc);
|
|
|
|
destroy(kvmgr);
|
|
}
|
|
|
|
static void test_kvmgr_del_key_int(void)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_INT,
|
|
.nslots = 4,
|
|
.delta = 4,
|
|
.datakiller = NULL };
|
|
var kvmgr = makeInstance(&xkvmgr_type, ¶ms);
|
|
int rc = 0;
|
|
void* data = NULL;
|
|
|
|
TEST_ASSERT_TRUE(kvmgr != NULL);
|
|
|
|
rc = key_add(kvmgr, (void*)(long)101, (void*)(long)10100);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
rc = key_add(kvmgr, (void*)(long)102, (void*)(long)10200);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
rc = key_del(kvmgr, (void*)(long)101);
|
|
TEST_ASSERT_EQUAL_INT(1, rc);
|
|
|
|
rc = key_find(kvmgr, (void*)(long)101, &data);
|
|
TEST_ASSERT_TRUE(rc < 0);
|
|
|
|
/* The int-keyed table's idx scalar was cleared with the entry — the
|
|
* slot the deleted key occupied now stores nothing. */
|
|
rc = key_find(kvmgr, (void*)(long)102, &data);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
TEST_ASSERT_EQUAL_INT(10200, (int)(long)data);
|
|
|
|
/* Delete the sibling too, then re-add: the freed slot must be
|
|
* reusable (vclear reset it in place). */
|
|
rc = key_del(kvmgr, (void*)(long)102);
|
|
TEST_ASSERT_EQUAL_INT(1, rc);
|
|
|
|
rc = key_add(kvmgr, (void*)(long)103, (void*)(long)10300);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
|
|
rc = key_find(kvmgr, (void*)(long)103, &data);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
TEST_ASSERT_EQUAL_INT(10300, (int)(long)data);
|
|
|
|
destroy(kvmgr);
|
|
}
|
|
|
|
void test_kvmgr(void)
|
|
{
|
|
printf("\n>> kvmgr test\n");
|
|
RUN_TEST(test_kvmgr_create);
|
|
RUN_TEST(test_kvmgr_add);
|
|
RUN_TEST(test_kvmgr_iter);
|
|
RUN_TEST(test_kvmgr_del_key);
|
|
RUN_TEST(test_kvmgr_del_key_int);
|
|
}
|