feat(raft-apply-decoupling): convert job submit to continuation-based commit (task 3.2)
Replace handle_jobsubmit()'s blocking commit call (task 3.1) with a real async registration against commit_submit_async(). On FAILED/timeout, replies failure immediately with no job constructed. On COMMITTED, hands off to a new pending-apply registration (persist_register_pending_apply) for the ordered apply pump (task 6.1, not yet built) to consume -- does not apply or reply directly, per design. Deviates from design.md's stated integration site: appends via the same ha_primary_log_append/persist_wal_async primitive the RETAIN-fact path already uses and calls commit_submit_async directly on p->repl.ce, bypassing repl_wait_majority_ack/repl_submit_majority_commit in repl.c. Verified safe -- majority-ack detection is driven entirely by (ce, lsn) events inside the commit engine, agnostic to which wrapper registered the continuation -- but narrows the reusable primitive design.md intended; documented in tasks.md. Deletes (not preserves-dead) task 3.1's bug-1408 notification fix -- apply_committed_entry()'s WAL_JOB_SUBMIT case (task 2.2) already fires the notification per-entry, immune to the batch-suppression class bug-1408 fixed. IMPORTANT: no live path applies a successfully-committed job submission until task 6 lands (only restart replay does) -- every successful submission commits durably but gets zero reply until then. This commit range must not be soak-tested or deployed against live traffic before task 6 completes.
This commit is contained in:
parent
6f1fa97fab
commit
df4d2fa0ed
5 changed files with 689 additions and 299 deletions
|
|
@ -38,6 +38,20 @@ extern int worker_from_capnp(cluster_t* c,
|
|||
extern int restore_queue_state(cluster_t* c,
|
||||
const char* name, uint32_t status);
|
||||
|
||||
/* raft-apply-decoupling task 3.2: pending command-apply hand-off list node
|
||||
* (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is
|
||||
* the consumer). Declared here (rather than persist.h) so the list stays
|
||||
* an implementation detail — persist.h exposes only the opaque `void*`
|
||||
* head field plus persist_register_pending_apply/persist_has_pending_apply.
|
||||
* See persist.h's doc comment on persist_register_pending_apply for the
|
||||
* full contract. */
|
||||
typedef struct pending_command_apply {
|
||||
uint64_t lsn;
|
||||
pending_apply_notify_fn on_applied;
|
||||
void* ctx;
|
||||
struct pending_command_apply* next;
|
||||
} pending_command_apply_t;
|
||||
|
||||
/* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */
|
||||
|
||||
/* Serialize xce_job_t to a malloc'd Cap'n Proto buffer.
|
||||
|
|
@ -617,6 +631,12 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
|
|||
p->term_index_count = 0;
|
||||
p->term_index_cap = 0;
|
||||
|
||||
/* Pending command-apply hand-off list starts empty (raft-apply-
|
||||
* decoupling task 3.2): no committed-outcome continuations have
|
||||
* registered yet. Freed in persist_fini. */
|
||||
p->pending_applies = NULL;
|
||||
pthread_mutex_init(&p->pending_applies_lock, NULL);
|
||||
|
||||
/* Create WAL and snapshot directories */
|
||||
if (mkdir_p(p->wal_path, 0700) != 0) return -errno;
|
||||
if (mkdir_p(p->snap_path, 0700) != 0) return -errno;
|
||||
|
|
@ -723,6 +743,23 @@ void persist_fini(persist_t* p)
|
|||
p->term_index = NULL;
|
||||
p->term_index_count = 0;
|
||||
p->term_index_cap = 0;
|
||||
|
||||
/* Free any pending command-apply hand-off entries never drained by a
|
||||
* (not-yet-built) task 6 apply pump (raft-apply-decoupling task 3.2).
|
||||
* Each node's `ctx` is opaque to persist.c (owned by the registering
|
||||
* caller, e.g. job.c's jobsubmit_async_ctx_t) — freeing the node here
|
||||
* does NOT free ctx; a caller shutting down with entries still pending
|
||||
* is responsible for its own ctx lifetime (test harnesses that tear
|
||||
* down mid-flight must account for this). */
|
||||
pending_command_apply_t* pn =
|
||||
(pending_command_apply_t*)p->pending_applies;
|
||||
while (pn != NULL) {
|
||||
pending_command_apply_t* next = pn->next;
|
||||
free(pn);
|
||||
pn = next;
|
||||
}
|
||||
p->pending_applies = NULL;
|
||||
pthread_mutex_destroy(&p->pending_applies_lock);
|
||||
}
|
||||
|
||||
/* Forward declaration — defined after checkpoint internals */
|
||||
|
|
@ -3706,6 +3743,194 @@ int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads,
|
|||
return rc;
|
||||
}
|
||||
|
||||
/* raft-apply-decoupling task 3.2: append-only counterpart of
|
||||
* submit_job_wal_entry_and_commit — writes the entry via the SAME
|
||||
* primary-append primitive wal_hook_append_classified already uses for
|
||||
* RETAIN-mode facts (ha_primary_log_append / persist_wal_async), WITHOUT
|
||||
* waiting on the majority-commit barrier. Deliberately does NOT reuse
|
||||
* submit_job_wal_entry_and_commit itself: that helper is shared with
|
||||
* persist_submit_wal_and_commit's still-synchronous, still-blocking
|
||||
* contract (relied on directly by test_egress_filter.c / test_e2e_hooks.c /
|
||||
* test_persist_hooks.c), which this task must not disturb. */
|
||||
static int append_job_wal_entry_local(persist_t* p, const xce_job_t* job,
|
||||
uint64_t* out_lsn)
|
||||
{
|
||||
uint8_t* buf = NULL;
|
||||
size_t sz = 0;
|
||||
int rc;
|
||||
uint64_t lsn;
|
||||
|
||||
if (capnp_encode_job(job, &buf, &sz) != 0)
|
||||
return -ENOMEM;
|
||||
|
||||
/* Snapshot the LSN this entry will receive BEFORE the append (same
|
||||
* convention as submit_job_wal_entry_and_commit / wal_hook_append_
|
||||
* classified: persist_wal_async / ha_primary_log_append assigns
|
||||
* hdr.lsn = current_lsn then increments). */
|
||||
pthread_mutex_lock(&p->lock);
|
||||
lsn = p->current_lsn;
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
|
||||
if (p->cluster != NULL)
|
||||
rc = ha_primary_log_append(p->cluster, WAL_JOB_SUBMIT, buf,
|
||||
(uint32_t)sz, NULL);
|
||||
else
|
||||
rc = persist_wal_async(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz);
|
||||
free(buf);
|
||||
|
||||
if (rc == 0)
|
||||
*out_lsn = lsn;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* raft-apply-decoupling task 3.2: bridges commit_submit_async's
|
||||
* commit_continuation_fn shape (commit_outcome_t + rc) to
|
||||
* persist_job_commit_notify_fn's shape (a single rc) so persist.h's public
|
||||
* declaration never needs commit.h's commit_outcome_t (see persist.h's
|
||||
* doc comment on persist_job_commit_notify_fn). One adapter is heap-
|
||||
* allocated per registered lsn and frees itself after its single
|
||||
* invocation — safe because commit_resolve guarantees exactly-once
|
||||
* delivery. */
|
||||
typedef struct {
|
||||
persist_job_commit_notify_fn fn;
|
||||
void* ctx;
|
||||
} job_commit_adapter_ctx_t;
|
||||
|
||||
static void job_commit_resolve_adapter(void* adapter_ctx, uint64_t lsn,
|
||||
commit_outcome_t outcome, int rc)
|
||||
{
|
||||
job_commit_adapter_ctx_t* a = (job_commit_adapter_ctx_t*)adapter_ctx;
|
||||
int final_rc = (outcome == COMMIT_OUTCOME_COMMITTED) ? 0 : rc;
|
||||
|
||||
a->fn(a->ctx, lsn, final_rc);
|
||||
free(a);
|
||||
}
|
||||
|
||||
int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads,
|
||||
uint64_t* out_lsns,
|
||||
persist_job_commit_notify_fn fn,
|
||||
void* ctx)
|
||||
{
|
||||
int first_rc = 0;
|
||||
|
||||
if (p == NULL || payloads == NULL || out_lsns == NULL || fn == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
for (int i = 0; i < payloads->count; i++) {
|
||||
xce_job_t* j = (xce_job_t*)xlist_get(payloads, i);
|
||||
uint64_t lsn = 0;
|
||||
int rc;
|
||||
|
||||
if (first_rc != 0) {
|
||||
/* A prior payload in this batch already failed locally: every
|
||||
* remaining payload resolves with that SAME failure now,
|
||||
* synchronously — so `fn` is invoked exactly once per payload
|
||||
* regardless of where the failure occurred, and the caller
|
||||
* never has to separately track "never going to be called"
|
||||
* entries. */
|
||||
fn(ctx, 0, first_rc);
|
||||
continue;
|
||||
}
|
||||
|
||||
rc = append_job_wal_entry_local(p, j, &lsn);
|
||||
if (rc != 0) {
|
||||
first_rc = rc;
|
||||
fn(ctx, 0, rc);
|
||||
continue;
|
||||
}
|
||||
out_lsns[i] = lsn;
|
||||
|
||||
/* LSN-0 edge (documented once-per-cluster-lifetime interim
|
||||
* behavior — ha-commit-engine Implementation Notes, mirrored by
|
||||
* repl_wait_majority_ack's walAck branch): the first-ever WAL
|
||||
* entry of a fresh cluster is structurally un-commitable and
|
||||
* commit_submit_async rejects lsn == 0, so resolve it inline.
|
||||
* Engine-not-wired (bootstrap / engine-less contexts) mirrors
|
||||
* persist_commit_barrier's own "no engine -> no-op success"
|
||||
* convention — there is no async completion authority to
|
||||
* register with. */
|
||||
if (lsn == 0 || p->repl.ce == NULL) {
|
||||
fn(ctx, lsn, 0);
|
||||
continue;
|
||||
}
|
||||
|
||||
job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a));
|
||||
if (a == NULL) {
|
||||
first_rc = -ENOMEM;
|
||||
fn(ctx, lsn, -ENOMEM);
|
||||
continue;
|
||||
}
|
||||
a->fn = fn;
|
||||
a->ctx = ctx;
|
||||
|
||||
int reg_rc = commit_submit_async(p->repl.ce, lsn,
|
||||
job_commit_resolve_adapter, a);
|
||||
if (reg_rc != 0) {
|
||||
free(a);
|
||||
first_rc = reg_rc;
|
||||
ERROR("persist_submit_job_payloads_async: lsn=%lu appended but "
|
||||
"commit_submit_async registration failed (rc=%d) -- the "
|
||||
"entry remains durable and will still be caught up "
|
||||
"eventually via replay, but this caller will not be told "
|
||||
"its outcome via this path",
|
||||
(unsigned long)lsn, reg_rc);
|
||||
fn(ctx, lsn, reg_rc);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return first_rc;
|
||||
}
|
||||
|
||||
void persist_register_pending_apply(persist_t* p, uint64_t lsn,
|
||||
pending_apply_notify_fn on_applied,
|
||||
void* ctx)
|
||||
{
|
||||
pending_command_apply_t* node;
|
||||
|
||||
if (p == NULL || on_applied == NULL)
|
||||
return;
|
||||
|
||||
node = calloc(1, sizeof(*node));
|
||||
if (node == NULL) {
|
||||
ERROR("persist_register_pending_apply: OOM registering the apply "
|
||||
"hand-off for lsn=%lu -- the entry is still durably committed "
|
||||
"and will still be applied by a future restart replay, but no "
|
||||
"continuation will fire for it via this path",
|
||||
(unsigned long)lsn);
|
||||
return;
|
||||
}
|
||||
node->lsn = lsn;
|
||||
node->on_applied = on_applied;
|
||||
node->ctx = ctx;
|
||||
|
||||
pthread_mutex_lock(&p->pending_applies_lock);
|
||||
node->next = (pending_command_apply_t*)p->pending_applies;
|
||||
p->pending_applies = node;
|
||||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||||
}
|
||||
|
||||
int persist_has_pending_apply(persist_t* p, uint64_t lsn)
|
||||
{
|
||||
pending_command_apply_t* n;
|
||||
int found = 0;
|
||||
|
||||
if (p == NULL)
|
||||
return 0;
|
||||
|
||||
pthread_mutex_lock(&p->pending_applies_lock);
|
||||
for (n = (pending_command_apply_t*)p->pending_applies; n != NULL;
|
||||
n = n->next) {
|
||||
if (n->lsn == lsn) {
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
/* P0 payload: just the detail value (4 bytes).
|
||||
* Full Cap'n Proto serialization is wired in Task 15. */
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue