lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)

Browse source 
ha_send_snapshot_to_standby ignored the network send's return value in its
chunk loop, silently truncating a transfer on a mid-stream error while still
logging "complete" (bug-1386). Fixing the silent drop exposed a second,
independent bug in the same pipeline: ha_handle_snapshot_chunk appended
received chunks in arrival order instead of at their wire-carried
chunk_offset, so out-of-order delivery across the vtask pool corrupted the
reassembled file even though every chunk's bytes were individually correct
(bug-1389). Fixed by checking every fallible call in the send loop and
switching the receiver to pwrite-at-offset with a count-based completion
tracker.

Adds snapverify/snapdiff xce_test commands (offline checkpoint-file
structural validator and byte-level comparator) used to diagnose bug-1389 and
kept as standing diagnostic tools for future snapshot-transfer triage.

g_drive_async_configpush_offload flipped back ON after both fixes, validated
via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
This commit is contained in:
Rongsong Shen 2026-07-03 02:40:12 +00:00
parent 02e46f6ba1
commit 5cb36794d2
14 changed files with 1553 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

86
src/controller/config_repl.c
View file

@ -23,6 +23,7 @@
#include <unistd.h>
#include <inttypes.h>
#include <zlib.h> /* bug-1389 triage: per-chunk crc32 instrumentation */
#include "byteorder.h"
#include "clusterconfig.h"
@ -1273,21 +1274,40 @@ typedef struct {
} push_ctx_t;
#endif
/* push_thread_send_hook: test-only seam (same pattern as
* xce_wal_stream_request_hook) letting a unit test intercept/park the
* per-standby send so it can probe endpoint_ref/unref bracketing around a
* concurrent teardown (bug-1381) without a real transport connection. NULL
* in production — push_thread falls through to the real
* xce_replconfigpush_req_raw. */
int (*push_thread_send_hook)(endpoint_t *ep, xce_peerinfo_t *from,
uint64_t new_tag, const uint8_t *sha256,
const uint8_t *bytes, size_t blen) = NULL;
static void *push_thread(void *arg)
{
push_ctx_t *ctx = (push_ctx_t *)arg;
int rc = xce_replconfigpush_req_raw(
ctx->ep, ctx->from,
ctx->new_tag,
ctx->sha256, 32,
ctx->bytes, ctx->blen);
int rc = (push_thread_send_hook != NULL)
? push_thread_send_hook(ctx->ep, ctx->from, ctx->new_tag,
ctx->sha256, ctx->bytes, ctx->blen)
: xce_replconfigpush_req_raw(
ctx->ep, ctx->from,
ctx->new_tag,
ctx->sha256, 32,
ctx->bytes, ctx->blen);
if (rc == 0)
ctx->acked = 1;
else
WARN("push_config_to_standbys: push to endpoint %p failed: %d", ctx->ep, rc);
/* bug-1381: drop the ref push_config_transport_only took on ctx->ep before
* spawning us (the general bug-1298 refcount system) — this thread is the
* one non-owner reader of ep for the full push, so it owns the matching
* release. Last use of ctx->ep in this thread; safe to unref now. */
endpoint_unref(ctx->ep);
ctx->done = 1;
return NULL;
}
@ -1361,6 +1381,12 @@ int wait_all_with_timeout_for_test(pthread_t *threads, push_ctx_t *ctxs,
* promotion, heartbeat-driven convergence, controller join). Alloc or
* thread failures degrade to fewer pushes, never to a fabricated quorum
* (the barrier wait in push_config_to_standbys owns the quorum verdict).
* bug-1381 (CLOSED): each standby ep is endpoint_ref()'d under c->ha.lock
* before the pointer is snapshotted into ctxs[], matched by exactly one
* endpoint_unref() (from push_thread, or from this function's own cleanup
* if the thread never started) — so a concurrent RDHUP teardown can drop
* the OWNER's reference but never frees ep out from under an in-flight
* push_thread.
* Stores the registered standby count in *n_out (may be NULL); returns
* the number of standbys that ACKed the transport push. */
int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
@ -1397,11 +1423,22 @@ int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
ctxs[i].bytes = bytes;
ctxs[i].blen = blen;
ctxs[i].sha256 = sha256;
/* bug-1381: take our own reference on each standby ep while still
* holding c->ha.lock — the same window the pointer is snapshotted —
* mirroring the snapshot-lane fix (bug-1197/bug-1376). Without this,
* a concurrent RDHUP teardown (handle_endpoint_error -> zarray_del ->
* endpoint_killer -> endpoint_unref) can free ep while push_thread
* still reads/writes it for up to CONF_PUSH_TIMEOUT_MS. Every ref
* taken here has exactly one matching endpoint_unref: from push_thread
* on success, or from the cleanup below if the thread never started. */
endpoint_ref(ctxs[i].ep);
}
pthread_mutex_unlock(&c->ha.lock);
pthread_t *threads = malloc((size_t)n * sizeof(pthread_t));
if (!threads) {
for (int i = 0; i < n; i++)
endpoint_unref(ctxs[i].ep);
free(ctxs);
WARN("push_config_transport_only: thread alloc failed (n=%d)"
" — skipping transport push", n);
@ -1410,8 +1447,12 @@ int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
for (int i = 0; i < n; i++) {
ctxs[i].started = (pthread_create(&threads[i], NULL, push_thread, &ctxs[i]) == 0);
if (!ctxs[i].started)
if (!ctxs[i].started) {
WARN("push_config_transport_only: pthread_create failed for standby %d", i);
/* No thread will run push_thread's matching endpoint_unref —
* release our ref right here (bug-1381). */
endpoint_unref(ctxs[i].ep);
}
}
atomic_int push_done;
@ -2602,6 +2643,18 @@ int ha_send_snapshot_to_standby(endpoint_t *ep, const char *snap_dir,
break;
}
/* bug-1389 triage: crc32 of the exact bytes this chunk is about to
* send, taken right after the local file read and before encode —
* correlate against ha_handle_snapshot_chunk's receive-side crc32
* log (ha.c) by (snapshot_lsn, chunk_index) to localize whether a
* mismatch is introduced before send, in the wire transport, or
* during receive-side assembly. */
TRACE("ha_send_snapshot_to_standby: chunk crc32 "
"checkpoint_lsn=%" PRIu64 " chunk_index=%u offset=%" PRIu64
" len=%zu crc32=%08x",
checkpoint_lsn, chunk_index, offset, chunk_len,
(unsigned)crc32(0L, (const Bytef *)buf, (uInt)chunk_len));
int is_last = (offset + chunk_len >= total_size);
snapshot_transfer_t chunk;
@ -2620,6 +2673,13 @@ int ha_send_snapshot_to_standby(endpoint_t *ep, const char *snap_dir,
chunk.raw_payload = buf; /* borrowed for hook/encode only */
chunk.raw_payload_len = chunk_len;
/* bug-1386: a failed chunk send (e.g. -ECONNRESET while the standby
* endpoint is mid-teardown/reconnect) MUST stop the transfer and be
* propagated. Previously this return value was discarded, so the
* loop kept "sending" past a dead endpoint and the function still
* reported success — the standby silently received a truncated
* snap.part that never gets an is_last chunk. */
int send_rc = 0;
#ifdef XCE_TESTING
if (xce_snapshot_chunk_hook) {
xce_request_t req;
@ -2627,11 +2687,16 @@ int ha_send_snapshot_to_standby(endpoint_t *ep, const char *snap_dir,
req.opcode = OPCode_snapshotTransfer;
req.body_which = Request_body_snapshottransfermsg;
req.body.snapshottransfermsg = &chunk;
xce_snapshot_chunk_hook(ep, &req);
send_rc = xce_snapshot_chunk_hook(ep, &req);
} else
#endif
if (ep)
xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len);
send_rc = xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len);
if (send_rc != 0) {
rc = send_rc;
break;
}
offset += chunk_len;
chunk_index++;
@ -2645,6 +2710,11 @@ int ha_send_snapshot_to_standby(endpoint_t *ep, const char *snap_dir,
INFO("ha_send_snapshot_to_standby: snapshot transfer complete "
"checkpoint_lsn=%" PRIu64 " size=%" PRIu64,
checkpoint_lsn, total_size);
else
WARN("ha_send_snapshot_to_standby: snapshot transfer failed "
"checkpoint_lsn=%" PRIu64 " size=%" PRIu64 " sent=%" PRIu64
" chunk_index=%u rc=%d",
checkpoint_lsn, total_size, offset, chunk_index, rc);
return rc;
}