Rongsong Shen
2aefa939db
After an election where the vote connections are not promoted, both standbys register later via on_standby_connected; the own-term no-op was appended before any standby existed, and catch-up waited for heartbeat- driven state reports (config push one 3 s cycle, WAL stream the next), so the no-op committed ~6 s late and every client write in that window failed -110. ha_send_catchup_probe() sends the ordinary heartbeat request (same fields as primary_heartbeat_fn; encode + enqueue only; primary only) at registration and once after the standby's first accepted config-push ACK, so the decision loop's next action follows at ~RTT. The decision loop, its one-action-per-report pacing, commit/majority rules and the dispatch gate are unchanged; WAL streaming still runs only on the vtask. The one-shot arm is cleared on disconnect and on every become_leader / become_follower. Tests: test_late_joiner_catchup_kick.c (6). Controller suite 1285/0. Live: catch-up starts 0-6 ms after a late registration (was up to ~6 s); /xce-test-ha sequential 20 rounds, chaotic 6 rounds and sequential 8 rounds under 40-spinner load all PASS with zero job loss. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
347 lines
13 KiB
Meson
347 lines
13 KiB
Meson
controller_src = [
|
|
'cluster.c',
|
|
'commit.c',
|
|
'config_repl.c',
|
|
'ctrlstatus.c',
|
|
'dataex.c',
|
|
'egress.c',
|
|
'ha.c',
|
|
'ha_convergence.c',
|
|
'job.c',
|
|
'join.c',
|
|
'main.c',
|
|
'misc.c',
|
|
'myapp.c',
|
|
'opget.c',
|
|
'opupd.c',
|
|
'queue.c',
|
|
'reply.c',
|
|
'resources.c',
|
|
'sched.c',
|
|
'startup_validation.c',
|
|
'task.c',
|
|
'worker.c',
|
|
'apiserver.c',
|
|
'myconf.c',
|
|
'persist.c',
|
|
'repl.c',
|
|
'ctrl_metrics.c',
|
|
'slowlane.c',
|
|
'resync.c',
|
|
]
|
|
|
|
incdir = include_directories('.', '../include', '../lib',
|
|
'include', '../harness/include')
|
|
|
|
# cc = meson.get_compiler('c')
|
|
# math_dep = cc.find_library('m', required : false)
|
|
|
|
zlib_dep = dependency('zlib')
|
|
|
|
controller_deps = [
|
|
libxce_dep,
|
|
librtti_dep,
|
|
c_capnproto_dep,
|
|
math_dep,
|
|
tinylog_dep,
|
|
thread_dep,
|
|
zlib_dep,
|
|
]
|
|
|
|
controller = executable('xce_controller',
|
|
controller_src,
|
|
include_directories: incdir,
|
|
dependencies: controller_deps,
|
|
link_args: wrap_link_args,
|
|
install: true)
|
|
|
|
# Controller Unity test suite (persist types and future controller unit tests)
|
|
controller_unit_test_src = [
|
|
'testsuite/controller_test.c',
|
|
'testsuite/test_persist_types.c',
|
|
'testsuite/test_persist_wal.c',
|
|
'testsuite/test_persist_schema.c',
|
|
'testsuite/test_persist_checkpoint.c',
|
|
'testsuite/test_persist_recovery.c',
|
|
'testsuite/test_hooks.c',
|
|
'testsuite/test_persist_hooks.c',
|
|
'testsuite/test_gc_retention.c',
|
|
'testsuite/test_cluster_persist.c',
|
|
'testsuite/test_e2e_wal.c',
|
|
'testsuite/test_e2e_checkpoint.c',
|
|
'testsuite/test_e2e_hooks.c',
|
|
'testsuite/test_e2e_gc.c',
|
|
'testsuite/test_e2e_capnp.c',
|
|
'testsuite/test_ha_role.c',
|
|
'testsuite/test_ha_role_lifecycle.c',
|
|
'testsuite/test_ha_cluster.c',
|
|
'testsuite/test_ha_redirect.c',
|
|
'testsuite/test_join_redirect.c',
|
|
'testsuite/test_p1_integration.c',
|
|
'testsuite/test_repl_unit.c',
|
|
'testsuite/test_repl_apply.c',
|
|
'testsuite/test_persist_repl.c',
|
|
'testsuite/test_repl_catchup.c',
|
|
'testsuite/test_p2_integration.c',
|
|
'testsuite/test_p3_hardening.c',
|
|
'testsuite/test_p3_manual_failover.c',
|
|
'testsuite/test_ctrl_metrics.c',
|
|
'testsuite/test_config_repl.c',
|
|
'testsuite/test_worker_join.c',
|
|
'testsuite/test_ha_startup.c',
|
|
'testsuite/test_ha_election.c',
|
|
'testsuite/test_ha_elect_ctx_lifecycle.c',
|
|
'testsuite/test_ha_bootstrap.c',
|
|
'testsuite/test_ha_bootstrap_support.c',
|
|
'testsuite/test_ha_durability.c',
|
|
'testsuite/test_repl_majority.c',
|
|
'testsuite/test_persist_truncate.c',
|
|
'testsuite/test_ha_ctrlhello.c',
|
|
'testsuite/test_ha_failover.c',
|
|
'testsuite/test_ha_vote_send.c',
|
|
'testsuite/test_ha_start_election_async.c',
|
|
'testsuite/test_ha_vote_gate.c',
|
|
'testsuite/test_ha_repl_push.c',
|
|
'testsuite/test_ha_connection_reuse.c',
|
|
'testsuite/test_ha_rdhup_integration.c',
|
|
'testsuite/test_ha_liveness.c',
|
|
'testsuite/test_ha_prevote.c',
|
|
'testsuite/test_ha_failover_prevote.c',
|
|
'testsuite/test_commit_engine.c',
|
|
'testsuite/test_commit_barrier_rollout.c',
|
|
'testsuite/test_commit_barrier_reply.c',
|
|
'testsuite/test_jobsubmit_validate.c',
|
|
'testsuite/test_egress_filter.c',
|
|
'testsuite/test_demotion_convergence.c',
|
|
'testsuite/test_convergence_verification.c',
|
|
'testsuite/test_coldstart_lsn0.c',
|
|
'testsuite/test_commit_durability.c',
|
|
'testsuite/test_ctrlstatus.c',
|
|
'testsuite/test_ha_heartbeat_committed.c',
|
|
'testsuite/test_ha_vote_floor_heartbeat.c',
|
|
'testsuite/test_term_safety_adapter.c',
|
|
'testsuite/test_drive_async_config.c',
|
|
'testsuite/test_slowlane.c',
|
|
'testsuite/test_commit_admission_pool.c',
|
|
'testsuite/test_commit_resolve.c',
|
|
'testsuite/test_ha_vep_teardown.c',
|
|
'testsuite/test_ha_dispatch_integration.c',
|
|
'testsuite/test_term_adoption_vectors.c',
|
|
'testsuite/test_ha_core_safety.c',
|
|
'testsuite/test_ha_partition.c',
|
|
# xce-harness-component task 1.1: these 4 of the 15 files relocated to
|
|
# src/harness/ are NOT purely xce_test-private -- test_config_repl.c/
|
|
# test_bug857_regression.c call snapdiff_paths() (snapdiff.c),
|
|
# test_ha_core_safety.c calls waldump_path_visit()/walcompare()
|
|
# (waldump.c/walcompare.c), and test_upgrade_snapshot_apply.c calls
|
|
# snapverify_path() (snapverify.c) -- a real cross-dependency the
|
|
# spec's own research.md audit missed (it only checked what xce_test's
|
|
# sources #include, not who #includes xce_test's headers back). Their
|
|
# own white-box test pairs (test_waldump.c etc.) DID move to the
|
|
# harness's future harness_unit_test (task 2.1) and are correctly gone
|
|
# from this list below; these 4 production files themselves stay
|
|
# compiled a second time here too, exactly mirroring today's existing
|
|
# dual-compilation pattern (recompiled from their new ../harness/
|
|
# location -- see incdir's added '../harness/include' entry above).
|
|
'../harness/waldump.c',
|
|
'../harness/walcompare.c',
|
|
'../harness/snapverify.c',
|
|
'../harness/snapdiff.c',
|
|
'testsuite/test_persist_snapshot_entities.c',
|
|
'testsuite/test_apply_seam.c',
|
|
'testsuite/test_apply_committed_entry.c',
|
|
'testsuite/test_replay_apply_seam.c',
|
|
'testsuite/test_wal_job_dispatch_fact.c',
|
|
'testsuite/test_wal_job_start_fact.c',
|
|
'testsuite/test_wal_job_finish_fact.c',
|
|
'testsuite/test_wal_job_fail_fact.c',
|
|
'testsuite/test_phase2_cold_boot.c',
|
|
'testsuite/test_persist_encryption.c',
|
|
'testsuite/test_config_queue_reconcile.c',
|
|
'testsuite/test_wal_job_rusage_fact.c',
|
|
'testsuite/test_wal_job_ckpt_fact.c',
|
|
'testsuite/test_wal_job_preason_fact.c',
|
|
'testsuite/test_job_dispatch_fail_count.c',
|
|
'testsuite/test_apply_reapply_idempotency.c',
|
|
'testsuite/test_apply_correctness_verification.c',
|
|
'testsuite/test_apply_pump.c',
|
|
'testsuite/test_apply_pump_integration.c',
|
|
'testsuite/test_persist_checkpoint_offload.c',
|
|
'testsuite/test_job_array_apply.c',
|
|
'testsuite/test_job_array_fence.c',
|
|
'testsuite/test_job_array_replay_fence.c',
|
|
'testsuite/test_job_dispatch.c',
|
|
'testsuite/test_dispatch_commit.c',
|
|
'testsuite/test_dispatch_fail_terminal.c',
|
|
'testsuite/test_dispatch_fail_retry.c',
|
|
'testsuite/test_jobdispatch_no_double_notify.c',
|
|
'testsuite/test_worker_resync.c',
|
|
'testsuite/test_standby_state_peer_idx.c',
|
|
'testsuite/test_late_joiner_catchup_kick.c',
|
|
'testsuite/test_upgrade_snapshot_restore.c',
|
|
'testsuite/test_upgrade_snapshot_apply.c',
|
|
'testsuite/test_bug857_regression.c',
|
|
'testsuite/test_sched_admin_dispatch.c',
|
|
'testsuite/test_jobstatereport_dispatch.c',
|
|
'testsuite/test_fetchfile_disconnect.c',
|
|
'ctrlstatus.c',
|
|
'commit.c',
|
|
'job.c',
|
|
'egress.c',
|
|
'ha_convergence.c',
|
|
'queue.c',
|
|
'join.c',
|
|
'opupd.c',
|
|
'startup_validation.c',
|
|
'worker.c',
|
|
'persist.c',
|
|
'ha.c',
|
|
'repl.c',
|
|
'ctrl_metrics.c',
|
|
'config_repl.c',
|
|
'slowlane.c',
|
|
'cluster.c',
|
|
'resync.c',
|
|
]
|
|
|
|
ssl_dep = dependency('openssl')
|
|
|
|
controller_unit_test_deps = [
|
|
libxce_dep,
|
|
librtti_dep,
|
|
unity_dep,
|
|
tinylog_dep,
|
|
thread_dep,
|
|
zlib_dep,
|
|
c_capnproto_dep,
|
|
ssl_dep,
|
|
struct2json_dep, # testsuite/listfmt_json.c + test_listfmt_json.c: cJSON
|
|
# (xce-test-cli-ux task 2.2)
|
|
]
|
|
|
|
controller_unit_test = executable('controller_unit_test',
|
|
controller_unit_test_src,
|
|
include_directories: [incdir],
|
|
dependencies: controller_unit_test_deps,
|
|
c_args: ['-DXCE_TESTING'],
|
|
install: false)
|
|
|
|
test('controller test', controller_unit_test,
|
|
suite: 'controller',
|
|
timeout: 300,
|
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
|
|
|
# job-array-single-wal-record task 5.1: sched.c's push logic (sched_job_new_fun /
|
|
# xcesched_push_jobarray) is exercised by a small, STANDALONE test binary rather
|
|
# than being folded into controller_unit_test_src above. sched.c/dataex.c are not
|
|
# part of that binary today -- several existing files there (test_commit_barrier_
|
|
# reply.c, test_worker_join.c) already provide their OWN local fake definitions
|
|
# of on_scheduler_join()/notify()/collect_confvars(), precisely because the real
|
|
# sched.c/dataex.c were never linked in. Linking the real ones alongside those
|
|
# fakes collides at link time (multiple definition). This binary links the real
|
|
# sched.c + its real dataex.c/egress.c dependencies, isolated from that
|
|
# collision; only opget.c's one-line check_worker_() is stubbed locally in the
|
|
# test file (linking all of opget.c would pull in its own further dependencies
|
|
# this test does not need).
|
|
sched_push_unit_test_src = [
|
|
'testsuite/test_job_array_sched_push.c',
|
|
'sched.c',
|
|
'dataex.c',
|
|
'egress.c',
|
|
]
|
|
|
|
sched_push_unit_test = executable('sched_push_unit_test',
|
|
sched_push_unit_test_src,
|
|
include_directories: [incdir],
|
|
dependencies: controller_unit_test_deps,
|
|
c_args: ['-DXCE_TESTING'],
|
|
install: false)
|
|
|
|
test('sched push test', sched_push_unit_test,
|
|
suite: 'controller',
|
|
timeout: 60,
|
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
|
|
|
# bug-1578 regression test: sched_worker_change()'s dangling-pointer fix
|
|
# (w->host.resvals_to_sched/nresvals_to_sched left aliased onto a freed
|
|
# xlist_t backing array -- see sched.c's sched_worker_change() for the full
|
|
# history). Same real-vs-fake link-collision reason as sched_push_unit_test
|
|
# just above requires its own standalone binary rather than folding into
|
|
# controller_unit_test_src: sched_worker_change() is only reachable by
|
|
# linking the REAL sched.c/dataex.c/egress.c, which collides with other
|
|
# controller_unit_test_src files' own fake on_scheduler_join()/notify()
|
|
# definitions. Mirrors sched_push_unit_test_src's exact source-set strategy
|
|
# (including opget.c's check_worker_() being stubbed locally rather than
|
|
# linking all of opget.c).
|
|
sched_worker_change_unit_test_src = [
|
|
'testsuite/test_sched_worker_change.c',
|
|
'sched.c',
|
|
'dataex.c',
|
|
'egress.c',
|
|
]
|
|
|
|
sched_worker_change_unit_test = executable('sched_worker_change_unit_test',
|
|
sched_worker_change_unit_test_src,
|
|
include_directories: [incdir],
|
|
dependencies: controller_unit_test_deps,
|
|
c_args: ['-DXCE_TESTING'],
|
|
install: false)
|
|
|
|
test('sched worker change test', sched_worker_change_unit_test,
|
|
suite: 'controller',
|
|
timeout: 60,
|
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
|
|
|
# job-array-single-wal-record task 5.2: apiserver.c's push logic
|
|
# (apiserver_job_new_fun / apiserver_push_jobarray) faces the identical
|
|
# real-vs-fake link collision task 5.1 hit for sched.c (test_commit_barrier_
|
|
# reply.c / test_worker_join.c already define their OWN fake notify()/
|
|
# on_scheduler_join(), so linking the real dataex.c into controller_unit_test
|
|
# collides at link time) -- so this is its own standalone binary too,
|
|
# mirroring sched_push_unit_test's strategy. UNLIKE sched.c, apiserver.c
|
|
# never calls job_submit_committed()/collect_jobs_committed() (egress.c), so
|
|
# this binary does NOT need egress.c -- verified by grep, not assumed
|
|
# identical to sched.c's dependency set.
|
|
apiserver_push_unit_test_src = [
|
|
'testsuite/test_job_array_apiserver_push.c',
|
|
'apiserver.c',
|
|
'dataex.c',
|
|
]
|
|
|
|
apiserver_push_unit_test = executable('apiserver_push_unit_test',
|
|
apiserver_push_unit_test_src,
|
|
include_directories: [incdir],
|
|
dependencies: controller_unit_test_deps,
|
|
c_args: ['-DXCE_TESTING'],
|
|
install: false)
|
|
|
|
test('apiserver push test', apiserver_push_unit_test,
|
|
suite: 'controller',
|
|
timeout: 60,
|
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
|
|
|
# bug-1735: misc.c's update_host_runusage()/aggregate_job_runusage() are
|
|
# small, dependency-light functions (calloc/realloc/strcmp only, no RTTI
|
|
# locking, no networking) -- standalone binary linking the REAL misc.c
|
|
# directly, mirroring resval_typeresolve_unit_test's isolation rationale
|
|
# above. get_ep_worker()/get_ep_role() (same file, RTTI-backed) are unused
|
|
# by these tests but still link fine via controller_unit_test_deps' existing
|
|
# librtti_dep.
|
|
host_runusage_unit_test_src = [
|
|
'testsuite/test_host_runusage.c',
|
|
'misc.c',
|
|
]
|
|
|
|
host_runusage_unit_test = executable('host_runusage_unit_test',
|
|
host_runusage_unit_test_src,
|
|
include_directories: [incdir],
|
|
dependencies: controller_unit_test_deps,
|
|
c_args: ['-DXCE_TESTING'],
|
|
install: false)
|
|
|
|
test('host runusage test', host_runusage_unit_test,
|
|
suite: 'controller',
|
|
timeout: 60,
|
|
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
|
|
|
|