Rongsong Shen
0cad68507f
A worker reported job start, finish and failure with a fire-and-forget
OPCode_notify the controller never answers, so a report lost while the
worker stayed connected (the live incident: 6 of 7 finish reports in a
burst) left the job RUN forever; the rejoin resync (165fc158) only
recovers reports lost around a reconnect.
- controller.capnp: OPCode_notifyAcked -- the same NotifyData body,
answered. A separate opcode because replies are matched through the
client's single wbuf, not by sequence number: answering plain notifies
would feed stale statuses to the next synchronous request.
- controller (cluster.c): do_notify_acked() applies the fact like
do_notify, then waits on the commit barrier and replies 0 only once it
is majority-committed. A barrier failure (the fact is still appended and
retained) replies -E_XCE_NOT_READY, so the worker retries in place
instead of rotating away from a healthy primary (cf. bug-1850). Admitted
by the no-quorum fact-bearing gate like OPCode_notify.
- libxce: xce_notify_acked_req/xce_notify_acked (send, wait for the
status) and xcecli_notify_acked (XCECLI_FAILOVER_CALL retry/failover).
xcecli_notify_jobstart/jobfinish/jobfail use it; on -E_XCE_BAD_OPCODE
(a controller predating the opcode) they fall back to the plain
fire-and-forget notify, so workers and controllers can be upgraded in
either order.
- Resends are safe: START/FINISH are duplicate-guarded and a repeated FAIL
yields the same state.
- worker: logs a WARN when a report is still unacknowledged after the
retry budget (the rejoin resync remains the backstop). These calls run
on the worker daemon's IPC callback, so while the controller lacks
quorum other jobs' IPC messages wait (buffered, not lost) up to the
retry budget.
Tests: test_notify_acked_reply_only_after_commit and
test_notify_acked_withheld_replies_not_ready (no reply before the covering
ACK; -E_XCE_NOT_READY with the fact retained when ACKs are withheld).
Written with the opcode, so no separate RED run. Full /xce-build all:
111 suites pass; controller suite 1245/0.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
52 lines
2.5 KiB
C
52 lines
2.5 KiB
C
#if !defined(_XCEAPI_H_)
|
|
|
|
#define _XCEAPI_H_ 1
|
|
|
|
#include "config.h"
|
|
|
|
#include "xce/types.h"
|
|
#include "xcectx.h"
|
|
|
|
extern int xce_hello(xce_ctx_t*, char*);
|
|
extern int xce_nop(xce_ctx_t*);
|
|
extern int xce_heartbeat(xce_ctx_t*, int);
|
|
extern int xce_shutdown(xce_ctx_t*);
|
|
extern int xce_reslist(xce_ctx_t*, xce_resource_t***);
|
|
extern int xce_join(xce_ctx_t*, int, hostattrs_t*);
|
|
extern int xce_workerlist(xce_ctx_t*, xce_worker_t***);
|
|
extern int xce_jobdispatch(xce_ctx_t*, xce_job_dispatch_t*);
|
|
extern int xce_jobsubmit(xce_ctx_t*, xce_job_submit_t*, uint64_t*);
|
|
extern int xce_joblist(xce_ctx_t*, xce_job_t***);
|
|
extern int xce_clusterconf(xce_ctx_t*, xce_clusterconf_t**);
|
|
extern int xce_ctrlstatus(xce_ctx_t*, xce_ctrlstatus_t**);
|
|
extern int xce_push(xce_ctx_t*, xce_check_fn_t, void*, int, int, int, void**);
|
|
extern int xce_update_rows(xce_ctx_t*, int, int, xlist_t*);
|
|
extern int xce_resval(xce_ctx_t*, char*, char*, xce_resval_t***);
|
|
|
|
extern int xce_notify(xce_ctx_t*, xce_notify_data_t*);
|
|
/* bug-1741: send a notify and wait for the controller's commit ACK. */
|
|
extern int xce_notify_acked(xce_ctx_t*, xce_notify_data_t*);
|
|
extern int xce_update_resvals(xce_ctx_t*, int, xce_resval_t**);
|
|
extern int xce_jobctrl(xce_ctx_t*, xce_jobctrl_t*);
|
|
extern int xce_queuectrl(xce_ctx_t*, xce_queuectrl_t*);
|
|
extern int xce_hostctrl(xce_ctx_t*, xce_hostctrl_t*);
|
|
/* xce_upgradeconfig: send OP_upgradeConfig (sha256 + serialised ClusterConfig
|
|
* blob) via xce_upgradeconfig_req_raw and wait for the reply. Returns the
|
|
* server's reply status (0 on success; a negative errno/E_XCE_* code on
|
|
* failure — the server logs the human-readable reason, see
|
|
* do_upgrade_config_with_bytes/send_upgrade_error, config_repl.c). Mirrors
|
|
* xce_hostctrl/xce_queuectrl's shape; the richer UpgradeConfigReply fields
|
|
* (version_tag/acked_count/error text) are not surfaced to the caller — no
|
|
* consumer of this API currently needs them beyond success/failure. */
|
|
extern int xce_upgradeconfig(xce_ctx_t*, const uint8_t *sha256,
|
|
size_t sha256_len, const uint8_t *bytes,
|
|
size_t bytes_len);
|
|
extern int xce_runtask(xce_ctx_t*, xce_taskrun_t*);
|
|
extern int xce_update_jusage(xce_ctx_t*, uint64_t, int, int*, int, int*, int,
|
|
xce_rusage_t**);
|
|
extern int xce_set_pend_reason(xce_ctx_t*, xce_preason_set_t*);
|
|
extern int xce_clear_reason(xce_ctx_t*, xce_clear_reason_t*);
|
|
extern int xce_jobinfo(xce_ctx_t*, uint64_t, xce_job_t**);
|
|
extern int xce_fetchfile(xce_ctx_t*, xce_fetchfile_t*, xce_fetchfile_reply_t**);
|
|
|
|
#endif
|