finalize configuration replication

2026-06-09 16:40:13 +08:00 · 2026-06-09 16:40:13 +08:00 · fa66d1ba9c
commit fa66d1ba9c
parent e1517bbf28
30 changed files with 8867 additions and 1560 deletions
--- a/src/.claude/ucai-formatter-cache.local.json
+++ b/src/.claude/ucai-formatter-cache.local.json
@ -1 +0,0 @@
-{"detected":false,"exe":null,"args":null,"extensions":[]}
--- a/src/controller/CLAUDE.md
+++ b/src/controller/CLAUDE.md
@ -160,7 +160,7 @@ VERBOSE=-vvv /tmp/xce-deploy/scripts/start.sh ctrl-3 &
 # Primary elected and config pushed
 grep -i "CTRL_PRIMARY\|Config committed" /tmp/xce-deploy/logs/ctrl-1.log | grep -v DEBUG3
 # Config files present on standbys
-find /tmp/xce-deploy/ctrl-2/conf /tmp/xce-deploy/ctrl-3/conf -type f
+find /tmp/xce-deploy/controller/ctrl-2/conf /tmp/xce-deploy/controller/ctrl-3/conf -type f
 ```

 **Clean restart** (keeps ctrl-1 config + all certs, wipes data/standby-conf/logs):
--- a/src/controller/ha.c
+++ b/src/controller/ha.c
@ -3544,6 +3544,9 @@ static void on_standby_connected(endpoint_t *ep)
        pthread_mutex_unlock(&ha->lock);
    }

+    /* Reset failure counter so the next disconnect restarts logging from attempt 1. */
+    ha->peer_connect_fail_count[sctx->peer_idx] = 0;
+
    /* Record last_push_time before any send so the first heartbeat tick after
     * connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */
    {
@ -3639,10 +3642,18 @@ static void ha_connect_to_peer(cluster_t *c, int peer_idx)
    endpoint_set_on_connected(ep, on_standby_connected);
    endpoint_set_err_handler(ep, standby_ep_err_handler_primary);

+    int n = ++ha->peer_connect_fail_count[peer_idx];
+    int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
+
+    if (should_log)
+        INFO("HA: peer[%d] (%s) not connected — initiating outbound connect (attempt %d)",
+             peer_idx, peer->hostname, n);
+
    int rc = endpoint_connect(ep, peer->hostname, peer->port);
    if (rc < 0) {
-        WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
-             peer_idx, peer->hostname, peer->port, rc);
+        if (should_log)
+            WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
+                 peer_idx, peer->hostname, peer->port, rc);
        /* endpoint_fini calls FREEUP(ep->data) which frees sctx — do not
         * double-free sctx here. */
        ep->data = NULL;
@ -3754,11 +3765,8 @@ static int primary_heartbeat_fn(void *arg)
                pthread_mutex_unlock(&ha->lock);
            }

-            if (!found) {
-                INFO("HA: peer[%d] (%s) not connected — initiating outbound connect",
-                     i, pcfg->candidates[i].name ? pcfg->candidates[i].name : "?");
+            if (!found)
                ha_connect_to_peer(c, i);
-            }
        }
    }

--- a/src/controller/ha.h
+++ b/src/controller/ha.h
@ -123,6 +123,8 @@ typedef struct {
 #define HA_DEFAULT_HEARTBEAT_TIMEOUT_MS      10000
 #define HA_DEFAULT_ELECTION_TIMEOUT_MS       5000
 #define HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS  500
+/* Log the first connect failure per peer, then every Nth attempt. */
+#define HA_CONNECT_LOG_THROTTLE              10
 #define HA_DEFAULT_MAX_PROMOTION_RETRIES     5
 #define HA_DEFAULT_ELECTION_JITTER_MAX_MS    200
 #define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500
@ -189,6 +191,7 @@ typedef struct {
    uint64_t         standby_reported_tag[MAX_HA_CANDIDATES]; /* Last config_tag reported by each standby */
    uint64_t         standby_reported_lsn[MAX_HA_CANDIDATES]; /* Last applied_lsn reported by each standby */
    bool             standby_converged[MAX_HA_CANDIDATES];    /* Whether standby[i] has converged with primary */
+    int              peer_connect_fail_count[MAX_HA_CANDIDATES]; /* throttle log spam on repeated connect failure */
 } ha_state_t;

 /* ── Public API ────────────────────────────────────────────── */
--- a/src/controller/main.c
+++ b/src/controller/main.c
@ -28,6 +28,7 @@ const char* argp_program_bug_address = BUG_EMAIL;
 #define OPT_CONF_KEEP_N        257
 #define OPT_CONF_STALE_TIMEOUT 258
 #define OPT_VCPUS              259
+#define OPT_CERTDIR            260

 struct arguments {
    char* myname;
@ -38,6 +39,7 @@ struct arguments {
    int   conf_keep_n;
    long  conf_stale_timeout;
    int   vcpus;
+    char* certdir;
 };

 static struct argp_option options[] = {
@ -54,6 +56,8 @@ static struct argp_option options[] = {
      "Seconds before saved config is flagged stale (0=disabled, default: 86400)" },
    { "vcpus",              OPT_VCPUS,               "N",    0,
      "Number of virtual CPUs for the task manager (must be >= 1)" },
+    { "certdir",            OPT_CERTDIR,  "DIR", 0,
+      "Directory containing xce.crt and xce.key (default: <workdir>/certs)" },
    { 0 }
 };

@ -93,6 +97,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
        if (validate_vcpus(arguments->vcpus) != 0)
            argp_error(state, "--vcpus must be >= 1 (got: %s)", arg);
        break;
+    case OPT_CERTDIR:
+        arguments->certdir = arg;
+        break;
    default:
        return ARGP_ERR_UNKNOWN;
    }
@ -147,8 +154,15 @@ int main(int argc, char* argv[])

    INFO("My name is %s", arguments.myname);

-    snprintf(key_file, XCE_MAXPATH_LEN, "%s/certs/xce.key", arguments.workdir);
-    snprintf(cert_file, XCE_MAXPATH_LEN, "%s/certs/xce.crt", arguments.workdir);
+    const char *certdir = arguments.certdir;
+    char certdir_buf[XCE_MAXPATH_LEN + 1];
+    if (!certdir) {
+        snprintf(certdir_buf, sizeof(certdir_buf),
+                 "%s/certs", arguments.workdir);
+        certdir = certdir_buf;
+    }
+    snprintf(key_file, XCE_MAXPATH_LEN, "%s/xce.key", certdir);
+    snprintf(cert_file, XCE_MAXPATH_LEN, "%s/xce.crt", certdir);

    myconf.cluster_config    = NULL;
    myconf.conffile          = NULL; /* set by load_primary_config() or fetch_config_from_primary() */
@ -180,10 +194,10 @@ int main(int argc, char* argv[])
            return 1;
        }

-        /* CA cert derived from workdir — same infrastructure as certfile/keyfile */
+        /* CA cert from certdir (same certificate as certfile) */
        snprintf(myconf.peers_cfg.ca_cert_path,
                 sizeof(myconf.peers_cfg.ca_cert_path),
-                 "%s/certs/xce.crt", arguments.workdir);
+                 "%s/xce.crt", certdir);
        strncpy(myconf.peers_cfg.identity, arguments.myname,
                sizeof(myconf.peers_cfg.identity) - 1);
        INFO("HA mode: %d controller candidates, identity=%s",
--- a/src/docs/cluster-config-split-analysis.md
+++ b/src/docs/cluster-config-split-analysis.md
@ -0,0 +1,272 @@
+# ClusterConfig Schema Split Analysis
+
+> Analysis date: 2026-06-08. Branch: persistent_ha.
+> Source schema: `src/schema/controller.capnp:680`, struct `ClusterConfig`.
+
+## Summary
+
+The `ClusterConfig` struct (20 fields) mixes three distinct configuration concerns:
+cluster-wide business config, HA peer membership, and per-controller performance
+tunables. These should be split into separate objects so each can evolve
+independently without forcing a re-push of the full config blob to all standbys.
+
+## Current State Discovery
+
+### Peer list is already structurally separate
+
+The `--peers` CLI flag (`main.c:178-210`) parses into `myconf.peers_cfg`
+(`xce_cluster_config_t`), which is a **separate struct** from the config blob's
+`cluster_config_t`. At startup, `cluster_init_phase1` uses `peers_cfg.candidates`
+to determine `my_name`, `my_port`, `my_priority`, and the peer list — all before
+the config blob is loaded (`cluster.c:865-880`). The config blob's `candidates`
+field is **not consulted** for HA peer discovery; it is carried for backward
+compatibility in the blob but superseded by `--peers`.
+
+### HA tunables are hardcoded at phase1
+
+`cluster_init_phase1` uses compile-time defaults for all HA timing parameters
+(`cluster.c:854-863`):
+```c
+ha_config_t ha_cfg = {
+    .heartbeat_interval_ms     = HA_DEFAULT_HEARTBEAT_INTERVAL_MS,
+    .heartbeat_timeout_ms      = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS,
+    .election_timeout_ms       = HA_DEFAULT_ELECTION_TIMEOUT_MS,
+    .committing_timeout_ms     = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
+    ...
+};
+```
+
+The `heartbeatInterval` and `inactiveHeartbeatCount` fields in
+`ClusterConfig` are consumed later (e.g., `cluster.c:581-585` writes them into
+a config key-value store for runtime access), but the HA init path does not
+read them. The heartbeat timeout validation in `startup_validation.c:144` works
+against the hardcoded defaults, not the config blob values.
+
+### vcpus is already deprecated
+
+Field `@7 vcpus` is marked deprecated in the schema comment: "use --vcpus CLI
+flag; kept for config blob backward compatibility." `cluster_init_phase2`
+reads `c->myconf->vcpus` (set from CLI), not the blob's `vcpus` field
+(`cluster.c:1045`).
+
+## Proposed Split: Three Configuration Objects
+
+```
+                         ┌──────────────────────────────┐
+                         │    Current ClusterConfig      │
+                         │         (20 fields)           │
+                         └──────────────┬───────────────┘
+                                        │
+                    ┌───────────────────┼───────────────────┐
+                    ▼                   ▼                   ▼
+        ┌───────────────────┐ ┌─────────────────┐ ┌──────────────────────┐
+        │  ClusterConfig     │ │  PeersConfig     │ │  ControllerConfig    │
+        │  (cluster-wide)   │ │  (HA membership) │ │  (per-controller)    │
+        └───────────────────┘ └─────────────────┘ └──────────────────────┘
+```
+
+### Group 1: Cluster-Wide Business Config (keep in `ClusterConfig`)
+
+These fields define the cluster's behavior, resources, and policy. They **must
+be identical** across all controllers. When modified via `OP_upgradeConfig`,
+the new blob is pushed to all standbys and committed on majority.
+
+| Field | Ord | Type | Rationale |
+|-------|-----|------|-----------|
+| `clusterName` | @0 | Text | Cluster identity — must be identical cluster-wide |
+| `resources` | @8 | List(Resource) | Resource catalog — shared state across all nodes |
+| `queues` | @9 | List(QueueConf) | Queue definitions — cluster scheduling policy |
+| `ugroups` | @10 | List(GroupConfig) | User group membership — access control |
+| `hgroups` | @11 | List(GroupConfig) | Host group membership — scheduling constraints |
+| `hostConfigs` | @12 | List(HostConfig) | Per-host overrides — cluster-wide host knowledge |
+| `hproviders` | @18 | List(ScalerConfig) | Autoscaler provider config — cluster infrastructure |
+| `limits` | @19 | List(LimitConfig) | Resource limits — cluster policy enforcement |
+| `configGeneration` | @20 | UInt64 | Monotonic version counter — replication consistency |
+
+**How consumed**: `cluster_init_phase2` (after election and config load)
+iterates these to populate `c->resmgr`, `c->jobmgr`, queue tables, host
+groups, etc.
+
+### Group 2: HA Peer Membership (extract to `PeersConfig`)
+
+These fields define **who** is in the cluster. They change independently when
+controllers are added, removed, or re-IP'd — without any change to the
+business config.
+
+| Field | Ord | Type | Rationale |
+|-------|-----|------|-----------|
+| `candidates` | @2 | List(MasterCandidate) | HA peer list — changes independently of business config |
+| `port` | @1 | UInt32 | Fallback listener port — derived from candidates; per-controller in practice |
+
+**Already structurally separate**: The `--peers` CLI flag populates
+`myconf.peers_cfg` (`xce_cluster_config_t` in `src/include/clusterconfig.h`),
+a struct distinct from the config blob's `cluster_config_t`. The candidates
+list in the blob is carried for backward compatibility but is not read for HA
+peer discovery at startup. Making the separation formal means:
+- `PeersConfig` travels via its own capnp message type and replication path
+- Cluster config upgrades via `OP_upgradeConfig` no longer need to carry the candidate list
+- Peer list changes (add/remove controllers) are a dedicated operation with its own quorum semantics
+
+**Candidate list has special quorum requirements**: Changing the peer set is a
+Raft cluster membership change. It must not use the same
+`repl_wait_majority_ack` path as business config because the majority
+denominator itself is changing. A separate message type enables a two-phase
+membership change protocol (joint consensus) in the future.
+
+**Impact of removing `candidates` from `ClusterConfig` (verified against codebase):**
+
+`cluster_build_config()` (`cluster.c:575–609`) is the **only** place that reads
+`cluster_config->candidates` and `cluster_config->ncandidates` — it copies them
+into the runtime KV store as `ncandidates` and `candidate_N_{name,host,port}`.
+No other component (scheduler, apiserver, opget, sched) reads those KV keys.
+The live peer list used by `ha.c:3266–3272` and `cluster_open_listener` comes
+exclusively from `peers_cfg.candidates[]` (parsed by `--peers`). Removing
+candidates from the blob is therefore a **zero-blast-radius** change.
+
+| File | Location | Change |
+|------|----------|--------|
+| `src/controller/cluster.c` | 587–606 | Remove `ncandidates` + `candidate_N_*` KV loop from `cluster_build_config()` |
+| `src/include/clusterconfig.h` | 34–35 | Remove `int ncandidates` and `master_candidate_t** candidates` from `cluster_config_t` |
+| `src/schema/controller.capnp` | 683 | Remove `candidates @2: List(MasterCandidate)` from `ClusterConfig` |
+
+**`MasterCandidate` fields**:
+```
+struct MasterCandidate {
+  name  @0: Text;    // controller identity (e.g., "ctrl-1")
+  host  @1: Text;    // hostname or IP
+  port  @2: UInt32;  // peer listener port
+}
+```
+
+### Group 3: Per-Controller Performance Tunables (extract to `ControllerConfig`)
+
+These fields configure **how** an individual controller process behaves. They
+can legitimately differ across controllers (e.g., a lower-memory node with
+fewer vcpus, or a WAN-connected standby with a longer heartbeat timeout).
+
+| Field | Ord | Type | Rationale |
+|-------|-----|------|-----------|
+| `heartbeatInterval` | @3 | UInt32 | Primary heartbeat send interval — can be tuned per primary |
+| `inactiveHeartbeatCount` | @4 | UInt32 | Missed-heartbeat tolerance multiplier |
+| `fetchJobInterval` | @5 | UInt32 | Job fetch poll interval — local scheduler interaction |
+| `netioTimeout` | @6 | UInt32 | Network I/O timeout — environment-specific |
+| `vcpus` | @7 | UInt32 | **Already deprecated** — use `--vcpus` CLI flag; remove from schema |
+| `jobmgr` | @13 | HashZArrayConfig | Job manager hash table sizing (nHashSlots, delta) |
+| `workermgr` | @14 | HashZArrayConfig | Worker manager hash table sizing |
+| `qmgr` | @15 | HashZArrayConfig | Queue manager hash table sizing |
+| `resmgr` | @16 | ResmgrConfig | Resource manager hash + instance table sizing |
+| `xstrings` | @17 | HashZArrayConfig | String interning table sizing |
+
+**How consumed**: 
+- `jobmgr`/`workermgr`/`qmgr`/`resmgr`/`xstrings` are read once at
+  `cluster_init_phase2` to construct the internal key-value stores
+  (`cluster.c:803-808`). After construction, the config values are not referenced again.
+- `heartbeatInterval`/`inactiveHeartbeatCount` are read at runtime by the
+  heartbeat and liveness tasks (`cluster.c:1180, 1195`).
+- `fetchJobInterval` drives the scheduler polling loop.
+- `netioTimeout` is used by the iomgr layer for TCP timeouts.
+
+**Why these can differ per controller**: A controller on a high-memory machine
+can be configured with larger hash tables. A controller on a high-latency WAN
+link can have a longer netioTimeout. These differences do not affect cluster
+correctness — they only affect local process behavior.
+
+**`vcpus` removal**: Already deprecated. The `--vcpus` CLI flag
+(`main.c:1045`) is the canonical source. Remove field @7 in the next
+schema revision.
+
+## Fields With Dual Usage (require care)
+
+### `port` (@1)
+
+**Decision: Remove. No fallback bind port.**
+
+`cluster_config->port` currently serves as a fallback in `cluster_open_listener`
+(`cluster.c:953–975`) when `--peers` does not specify a port for the matching
+candidate. This fallback is removed: the `--peers` port is mandatory.
+
+**Affected code (all changes mechanical):**
+
+| File | Location | Change |
+|------|----------|--------|
+| `src/controller/cluster.c` | 953–975 | Remove the `if (my_port == 0) { cluster_config->port }` branch; return `EINVAL` immediately if no port found in `peers_cfg` |
+| `src/controller/myconf.c` | 74 | Remove `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` |
+| `src/controller/config_repl.c` | 55 | Remove `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` |
+| `src/include/clusterconfig.h` | 33 | Remove `int port` from `cluster_config_t` |
+| `src/schema/controller.capnp` | 682 | Remove `port @1: UInt32` from `ClusterConfig` |
+
+Note: `"port"` in `testsuite/test.c:124,266` is the scheduler/worker
+connect-to-primary port (via `get_setting("port")`), not the listener bind
+port. It is unaffected.
+
+## Implementation Approach
+
+### Phase 1: Remove `port` and `candidates` from `ClusterConfig` (immediate)
+
+No backward-compat transition needed for these two fields — the live code
+already ignores both (peer discovery uses `peers_cfg`; port fallback is being
+deleted). All changes are mechanical.
+
+1. `src/schema/controller.capnp` — Remove `port @1` and `candidates @2` from
+   `ClusterConfig`; mark or remove `MasterCandidate` struct if no longer
+   referenced elsewhere
+2. `src/include/clusterconfig.h` — Remove `int port`, `int ncandidates`,
+   `master_candidate_t** candidates` from `cluster_config_t`
+3. `src/controller/cluster.c` — Remove `if (my_port == 0)` fallback block in
+   `cluster_open_listener` (lines 967–975); remove `ncandidates` + `candidate_N_*`
+   KV loop in `cluster_build_config` (lines 587–606)
+4. `src/controller/myconf.c` and `src/controller/config_repl.c` — Remove
+   `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` from `fill_default_cluster_config`
+5. Regenerate capnp bindings
+
+### Phase 2: Schema split (wire-compatible)
+
+1. Define `PeersConfig` and `ControllerConfig` structs in `controller.capnp`
+2. Add them as new fields in the top-level `ConfigData` union alongside
+   the existing `clusterconfig @2`
+3. Regenerate capnp bindings
+4. **Keep the remaining `ClusterConfig` fields intact** during transition —
+   new fields are additive
+
+### Phase 3: Controller reads from new structs
+
+1. `cluster_init_phase2`: read `ControllerConfig` for tunables
+2. `cluster_config_t` retains business config fields only
+3. Fall back to old `ClusterConfig` fields when new structs are absent
+   (backward compatibility for existing `cluster.bin` files)
+
+### Phase 4: Remove superseded tunables from `ClusterConfig`
+
+1. Remove `heartbeatInterval`, `inactiveHeartbeatCount`, `fetchJobInterval`,
+   `netioTimeout`, `vcpus` from `ClusterConfig`
+2. Remove `jobmgr`, `workermgr`, `qmgr`, `resmgr`, `xstrings` from
+   `ClusterConfig`
+
+### Phase 5 (future): Peer list replication
+
+1. Define `OPCode_peersUpdate` / `OPCode_peersUpdateAck` for peer list
+   replication with joint-consensus semantics
+2. Cluster membership changes become a dedicated operation with its own
+   quorum protocol
+
+## Effort and Risk
+
+| Phase | Effort | Risk | Justification |
+|-------|--------|------|---------------|
+| 1 — Remove port + candidates | XS (½d) | Low | No transition needed; dead code removal + schema field removal |
+| 2 — Schema split | S (1d) | Low | Additive capnp change; old fields preserved |
+| 3 — Read from new structs | M (2-3d) | Medium | Touches cluster_init paths; backward compat needed |
+| 4 — Remove superseded tunables | S (1d) | Low | Mechanical removal after Phase 3 is validated |
+| 5 — Peer replication | L (1-2wk) | High | New protocol; joint consensus; extensive testing |
+
+## References
+
+- `src/schema/controller.capnp:680-700` — `ClusterConfig` definition
+- `src/schema/controller.capnp:253` — `ConfigData.clusterconfig @2`
+- `src/controller/main.c:178-213` — `--peers` CLI parsing into `peers_cfg`
+- `src/controller/cluster.c:854-880` — `ha_config_t` hardcoded defaults + priority derivation
+- `src/controller/cluster.c:945-977` — port derivation from candidates vs config blob
+- `src/controller/cluster.c:803-808` — internal manager construction
+- `src/controller/ha.h:134` — `ha_config_t.heartbeat_interval_ms` default via `ha_init`
+- `src/controller/startup_validation.c:133-163` — heartbeat config validation
+- `src/include/clusterconfig.h` — `xce_cluster_config_t` (separate peers struct)
--- a/src/tools/deploy-ha.sh
+++ b/src/tools/deploy-ha.sh
@ -22,8 +22,8 @@ mkdir -p "$TOP/certs"
 mkdir -p "$TOP/logs"
 mkdir -p "$TOP/scripts"
 for i in $(seq 1 $N); do
-    mkdir -p "$TOP/ctrl-$i/conf"
-    mkdir -p "$TOP/ctrl-$i/data"
+    mkdir -p "$TOP/controller/ctrl-$i/conf"
+    mkdir -p "$TOP/controller/ctrl-$i/data"
 done

 # Copy certificates
@ -32,8 +32,8 @@ cp "$DATADIR/xce.key" "$TOP/certs/"

 # Tag and copy cluster.bin
 TAG=$(sha256sum "$DATADIR/cluster.bin" | cut -c1-12)
-cp "$DATADIR/cluster.bin" "$TOP/ctrl-1/conf/cluster.bin.$TAG"
-echo "cluster.bin.$TAG" > "$TOP/ctrl-1/conf/current"
+cp "$DATADIR/cluster.bin" "$TOP/controller/ctrl-1/conf/cluster.bin.$TAG"
+echo "cluster.bin.$TAG" > "$TOP/controller/ctrl-1/conf/current"

 # Build peers string: ctrl-1=host1:port1,ctrl-2=host2:port2,...
 PEERS_STR=""
@ -52,12 +52,12 @@ if [ -z "\$NAME" ]; then
    echo "Usage: \$0 <ctrl-name>"
    exit 1
 fi
-CTRL_DIR=$TOP/\$NAME
+CTRL_DIR=$TOP/controller/\$NAME
 if [ ! -d "\$CTRL_DIR/conf" ] || [ ! -d "\$CTRL_DIR/data" ]; then
    echo "Error: \$CTRL_DIR/conf or \$CTRL_DIR/data does not exist"
    exit 1
 fi
-$BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP --logfile $TOP/logs/\$NAME.log --vcpus 4 \$VERBOSE
+$BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP/controller --certdir $TOP/certs --logfile $TOP/logs/\$NAME.log --vcpus 4 \$VERBOSE
 EOF
 chmod +x "$TOP/scripts/start.sh"

@ -67,10 +67,10 @@ cat > "$TOP/scripts/cleanup.sh" << EOF
 set -e
 rm -f $TOP/logs/*.log
 for i in \$(seq 1 $N); do
-    rm -rf $TOP/ctrl-\$i/data/*
+    rm -rf $TOP/controller/ctrl-\$i/data/*
 done
 for i in \$(seq 2 $N); do
-    rm -rf $TOP/ctrl-\$i/conf/*
+    rm -rf $TOP/controller/ctrl-\$i/conf/*
 done
 EOF
 chmod +x "$TOP/scripts/cleanup.sh"
				`@ -1 +0,0 @@`
				`{"detected":false,"exe":null,"args":null,"extensions":[]}`