#!/usr/bin/env node

const assert = require("assert");
const cp = require("child_process");
const crypto = require("crypto");
const fs = require("fs");
const https = require("https");
const os = require("os");
const path = require("path");
const { DapClient } = require("./dap-client");
const { agentIdentity, signedAgentWorkflowRequest } = require("./agent-signing");
const { coordinatorWireRequest } = require("./coordinator-wire");
const {
  nodeIdentity,
  nodeIdentityFromPrivateKey,
  signedNodeHeartbeat,
  signedNodeRequest,
} = require("./node-signing");
const { configurePodmanTestEnvironment } = require("./podman-test-env");

const repo = path.resolve(__dirname, "..");
configurePodmanTestEnvironment(repo);
if (
  !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL &&
  cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 &&
  cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0
) {
  cp.execFileSync(
    "nix",
    ["shell", "nixpkgs#podman", "--command", "node", __filename],
    {
      cwd: repo,
      env: { ...process.env, CLUSTERFLUX_PODMAN_NIX_SHELL: "1" },
      stdio: "inherit",
    }
  );
  process.exit(0);
}

const releaseRoot = path.resolve(
  process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR ||
    path.join(repo, "target/public-release")
);
const acceptanceRoot = path.join(repo, "target/acceptance");
const manifestPath = path.join(releaseRoot, "public-release-manifest.json");
const reportPath = path.join(acceptanceRoot, "cli-happy-path-live.json");
const serviceEndpoint = "https://clusterflux.michelpaulissen.com";
const serviceAddr =
  process.env.CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR ||
  "clusterflux.michelpaulissen.com:443";
const browserOpenCommand =
  process.env.CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND;
const reuseSessionFile =
  process.env.CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE;
const qualityGateEvidencePath =
  process.env.CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH;
const enabled = process.env.CLUSTERFLUX_CLI_HAPPY_PATH_LIVE === "1";
const strictFullRelease = process.env.CLUSTERFLUX_STRICT_FULL_RELEASE === "1";
const strictVpsRestart = process.env.CLUSTERFLUX_STRICT_VPS_RESTART === "1";
const strictVpsHost = process.env.CLUSTERFLUX_STRICT_VPS_HOST;
const strictVpsIdentity = process.env.CLUSTERFLUX_STRICT_VPS_IDENTITY;
const strictServiceUnit =
  process.env.CLUSTERFLUX_STRICT_SERVICE_UNIT ||
  "clusterflux-hosted.service";
const strictSoakSeconds = Number(
  process.env.CLUSTERFLUX_STRICT_SOAK_SECONDS || "300"
);
const commands = [];

function requireEnabled() {
  if (!enabled) {
    throw new Error(
      "CLUSTERFLUX_CLI_HAPPY_PATH_LIVE=1 is required because this runs the live CLI happy path"
    );
  }
  if (!browserOpenCommand && !reuseSessionFile) {
    throw new Error(
      "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND or CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE is required"
    );
  }
  if (
    strictFullRelease &&
    !process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE
  ) {
    throw new Error(
      "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE for live isolation and quota independence"
    );
  }
  if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) {
    throw new Error(
      "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE"
    );
  }
  if (strictFullRelease && !qualityGateEvidencePath) {
    throw new Error(
      "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH"
    );
  }
  if (strictFullRelease && !strictVpsRestart) {
    throw new Error(
      "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_STRICT_VPS_RESTART=1"
    );
  }
  if (strictVpsRestart && (!strictVpsHost || !strictVpsIdentity)) {
    throw new Error(
      "strict VPS restart requires CLUSTERFLUX_STRICT_VPS_HOST and CLUSTERFLUX_STRICT_VPS_IDENTITY"
    );
  }
  if (
    !Number.isInteger(strictSoakSeconds) ||
    strictSoakSeconds < 120 ||
    strictSoakSeconds > 1800
  ) {
    throw new Error("CLUSTERFLUX_STRICT_SOAK_SECONDS must be an integer from 120 to 1800");
  }
}

function readJson(file) {
  return JSON.parse(fs.readFileSync(file, "utf8"));
}

function strictQualityGateEvidence(manifest) {
  if (!strictFullRelease) return null;
  const evidence = readJson(path.resolve(qualityGateEvidencePath));
  assert.strictEqual(evidence.source_commit, manifest.source_commit);
  assert.strictEqual(evidence.source_tree_digest, manifest.source_tree_digest);
  for (const gate of [evidence.private, evidence.public]) {
    assert.strictEqual(gate.status, "passed");
    assert(Number.isFinite(gate.duration_ms) && gate.duration_ms > 0);
  }
  assert.strictEqual(evidence.public.independent_filtered_tree, true);
  return evidence;
}

function ensureDir(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

function sha256(bytes) {
  return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`;
}

function nonInteractiveGitEnv(extra = {}) {
  return {
    ...process.env,
    GIT_TERMINAL_PROMPT: "0",
    GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false",
    SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false",
    GIT_SSH_COMMAND:
      process.env.GIT_SSH_COMMAND ||
      "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0",
    ...extra,
  };
}

function commandEnv(command, env) {
  if (command !== "git") return env;
  return nonInteractiveGitEnv(env);
}

function recordCommand(command, args) {
  const redacted = [];
  let hideNext = false;
  for (const argument of args) {
    if (hideNext) {
      redacted.push("<redacted>");
      hideNext = false;
    } else {
      redacted.push(argument);
      hideNext = ["--enrollment-grant", "--admin-token"].includes(argument);
    }
  }
  commands.push([command, ...redacted].join(" "));
}

function run(command, args, options = {}) {
  recordCommand(command, args);
  return cp.execFileSync(command, args, {
    cwd: repo,
    encoding: "utf8",
    stdio: ["ignore", "pipe", "pipe"],
    timeout: 15 * 60 * 1000,
    ...options,
    env: commandEnv(command, options.env),
  });
}

function parseJsonOutput(output) {
  const trimmed = output.trim();
  if (!trimmed) throw new Error("expected JSON output, got empty stdout");
  try {
    return JSON.parse(trimmed);
  } catch (_) {
    // Commands may print progress before their final JSON report.
  }
  const lines = trimmed.split(/\r?\n/).filter(Boolean);
  for (let index = lines.length - 1; index >= 0; index -= 1) {
    try {
      return JSON.parse(lines.slice(index).join("\n"));
    } catch (_) {
      // Keep scanning for the trailing JSON value.
    }
  }
  throw new Error(`could not parse JSON output:\n${trimmed}`);
}

function runJson(command, args, options = {}) {
  return parseJsonOutput(run(command, args, options));
}

function authenticatedRequest(sessionSecret, request) {
  return {
    type: "authenticated",
    session_secret: sessionSecret,
    request,
  };
}

function sendHostedControl(payload) {
  const body = Buffer.from(
    JSON.stringify(coordinatorWireRequest(payload, "strict-live"))
  );
  const url = new URL("/api/v1/control", serviceEndpoint);
  return new Promise((resolve, reject) => {
    const request = https.request(
      url,
      {
        method: "POST",
        headers: {
          "content-type": "application/json",
          "content-length": body.length,
        },
        timeout: 30_000,
      },
      (response) => {
        const chunks = [];
        response.on("data", (chunk) => chunks.push(chunk));
        response.on("end", () => {
          const responseBody = Buffer.concat(chunks).toString("utf8");
          if (response.statusCode < 200 || response.statusCode >= 300) {
            reject(
              new Error(
                `hosted control HTTP ${response.statusCode}: ${responseBody}`
              )
            );
            return;
          }
          try {
            resolve(JSON.parse(responseBody));
          } catch (error) {
            reject(
              new Error(`hosted control returned invalid JSON: ${error.message}`)
            );
          }
        });
      }
    );
    request.on("timeout", () =>
      request.destroy(new Error("hosted control request timed out"))
    );
    request.on("error", reject);
    request.end(body);
  });
}

function sendHostedControlDroppingResponse(payload) {
  const body = Buffer.from(
    JSON.stringify(coordinatorWireRequest(payload, "strict-live-dropped-response"))
  );
  const url = new URL("/api/v1/control", serviceEndpoint);
  return new Promise((resolve, reject) => {
    let settled = false;
    const request = https.request(
      url,
      {
        method: "POST",
        headers: {
          "content-type": "application/json",
          "content-length": body.length,
        },
        timeout: 30_000,
      },
      (response) => {
        settled = true;
        const statusCode = response.statusCode;
        response.destroy();
        resolve({ status_code: statusCode, response_body_consumed: false });
      }
    );
    request.on("timeout", () =>
      request.destroy(new Error("hosted dropped-response request timed out"))
    );
    request.on("error", (error) => {
      if (!settled) reject(error);
    });
    request.end(body);
  });
}

function sendHostedLoginStatus(extraHeaders = {}) {
  const body = Buffer.from(
    JSON.stringify(
      coordinatorWireRequest(
        { type: "begin_oidc_browser_login" },
        `strict-live-login-${Date.now()}-${Math.random()}`
      )
    )
  );
  const url = new URL("/api/v1/login", serviceEndpoint);
  return new Promise((resolve, reject) => {
    const request = https.request(
      url,
      {
        method: "POST",
        headers: {
          "content-type": "application/json",
          "content-length": body.length,
          ...extraHeaders,
        },
        timeout: 30_000,
      },
      (response) => {
        const chunks = [];
        response.on("data", (chunk) => chunks.push(chunk));
        response.on("end", () =>
          resolve({
            status_code: response.statusCode,
            body: Buffer.concat(chunks).toString("utf8"),
          })
        );
      }
    );
    request.on("timeout", () =>
      request.destroy(new Error("hosted login request timed out"))
    );
    request.on("error", reject);
    request.end(body);
  });
}

async function runHostedLoginIsolationBeforeRestart() {
  const scenarioStartedAt = Date.now();
  const controlBefore = await sendHostedControl({ type: "ping" });
  assert.strictEqual(controlBefore.type, "pong");
  const statuses = [];
  let limited;
  for (let index = 0; index < 128; index += 1) {
    const response = await sendHostedLoginStatus();
    statuses.push(response.status_code);
    if (response.status_code === 429) {
      limited = response;
      break;
    }
    assert.strictEqual(response.status_code, 200, response.body);
  }
  assert(limited, `hosted login route was not rate limited: ${statuses.join(",")}`);
  const spoofed = await sendHostedLoginStatus({
    forwarded: "for=203.0.113.99",
    "x-forwarded-for": "203.0.113.99",
    "x-real-ip": "203.0.113.99",
  });
  assert.strictEqual(
    spoofed.status_code,
    429,
    "caller-supplied forwarding headers bypassed the client rate limit"
  );

  const remoteBody = JSON.stringify(
    coordinatorWireRequest(
      { type: "begin_oidc_browser_login" },
      `strict-vps-login-${Date.now()}`
    )
  );
  const remoteStatus = Number(
    run(
      "ssh",
      sshArgs(
        "curl",
        "--silent",
        "--show-error",
        "--output",
        "/dev/null",
        "--write-out=%{http_code}",
        "-HContent-Type:application/json",
        `--data-binary=${remoteBody}`,
        `${serviceEndpoint}/api/v1/login`
      )
    ).trim()
  );
  assert.strictEqual(
    remoteStatus,
    200,
    "one client exhausted the browser-login allowance for a different client"
  );
  const controlAfter = await sendHostedControl({ type: "ping" });
  assert.strictEqual(controlAfter.type, "pong");
  return {
    scenario_started_at_ms: scenarioStartedAt,
    local_statuses: statuses,
    local_limited_status: limited.status_code,
    forwarding_spoof_status: spoofed.status_code,
    independent_vps_client_status: remoteStatus,
    control_route_before: controlBefore.type,
    control_route_after: controlAfter.type,
  };
}

async function finishHostedLoginIsolationAfterRestart(evidence) {
  const control = await sendHostedControl({ type: "ping" });
  assert.strictEqual(control.type, "pong");
  const response = await sendHostedLoginStatus();
  assert(
    [200, 429].includes(response.status_code),
    `login route did not recover after service restart: ${response.status_code} ${response.body}`
  );
  evidence.after_service_restart = {
    control_route: control.type,
    login_route_status: response.status_code,
  };
  evidence.duration_ms = Date.now() - evidence.scenario_started_at_ms;
  delete evidence.scenario_started_at_ms;
  return evidence;
}

function assertDenied(response, pattern, label) {
  assert.strictEqual(response.type, "error", `${label}: ${JSON.stringify(response)}`);
  assert.match(response.message, pattern, `${label}: ${JSON.stringify(response)}`);
  return {
    denied: true,
    category: response.error?.category || null,
    message: response.message,
  };
}

function readNodeCredential(projectDir, node) {
  const directory = path.join(projectDir, ".clusterflux", "nodes");
  for (const file of fs.readdirSync(directory)) {
    if (!file.endsWith(".json")) continue;
    const absolute = path.join(directory, file);
    const credential = readJson(absolute);
    if (credential.node === node) return { absolute, credential };
  }
  throw new Error(`persisted credential for ${node} was not found`);
}

function directoryStats(root) {
  if (!fs.existsSync(root)) return { files: 0, bytes: 0 };
  let files = 0;
  let bytes = 0;
  const visit = (directory) => {
    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
      const absolute = path.join(directory, entry.name);
      if (entry.isDirectory()) visit(absolute);
      else if (entry.isFile()) {
        files += 1;
        bytes += fs.statSync(absolute).size;
      }
    }
  };
  visit(root);
  return { files, bytes };
}

function shellQuote(argument) {
  return `'${String(argument).replaceAll("'", "'\"'\"'")}'`;
}

function sshArgs(...remoteArgs) {
  assert(strictVpsHost && strictVpsIdentity);
  return [
    "-i",
    path.resolve(strictVpsIdentity.replace(/^~(?=\/)/, os.homedir())),
    "-o",
    "BatchMode=yes",
    strictVpsHost,
    remoteArgs.map(shellQuote).join(" "),
  ];
}

function strictInfrastructureSample(projectDir) {
  const memory = Object.fromEntries(
    run(
    "ssh",
    sshArgs(
      "systemctl",
      "show",
      strictServiceUnit,
      "--property=MemoryCurrent",
      "--property=MemoryPeak"
    )
  )
    .trim()
    .split(/\r?\n/)
    .map((line) => line.split("="))
  );
  const serviceDisk = Number(
    run(
      "ssh",
      sshArgs("du", "-sb", "/var/lib/clusterflux-public-release")
    )
      .trim()
      .split(/\s+/)[0]
  );
  const database = run(
    "ssh",
    sshArgs(
      "runuser",
      "-u",
      "postgres",
      "--",
      "psql",
      "-d",
      "clusterflux",
      "-AtF,"
    ),
    {
      input:
        "SELECT pg_database_size('clusterflux'), (SELECT count(*) FROM clusterflux_tenants), (SELECT count(*) FROM clusterflux_projects), (SELECT count(*) FROM clusterflux_node_identities), (SELECT count(*) FROM clusterflux_cli_sessions);\n",
      stdio: ["pipe", "pipe", "pipe"],
    }
  )
    .trim()
    .split(",")
    .map(Number);
  assert(Number.isFinite(Number(memory.MemoryCurrent)));
  assert(Number.isFinite(Number(memory.MemoryPeak)));
  assert.strictEqual(database.length, 5);
  return {
    sampled_at_epoch_ms: Date.now(),
    service_memory_current_bytes: Number(memory.MemoryCurrent),
    service_memory_peak_bytes: Number(memory.MemoryPeak),
    service_state_disk_bytes: serviceDisk,
    postgres_database_bytes: database[0],
    durable_rows: {
      tenants: database[1],
      projects: database[2],
      node_identities: database[3],
      cli_sessions: database[4],
    },
    local_node_state: directoryStats(path.join(projectDir, ".clusterflux")),
  };
}

function executable(root, name) {
  return path.join(root, "bin", process.platform === "win32" ? `${name}.exe` : name);
}

function binaryArchive(manifest) {
  const platform = `${os.platform()}-${os.arch()}`;
  const asset = manifest.assets.find(
    (candidate) =>
      candidate.name.startsWith("clusterflux-public-binaries-") &&
      candidate.name.includes(platform)
  );
  assert(asset, `missing released binary archive for ${platform}`);
  assert(fs.existsSync(asset.file), `missing released binary archive ${asset.file}`);
  return asset.file;
}

function publicRepoUrl(manifest) {
  const url =
    process.env.CLUSTERFLUX_PUBLIC_REPO_URL ||
    process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE ||
    manifest.public_repo_url ||
    manifest.public_repo_remote;
  assert(url, "public repository URL is required");
  assert(url.includes("git.michelpaulissen.com"));
  return url;
}

function stagePublicCheckout(manifest, checkout) {
  const candidateTree = process.env.CLUSTERFLUX_LIVE_PUBLIC_TREE;
  if (candidateTree) {
    const source = path.resolve(candidateTree);
    assert.strictEqual(
      source,
      path.resolve(manifest.public_tree),
      "explicit live candidate tree must be the tree recorded by the release manifest"
    );
    fs.cpSync(source, checkout, { recursive: true });
    return {
      kind: "local_filtered_release_candidate",
      path: source,
      published: false,
    };
  }
  run("git", ["clone", "--depth", "1", publicRepoUrl(manifest), checkout]);
  return {
    kind: "published_forgejo_checkout",
    url: publicRepoUrl(manifest),
    published: true,
  };
}

function delay(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}

async function waitForCli(label, action, predicate, timeoutMs = 10 * 60 * 1000) {
  const deadline = Date.now() + timeoutMs;
  let lastValue;
  let lastError;
  while (Date.now() < deadline) {
    try {
      lastValue = action();
      if (predicate(lastValue)) return lastValue;
      lastError = undefined;
    } catch (error) {
      lastError = error;
    }
    await delay(250);
  }
  throw new Error(
    `${label} timed out; last value=${JSON.stringify(lastValue)}${
      lastError ? `; last error=${lastError.message}` : ""
    }`
  );
}

function spawnJsonLines(command, args, options) {
  recordCommand(command, args);
  const child = cp.spawn(command, args, {
    ...options,
    stdio: ["ignore", "pipe", "pipe"],
  });
  let stdout = "";
  let stderr = "";
  const values = [];
  const waiters = [];

  function deliver(value) {
    values.push(value);
    for (let index = waiters.length - 1; index >= 0; index -= 1) {
      const waiter = waiters[index];
      if (waiter.predicate(value)) {
        waiters.splice(index, 1);
        clearTimeout(waiter.timer);
        waiter.resolve(value);
      }
    }
  }

  child.stdout.on("data", (chunk) => {
    stdout += chunk.toString();
    while (stdout.includes("\n")) {
      const newline = stdout.indexOf("\n");
      const line = stdout.slice(0, newline).trim();
      stdout = stdout.slice(newline + 1);
      if (!line) continue;
      try {
        deliver(JSON.parse(line));
      } catch (_) {
        // Non-JSON progress stays out of the evidence report.
      }
    }
  });
  child.stderr.on("data", (chunk) => {
    stderr += chunk.toString();
  });

  function waitFor(predicate, label, timeoutMs = 120000) {
    const existing = values.find(predicate);
    if (existing) return Promise.resolve(existing);
    return new Promise((resolve, reject) => {
      const waiter = { predicate, resolve, reject, timer: undefined };
      waiter.timer = setTimeout(() => {
        const index = waiters.indexOf(waiter);
        if (index >= 0) waiters.splice(index, 1);
        reject(new Error(`${label} timed out\n${stderr}`));
      }, timeoutMs);
      waiters.push(waiter);
    });
  }

  child.once("exit", (code) => {
    for (const waiter of waiters.splice(0)) {
      clearTimeout(waiter.timer);
      waiter.reject(new Error(`${waiter.label || "child"} exited with ${code}\n${stderr}`));
    }
  });
  return { child, values, waitFor, stderr: () => stderr };
}

async function stopChild(child) {
  if (!child || child.exitCode !== null) return;
  child.kill("SIGTERM");
  const exited = new Promise((resolve) => child.once("exit", resolve));
  const forced = delay(5000).then(() => {
    if (child.exitCode === null) child.kill("SIGKILL");
  });
  await Promise.race([exited, forced]);
  if (child.exitCode === null) await exited;
}

function rawTaskEvents(report) {
  return report?.events?.response?.events || [];
}

function nodeDescriptor(status, node) {
  return status?.response?.descriptors?.find(
    (descriptor) => descriptor.id === node || descriptor.node === node
  );
}

function completedFlagship(events) {
  const completedDefinitions = new Set(
    events
      .filter((event) => event.terminal_state === "completed")
      .map((event) => event.task_definition)
  );
  return (
    completedDefinitions.has("prepare_source") &&
    completedDefinitions.has("compile_linux") &&
    completedDefinitions.has("package_release") &&
    events.some(
      (event) =>
        event.executor === "coordinator_main" && event.terminal_state === "completed"
    )
  );
}

function contentAddressedArtifactId(event, logicalName) {
  const digest = event?.artifact_digest;
  if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) {
    return null;
  }
  const artifactId = `${logicalName}-${digest.slice("sha256:".length)}`;
  return event.artifact_path?.endsWith(`/vfs/artifacts/${artifactId}`)
    ? artifactId
    : null;
}

async function prepareLiveNodeCredentialSecurity({
  clusterflux,
  projectDir,
  scope,
  tenant,
  project,
  suffix,
}) {
  const node = `security-node-${suffix}`;
  const grant = runJson(clusterflux, ["node", "enroll", ...scope], {
    cwd: projectDir,
  });
  const attach = runJson(
    clusterflux,
    [
      "node",
      "attach",
      "--coordinator",
      serviceEndpoint,
      "--tenant",
      tenant,
      "--project-id",
      project,
      "--node",
      node,
      "--enrollment-grant",
      grant.enrollment_grant.grant,
      "--json",
    ],
    { cwd: projectDir }
  );
  assert.strictEqual(attach.boundary.used_enrollment_exchange, true);
  const stored = readNodeCredential(projectDir, node);
  const identity = nodeIdentityFromPrivateKey(stored.credential.private_key);
  assert.strictEqual(identity.publicKey, stored.credential.public_key);

  const heartbeat = {
    type: "node_heartbeat",
    node,
    node_signature: signedNodeHeartbeat(node, identity, {
      nonce: `strict-valid-${suffix}`,
    }),
  };
  const valid = await sendHostedControl(heartbeat);
  assert.strictEqual(valid.type, "node_heartbeat");
  const replay = assertDenied(
    await sendHostedControl(heartbeat),
    /nonce.*already.*used|replay/i,
    "replayed node credential"
  );

  const expired = assertDenied(
    await sendHostedControl({
      type: "node_heartbeat",
      node,
      node_signature: signedNodeHeartbeat(node, identity, {
        nonce: `strict-expired-${suffix}`,
        issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
      }),
    }),
    /expired|clock skew/i,
    "expired node credential"
  );

  const forgedIdentity = nodeIdentity("strict-forged-node", node);
  const forged = assertDenied(
    await sendHostedControl({
      type: "node_heartbeat",
      node,
      node_signature: signedNodeHeartbeat(node, forgedIdentity, {
        nonce: `strict-forged-${suffix}`,
      }),
    }),
    /signature|public key/i,
    "forged node credential"
  );

  const originalCapabilityBody = {
    type: "report_node_capabilities",
    tenant,
    project,
    node,
    capabilities: {
      os: "Linux",
      arch: process.arch === "x64" ? "x86_64" : process.arch,
      capabilities: [],
      environment_backends: [],
      source_providers: [],
    },
    cached_environment_digests: [],
    dependency_cache_digests: [],
    source_snapshots: [],
    artifact_locations: [],
    direct_connectivity: false,
    online: true,
  };
  const modifiedEnvelope = signedNodeRequest(
    node,
    identity,
    "report_node_capabilities",
    originalCapabilityBody,
    { nonce: `strict-modified-${suffix}` }
  );
  modifiedEnvelope.request.online = false;
  const bodyModified = assertDenied(
    await sendHostedControl(modifiedEnvelope),
    /signature/i,
    "body-modified node credential"
  );

  return {
    node,
    identity,
    credential_path: stored.absolute,
    credential_digest: sha256(fs.readFileSync(stored.absolute)),
    capability_body: originalCapabilityBody,
    evidence: {
      valid_request: valid.type,
      replay,
      expired,
      forged,
      body_modified: bodyModified,
    },
  };
}

async function runLiveAgentCredentialSecurity({
  clusterfluxDap,
  clusterflux,
  projectDir,
  scope,
  sessionSecret,
  tenant,
  project,
  suffix,
  bundle,
  securityNode,
  workerRuntime,
}) {
  const agent = `security-agent-${suffix}`;
  const identity = agentIdentity("strict-live-agent", agent);
  const added = runJson(
    clusterflux,
    ["key", "add", ...scope, "--agent", agent, "--public-key", identity.publicKey],
    { cwd: projectDir }
  );
  assert.strictEqual(added.command, "key add");

  const processId = `vp-agent-security-${suffix}`;
  const unsigned = {
    type: "start_process",
    tenant,
    project,
    actor_agent: agent,
    process: processId,
    restart: false,
  };
  const validRequest = signedAgentWorkflowRequest(identity, unsigned, {
    nonce: `strict-agent-valid-${suffix}`,
  });
  const valid = await sendHostedControl(validRequest);
  assert.strictEqual(valid.type, "process_started", JSON.stringify(valid));
  const replay = assertDenied(
    await sendHostedControl(validRequest),
    /nonce.*already.*used|replay/i,
    "replayed agent credential"
  );

  const expired = assertDenied(
    await sendHostedControl(
      signedAgentWorkflowRequest(
        identity,
        { ...unsigned, process: `${processId}-expired` },
        {
          nonce: `strict-agent-expired-${suffix}`,
          issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
        }
      )
    ),
    /expired|clock skew/i,
    "expired agent credential"
  );

  const modifiedRequest = signedAgentWorkflowRequest(
    identity,
    { ...unsigned, process: `${processId}-modified` },
    { nonce: `strict-agent-modified-${suffix}` }
  );
  modifiedRequest.restart = true;
  const bodyModified = assertDenied(
    await sendHostedControl(modifiedRequest),
    /signature/i,
    "body-modified agent credential"
  );

  const forgedIdentity = agentIdentity("strict-live-forged-agent", agent);
  const forged = assertDenied(
    await sendHostedControl(
      signedAgentWorkflowRequest(
        forgedIdentity,
        { ...unsigned, process: `${processId}-forged` },
        {
          nonce: `strict-agent-forged-${suffix}`,
          publicKeyFingerprint: identity.publicKeyFingerprint,
        }
      )
    ),
    /signature|public key/i,
    "forged agent credential"
  );

  const fakeEnvironmentDigest = sha256(
    Buffer.from(`strict-debug-missing-participant-${suffix}`)
  );
  const fakeCapabilityBody = {
    ...securityNode.capability_body,
    cached_environment_digests: [fakeEnvironmentDigest],
    online: true,
  };
  const mainTask = `ti:${processId}:main`;
  const fakeTask = `missing-debug-participant-${suffix}`;
  const mainLaunchBody = {
    type: "launch_task",
    tenant,
    project,
    actor_agent: agent,
    task_spec: {
      tenant,
      project,
      process: processId,
      task_definition: bundle.entryStableId,
      task_instance: mainTask,
      dispatch: {
        kind: "coordinator_node_wasm",
        export: bundle.entryExport,
        abi: "entrypoint_v1",
      },
      environment_id: null,
      environment: null,
      environment_digest: null,
      required_capabilities: [],
      dependency_cache: null,
      source_snapshot: null,
      required_artifacts: [],
      args: [],
      vfs_epoch: valid.epoch,
      bundle_digest: bundle.digest,
    },
    wait_for_node: true,
    artifact_path: `/vfs/artifacts/${mainTask}-output.txt`,
    wasm_module_base64: bundle.moduleBase64,
  };
  let mainLaunch;
  let directTaskV1;
  let fakeLaunch;
  let liveParentTask;
  let liveParentAssignment;
  let realDebugLaunch;
  const realDebugTask = `real-debug-participant-${suffix}`;
  let workerPaused = false;
  const baselineContainers = new Set(
    run("podman", ["ps", "-q"])
      .trim()
      .split(/\s+/)
      .filter(Boolean)
  );
  try {
    mainLaunch = await sendHostedControl(
      signedAgentWorkflowRequest(identity, mainLaunchBody, {
        nonce: `strict-agent-main-${suffix}`,
      })
    );
    assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch));
    assert.strictEqual(mainLaunch.task_instance, mainTask);
    assert.strictEqual(mainLaunch.state, "running");
    assert.strictEqual(mainLaunch.actor.kind, "agent");
    assert.strictEqual(mainLaunch.actor.authenticated_without_browser, true);

    liveParentAssignment = await workerRuntime.worker.waitFor(
      (value) => {
        const taskSpec = value.task_assignment_response?.task_spec;
        return (
          value.node_status === "assignment_started" &&
          value.node === workerRuntime.node &&
          value.process === processId &&
          taskSpec?.dispatch?.abi === "task_v1" &&
          taskSpec.task_definition === "compile_linux" &&
          taskSpec.environment_id === "linux" &&
          typeof taskSpec.environment_digest === "string" &&
          typeof taskSpec.source_snapshot === "string"
        );
      },
      "agent main to dispatch an environment-bound live child to the real worker",
      120000
    );
    liveParentTask = liveParentAssignment.virtual_thread;
    assert.strictEqual(typeof liveParentTask, "string");
    assert.notStrictEqual(liveParentTask, "");
    assert.strictEqual(
      workerRuntime.child.kill("SIGSTOP"),
      true,
      "failed to pause the real worker after observing its live child"
    );
    workerPaused = true;

    const fakeCapabilities = await sendHostedControl(
      signedNodeRequest(
        securityNode.node,
        securityNode.identity,
        "report_node_capabilities",
        fakeCapabilityBody,
        { nonce: `strict-agent-fake-capabilities-${suffix}` }
      )
    );
    assert.strictEqual(
      fakeCapabilities.type,
      "node_capabilities_recorded",
      JSON.stringify(fakeCapabilities)
    );

    const fakeTaskSpec = {
    tenant,
    project,
    process: processId,
    task_definition: "task_add_one",
    task_instance: fakeTask,
    dispatch: {
      kind: "coordinator_node_wasm",
      export: null,
      abi: "task_v1",
    },
    environment_id: "strict-debug-missing-participant",
    environment: null,
    environment_digest: fakeEnvironmentDigest,
    required_capabilities: [],
    dependency_cache: null,
    source_snapshot: null,
    required_artifacts: [],
    args: [{ SmallJson: 41 }],
    vfs_epoch: valid.epoch,
    bundle_digest: bundle.digest,
    };
    directTaskV1 = assertDenied(
      await sendHostedControl(
        signedAgentWorkflowRequest(
          identity,
          {
            type: "launch_task",
            tenant,
            project,
            actor_agent: agent,
            task_spec: fakeTaskSpec,
            wait_for_node: false,
            artifact_path: `/vfs/artifacts/${fakeTask}.txt`,
            wasm_module_base64: bundle.moduleBase64,
          },
          { nonce: `strict-agent-direct-task-v1-${suffix}` }
        )
      ),
      /external callers.*EntrypointV1|TaskV1 requires an authenticated live parent/i,
      "external direct TaskV1 launch"
    );

    const liveParentSpec =
      liveParentAssignment.task_assignment_response.task_spec;
    realDebugLaunch = await sendHostedControl(
      signedNodeRequest(
        workerRuntime.node,
        workerRuntime.identity,
        "launch_child_task",
        {
          type: "launch_child_task",
          tenant,
          project,
          process: processId,
          node: workerRuntime.node,
          parent_task: liveParentTask,
          task_spec: {
            tenant,
            project,
            process: processId,
            task_definition: "abort_probe",
            task_instance: realDebugTask,
            dispatch: {
              kind: "coordinator_node_wasm",
              export: null,
              abi: "task_v1",
            },
            environment_id: liveParentSpec.environment_id,
            environment: liveParentSpec.environment,
            environment_digest: liveParentSpec.environment_digest,
            required_capabilities: ["Command"],
            dependency_cache: null,
            source_snapshot: liveParentSpec.source_snapshot,
            required_artifacts: [],
            args: liveParentSpec.args,
            vfs_epoch: valid.epoch,
            bundle_digest: bundle.digest,
          },
          wait_for_node: false,
          artifact_path: `/vfs/artifacts/${realDebugTask}.txt`,
          wasm_module_base64: bundle.moduleBase64,
        },
        { nonce: `strict-real-debug-participant-${suffix}` }
      )
    );
    assert.strictEqual(
      realDebugLaunch.type,
      "task_launched",
      JSON.stringify(realDebugLaunch)
    );

    const fakeLaunchBody = {
      type: "launch_child_task",
      tenant,
      project,
      process: processId,
      node: workerRuntime.node,
      parent_task: liveParentTask,
      task_spec: fakeTaskSpec,
      wait_for_node: false,
      artifact_path: `/vfs/artifacts/${fakeTask}.txt`,
      wasm_module_base64: bundle.moduleBase64,
    };
    fakeLaunch = await sendHostedControl(
      signedNodeRequest(
        workerRuntime.node,
        workerRuntime.identity,
        "launch_child_task",
        fakeLaunchBody,
        { nonce: `strict-parent-runtime-fake-task-${suffix}` }
      )
    );
    assert.strictEqual(fakeLaunch.type, "task_launched", JSON.stringify(fakeLaunch));
    assert.strictEqual(fakeLaunch.placement.node, securityNode.node);
  } finally {
    if (workerPaused) {
      assert.strictEqual(
        workerRuntime.child.kill("SIGCONT"),
        true,
        "failed to resume the real worker after authenticated child launch"
      );
    }
  }
  await workerRuntime.worker.waitFor(
    (value) =>
      value.node_status === "assignment_started" &&
      value.virtual_thread === realDebugTask,
    "real Podman debug participant to start",
    120_000
  );
  const realDebugContainerDeadline = Date.now() + 120_000;
  let realDebugContainerIds = new Set();
  let lastRealDebugPodmanStates = [];
  while (Date.now() < realDebugContainerDeadline) {
    const containers = runJson("podman", ["ps", "--format", "json"]);
    lastRealDebugPodmanStates = containers;
    realDebugContainerIds = new Set(
      containers
        .map((container) => container.Id || container.ID || container.IdHex || "")
        .filter((id) => id && !baselineContainers.has(id))
    );
    if (realDebugContainerIds.size > 0) break;
    await delay(100);
  }
  assert(
    realDebugContainerIds.size > 0,
    `real debug participant did not start a Podman container: ${JSON.stringify(
      lastRealDebugPodmanStates
    )}`
  );

  const debugClient = new DapClient({
    cwd: projectDir,
    command: clusterfluxDap,
    args: [],
    env: { ...process.env },
  });
  const initialize = debugClient.send("initialize", {
    adapterID: "clusterflux",
    linesStartAt1: true,
    columnsStartAt1: true,
  });
  await debugClient.response(initialize, "initialize");
  const attachRequest = debugClient.send("attach", {
    entry: "build",
    project: projectDir,
    processId,
    runtimeBackend: "live-services",
    coordinatorEndpoint: serviceEndpoint,
  });
  await debugClient.response(attachRequest, "attach");
  await debugClient.waitFor(
    (message) => message.type === "event" && message.event === "initialized"
  );
  const configurationDone = debugClient.send("configurationDone");
  await debugClient.response(configurationDone, "configurationDone");
  const attachDeadline = Date.now() + 120_000;
  let attachedThreads = [];
  while (Date.now() < attachDeadline) {
    const threadsRequest = debugClient.send("threads");
    attachedThreads = (
      await debugClient.response(threadsRequest, "threads")
    ).body.threads;
    if (attachedThreads.length > 0) break;
    await delay(100);
  }
  assert(attachedThreads.length > 0, "DAP attach reported no live threads");

  const freezeStartedAt = Date.now();
  const pauseRequest = debugClient.send("pause", {
    threadId: attachedThreads[0].id,
  });
  await debugClient.response(pauseRequest, "pause");
  const dapPartialStop = await debugClient.waitFor(
    (message) =>
      message.type === "event" &&
      message.event === "stopped" &&
      message.body.reason === "pause",
    30_000
  );
  assert.strictEqual(dapPartialStop.body.allThreadsStopped, false);
  const epochRequest = debugClient.send("evaluate", {
    expression: "debug_epoch",
    context: "watch",
  });
  const freeze = {
    epoch: Number(
      (await debugClient.response(epochRequest, "evaluate")).body.result
    ),
  };
  assert(Number.isSafeInteger(freeze.epoch) && freeze.epoch > 0);
  const partialFreeze = await sendHostedControl(
    authenticatedRequest(sessionSecret, {
      type: "inspect_debug_epoch",
      process: processId,
      epoch: freeze.epoch,
    })
  );
  assert.strictEqual(partialFreeze.type, "debug_epoch_status");
  assert.strictEqual(partialFreeze.partially_frozen, true, JSON.stringify(partialFreeze));
  assert.strictEqual(partialFreeze.fully_frozen, false);
  assert.strictEqual(partialFreeze.failed, true);
  assert.match(
    partialFreeze.failure_messages.join("; "),
    /did not acknowledge frozen state within \d+ ms/
  );
  const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]);
  const pausedContainers = podmanStates.filter((container) => {
    const id = container.Id || container.ID || container.IdHex || "";
    const state = String(container.State || container.Status || "").toLowerCase();
    return realDebugContainerIds.has(id) && state.includes("paused");
  });
  assert(
    pausedContainers.length > 0,
    `partial Debug Epoch did not pause a real Podman container: ${JSON.stringify(
      podmanStates
    )}`
  );
  const pausedContainerIds = pausedContainers.map(
    (container) => container.Id || container.ID || container.IdHex
  );

  const mismatchedDigest = sha256(Buffer.from(`strict-nested-mismatch-${suffix}`));
  const mismatchedChild = await sendHostedControl(
    signedNodeRequest(
      securityNode.node,
      securityNode.identity,
      "launch_child_task",
      {
        type: "launch_child_task",
        tenant,
        project,
        process: processId,
        node: securityNode.node,
        parent_task: fakeTask,
        task_spec: {
          tenant,
          project,
          process: processId,
          task_definition: "task_add_one",
          task_instance: `${fakeTask}:child:mismatched-environment`,
          dispatch: {
            kind: "coordinator_node_wasm",
            export: null,
            abi: "task_v1",
          },
          environment_id: "strict-nested-mismatch",
          environment: null,
          environment_digest: mismatchedDigest,
          required_capabilities: [],
          dependency_cache: null,
          source_snapshot: null,
          required_artifacts: [],
          args: [{ SmallJson: 41 }],
          vfs_epoch: valid.epoch,
          bundle_digest: bundle.digest,
        },
        wait_for_node: false,
        artifact_path: `/vfs/artifacts/${fakeTask}-mismatched-child.txt`,
        wasm_module_base64: bundle.moduleBase64,
      },
      { nonce: `strict-nested-mismatch-${suffix}` }
    )
  );
  const nestedEnvironmentMismatch = assertDenied(
    mismatchedChild,
    /environment|digest|compatible|placement|no node/i,
    "nested environment mismatch"
  );

  const continueStartedAt = Date.now();
  const continueRequest = debugClient.send("continue", {
    threadId: dapPartialStop.body.threadId,
  });
  await debugClient.response(continueRequest, "continue");
  const continueResponseMs = Date.now() - continueStartedAt;
  assert(
    continueResponseMs < 2_000,
    `partial-freeze continue blocked for ${continueResponseMs} ms`
  );
  await delay(1000);
  const resumed = await sendHostedControl(
    authenticatedRequest(sessionSecret, {
      type: "inspect_debug_epoch",
      process: processId,
      epoch: freeze.epoch,
    })
  );
  assert.strictEqual(resumed.type, "debug_epoch_status");
  const resumedAcknowledgements = resumed.acknowledgements.filter(
    (acknowledgement) => acknowledgement.state === "running"
  );
  assert(resumedAcknowledgements.length > 0, JSON.stringify(resumed));
  const resumedPodmanStates = runJson("podman", ["ps", "--format", "json"]);
  assert(
    !resumedPodmanStates.some((container) => {
      const id = container.Id || container.ID || container.IdHex || "";
      const state = String(container.State || container.Status || "").toLowerCase();
      return pausedContainerIds.includes(id) && state.includes("paused");
    }),
    `DAP continue left a Clusterflux container paused: ${JSON.stringify(
      resumedPodmanStates
    )}`
  );
  await debugClient.close();

  const completed = await waitForCli(
    "agent-authenticated coordinator main and child workflow completion",
    () =>
      runJson(clusterflux, ["task", "list", ...scope, "--process", processId], {
        cwd: projectDir,
      }),
    (tasks) => completedFlagship(rawTaskEvents(tasks)),
    120000
  );
  const completedEvents = rawTaskEvents(completed);
  const concurrentCompileTasks = [
    ...new Set(
      completedEvents
        .filter(
          (event) =>
            event.task_definition === "compile_linux" &&
            event.terminal_state === "completed"
        )
        .map((event) => event.task)
    ),
  ];
  assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents));

  const revoked = runJson(
    clusterflux,
    ["key", "revoke", ...scope, "--agent", agent, "--yes"],
    { cwd: projectDir }
  );
  assert.strictEqual(revoked.command, "key revoke");
  const revokedUse = assertDenied(
    await sendHostedControl(
      signedAgentWorkflowRequest(
        identity,
        { ...unsigned, process: `${processId}-revoked` },
        { nonce: `strict-agent-revoked-${suffix}` }
      )
    ),
    /revoked/i,
    "revoked agent credential"
  );
  return {
    agent,
    valid_request: valid.type,
    replay,
    expired,
    forged,
    body_modified: bodyModified,
    revoked: revokedUse,
    workflow: {
      process: processId,
      main_launch: mainLaunch.type,
      external_direct_task_v1: directTaskV1,
      child_launch: fakeLaunch.type,
      child_parent: liveParentTask,
      real_debug_child_launch: realDebugLaunch.type,
      real_debug_child: realDebugTask,
      actor_kind: mainLaunch.actor.kind,
      authenticated_without_browser: mainLaunch.actor.authenticated_without_browser,
      flagship_completed: completedFlagship(completedEvents),
      concurrent_compile_tasks: concurrentCompileTasks,
    },
    partial_freeze: {
      epoch: freeze.epoch,
      elapsed_ms: Date.now() - freezeStartedAt,
      partially_frozen: partialFreeze.partially_frozen,
      fully_frozen: partialFreeze.fully_frozen,
      dap_all_threads_stopped: dapPartialStop.body.allThreadsStopped,
      dap_continue_response_ms: continueResponseMs,
      podman_paused_container_ids: pausedContainerIds,
      warning: partialFreeze.failure_messages,
      resumed_participants: resumedAcknowledgements.map(
        (acknowledgement) => acknowledgement.task
      ),
    },
    nested_environment_mismatch: nestedEnvironmentMismatch,
  };
}

async function runSecondTenantIsolation({
  firstTenant,
  firstProject,
  firstProcess,
  firstNode,
  firstArtifact,
  manifest,
}) {
  const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE;
  const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE;
  if (!file && evidenceFile) {
    const evidenceBytes = fs.readFileSync(path.resolve(evidenceFile));
    const previous = JSON.parse(evidenceBytes);
    const isolation = previous.tenant_isolation;
    assert.strictEqual(isolation?.executed, true);
    assert.notStrictEqual(isolation.second_tenant, firstTenant);
    assert.strictEqual(isolation.project?.denied, true);
    assert.strictEqual(isolation.process_hidden, true);
    assert.strictEqual(isolation.node_hidden, true);
    assert.strictEqual(isolation.tasks_and_logs?.denied, true);
    assert.strictEqual(isolation.debug?.denied, true);
    assert.strictEqual(isolation.debug?.audited, true);
    assert.strictEqual(isolation.debug?.charged_debug_read_bytes, 0);
    assert.strictEqual(isolation.artifact_and_download?.denied, true);
    assert.strictEqual(isolation.process_control?.denied, true);
    const previousDeployment = previous.release_binding?.deployment;
    const currentDeployment = deploymentProvenance();
    assert.strictEqual(
      previousDeployment?.hosted_service_sha256,
      currentDeployment.hosted_service_sha256,
      "reused isolation evidence must target the exact deployed hosted binary"
    );
    assert.deepStrictEqual(
      previous.release_binding?.binary_digests,
      manifest.binary_digests,
      "reused isolation evidence must target the exact public binaries"
    );
    return {
      ...isolation,
      reused_from_immutable_evidence: {
        report_sha256: sha256(evidenceBytes),
        source_commit: previous.source_commit,
        hosted_service_sha256: currentDeployment.hosted_service_sha256,
        public_binary_digests: manifest.binary_digests,
      },
    };
  }
  if (!file) return { executed: false, reason: "second tenant session not supplied" };
  const second = readJson(path.resolve(file));
  assert.strictEqual(second.coordinator, serviceEndpoint);
  assert(second.session_secret, "second tenant session omitted its session secret");
  assert.notStrictEqual(
    second.tenant,
    firstTenant,
    "isolation proof requires a genuinely distinct hosted tenant"
  );
  const call = (request) =>
    sendHostedControl(authenticatedRequest(second.session_secret, request));

  const project = assertDenied(
    await call({ type: "select_project", project: firstProject }),
    /outside|not visible|tenant|project/i,
    "cross-tenant project selection"
  );
  const processes = await call({ type: "list_processes" });
  assert.strictEqual(
    processes.type,
    "process_statuses",
    JSON.stringify(processes)
  );
  assert(!JSON.stringify(processes).includes(firstProcess));
  const nodes = await call({ type: "list_node_descriptors" });
  assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes));
  assert(!JSON.stringify(nodes).includes(firstNode));
  const tasksAndLogs = assertDenied(
    await call({ type: "list_task_events", process: firstProcess }),
    /outside|scope|tenant|project|not active|requires an active|unknown/i,
    "cross-tenant task and log listing"
  );
  const debugResponse = await call({
    type: "debug_attach",
    process: firstProcess,
  });
  assert.strictEqual(debugResponse.type, "debug_attach", JSON.stringify(debugResponse));
  assert.strictEqual(debugResponse.authorization?.allowed, false);
  assert.strictEqual(debugResponse.audit_event?.allowed, false);
  assert.match(
    debugResponse.authorization?.reason || "",
    /outside|scope|permission|tenant|project|not active|unknown/i
  );
  assert.strictEqual(debugResponse.charged_debug_read_bytes, 0);
  const debug = {
    denied: true,
    reason: debugResponse.authorization.reason,
    audited: true,
    charged_debug_read_bytes: debugResponse.charged_debug_read_bytes,
  };
  const artifact = assertDenied(
    await call({
      type: "create_artifact_download_link",
      artifact: firstArtifact,
      max_bytes: 1024,
      ttl_seconds: 60,
    }),
    /outside|scope|tenant|project|not found|unavailable/i,
    "cross-tenant artifact download"
  );
  const control = assertDenied(
    await call({ type: "abort_process", process: firstProcess }),
    /outside|scope|tenant|project|not active|requires an active|unknown/i,
    "cross-tenant process control"
  );
  return {
    executed: true,
    second_tenant: second.tenant,
    project,
    process_hidden: true,
    node_hidden: true,
    tasks_and_logs: tasksAndLogs,
    debug,
    artifact_and_download: artifact,
    process_control: control,
  };
}

async function runLiveSoak({
  clusterflux,
  projectDir,
  scope,
  securityNode,
}) {
  if (!strictVpsRestart) {
    return { executed: false, reason: "strict VPS measurement access not supplied" };
  }
  const runReport = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(runReport.status, "main_launched");
  const processId = runReport.process;
  const parked = await waitForCli(
    "soak coordinator main to park without a capable node",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", processId],
        { cwd: projectDir }
      ),
    (status) =>
      status.live_process?.main_state === "running" &&
      status.live_process?.main_wait_state === "waiting_for_task" &&
      status.current_task_count === 0 &&
      status.live_process?.connected_nodes?.length === 0,
    120000
  );

  const pollSecurityNode = (nonce) =>
    sendHostedControl(
      signedNodeRequest(
        securityNode.node,
        securityNode.identity,
        "poll_task_assignment",
        {
          type: "poll_task_assignment",
          tenant: securityNode.capability_body.tenant,
          project: securityNode.capability_body.project,
          node: securityNode.node,
        },
        { nonce }
      )
    );
  const drainedPriorAssignments = [];
  for (let drainIndex = 0; drainIndex < 16; drainIndex += 1) {
    const poll = await pollSecurityNode(
      `strict-soak-drain-${drainIndex}-${Date.now()}`
    );
    assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll));
    if (poll.assignment === null) break;
    assert.notStrictEqual(
      poll.assignment.process,
      processId,
      "soak process unexpectedly received a task assignment"
    );
    drainedPriorAssignments.push({
      process: poll.assignment.process,
      task: poll.assignment.task,
    });
    assert(drainIndex < 15, "pre-soak assignment drain did not quiesce");
  }

  const startedAt = Date.now();
  const deadline = startedAt + strictSoakSeconds * 1000;
  const samples = [];
  let sampleIndex = 0;
  while (true) {
    const capabilities = await sendHostedControl(
      signedNodeRequest(
        securityNode.node,
        securityNode.identity,
        "report_node_capabilities",
        securityNode.capability_body,
        { nonce: `strict-soak-capabilities-${sampleIndex}-${Date.now()}` }
      )
    );
    assert.strictEqual(
      capabilities.type,
      "node_capabilities_recorded",
      JSON.stringify(capabilities)
    );
    const poll = await pollSecurityNode(
      `strict-soak-poll-${sampleIndex}-${Date.now()}`
    );
    assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll));
    assert.strictEqual(poll.assignment, null);
    const status = runJson(
      clusterflux,
      ["process", "status", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    assert.strictEqual(status.live_process.main_wait_state, "waiting_for_task");
    samples.push(strictInfrastructureSample(projectDir));
    sampleIndex += 1;
    if (Date.now() >= deadline) break;
    await delay(Math.min(30_000, deadline - Date.now()));
  }

  const currentMemory = samples.map(
    (sample) => sample.service_memory_current_bytes
  );
  const serviceDisk = samples.map((sample) => sample.service_state_disk_bytes);
  const databaseDisk = samples.map((sample) => sample.postgres_database_bytes);
  const localDisk = samples.map((sample) => sample.local_node_state.bytes);
  const spread = (values) => Math.max(...values) - Math.min(...values);
  assert(spread(currentMemory) <= 128 * 1024 * 1024);
  assert(spread(serviceDisk) <= 16 * 1024 * 1024);
  assert(spread(databaseDisk) <= 32 * 1024 * 1024);
  assert(spread(localDisk) <= 16 * 1024 * 1024);
  assert.deepStrictEqual(
    samples.at(-1).durable_rows,
    samples[0].durable_rows,
    "durable object counts grew during the parked-process soak"
  );
  const aborted = runJson(
    clusterflux,
    ["process", "abort", ...scope, "--process", processId, "--yes"],
    { cwd: projectDir }
  );
  assert.strictEqual(aborted.abort_request.process_slot_released, true);
  return {
    executed: true,
    process: processId,
    duration_seconds: Math.floor((Date.now() - startedAt) / 1000),
    sample_interval_seconds: 30,
    sample_count: samples.length,
    parked_main: {
      main_state: parked.live_process.main_state,
      main_wait_state: parked.live_process.main_wait_state,
    },
    drained_prior_assignments: drainedPriorAssignments,
    signed_node_poll_cycles: samples.length,
    memory_spread_bytes: spread(currentMemory),
    service_disk_spread_bytes: spread(serviceDisk),
    postgres_disk_spread_bytes: spread(databaseDisk),
    local_node_state_spread_bytes: spread(localDisk),
    samples,
  };
}

async function revokeSecurityNode({
  clusterflux,
  projectDir,
  scope,
  securityNode,
}) {
  const revoked = runJson(
    clusterflux,
    ["node", "revoke", ...scope, "--node", securityNode.node, "--yes"],
    { cwd: projectDir }
  );
  assert.strictEqual(revoked.command, "node revoke");
  const denied = assertDenied(
    await sendHostedControl({
      type: "node_heartbeat",
      node: securityNode.node,
      node_signature: signedNodeHeartbeat(
        securityNode.node,
        securityNode.identity,
        { nonce: `strict-node-revoked-${Date.now()}` }
      ),
    }),
    /revoked|unknown node|not recognized|not enrolled/i,
    "revoked node credential"
  );
  return { node: securityNode.node, denied };
}

async function runLiveQuotaPreallocation({ sessionSecret, suffix }) {
  const readSpawnQuota = async () => {
    const status = await sendHostedControl(
      authenticatedRequest(sessionSecret, { type: "quota_status" })
    );
    assert.strictEqual(status.type, "quota_status", JSON.stringify(status));
    const limit = status.limits?.limits?.Spawn;
    const usage = status.usage?.Spawn;
    const windowSeconds = status.window_seconds?.Spawn;
    const windowStarted = status.window_started_epoch_seconds?.Spawn;
    assert(Number.isInteger(limit) && limit > 0, JSON.stringify(status));
    assert(Number.isInteger(usage) && usage >= 0, JSON.stringify(status));
    assert(
      Number.isInteger(windowSeconds) && windowSeconds > 0,
      JSON.stringify(status)
    );
    assert(Number.isInteger(windowStarted), JSON.stringify(status));
    return { limit, usage, windowSeconds, windowStarted };
  };

  let spawnQuota = await readSpawnQuota();
  const windowElapsed = Math.max(
    0,
    Math.floor(Date.now() / 1000) - spawnQuota.windowStarted
  );
  if (spawnQuota.usage > 0 || windowElapsed > 1) {
    await delay(
      Math.max(1000, (spawnQuota.windowSeconds - windowElapsed + 1) * 1000)
    );
    spawnQuota = await readSpawnQuota();
  }
  let denial;
  let deniedProcess;
  let acceptedStarts = 0;
  const maxAttempts = spawnQuota.limit - spawnQuota.usage + 2;
  for (let index = 0; index < maxAttempts; index += 1) {
    const processId = `vp-quota-${suffix}-${index}`;
    const started = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "start_process",
        process: processId,
        restart: false,
      })
    );
    if (started.type === "error") {
      assert.match(started.message, /quota|spawn|limit|community tier/i);
      denial = started;
      deniedProcess = processId;
      break;
    }
    assert.strictEqual(started.type, "process_started", JSON.stringify(started));
    acceptedStarts += 1;
    const aborted = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "abort_process",
        process: processId,
      })
    );
    assert.strictEqual(aborted.type, "process_aborted", JSON.stringify(aborted));
  }
  assert(
    denial,
    `live community-tier spawn quota was not reached in ${maxAttempts} attempts`
  );
  const processes = await sendHostedControl(
    authenticatedRequest(sessionSecret, { type: "list_processes" })
  );
  assert.strictEqual(
    processes.type,
    "process_statuses",
    JSON.stringify(processes)
  );
  assert(
    !JSON.stringify(processes).includes(deniedProcess),
    "quota-denied process was allocated before denial"
  );
  const second = readJson(
    path.resolve(process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE)
  );
  assert.notStrictEqual(second.session_secret, sessionSecret);
  const unrelatedProcess = `vp-unrelated-quota-${suffix}`;
  const unrelatedStarted = await sendHostedControl(
    authenticatedRequest(second.session_secret, {
      type: "start_process",
      process: unrelatedProcess,
      restart: false,
    })
  );
  assert.strictEqual(
    unrelatedStarted.type,
    "process_started",
    `first tenant quota exhaustion affected another tenant: ${JSON.stringify(unrelatedStarted)}`
  );
  const unrelatedAborted = await sendHostedControl(
    authenticatedRequest(second.session_secret, {
      type: "abort_process",
      process: unrelatedProcess,
    })
  );
  assert.strictEqual(
    unrelatedAborted.type,
    "process_aborted",
    JSON.stringify(unrelatedAborted)
  );
  return {
    limit: spawnQuota.limit,
    initial_usage: spawnQuota.usage,
    window_seconds: spawnQuota.windowSeconds,
    accepted_starts_before_denial: acceptedStarts,
    denied_process: deniedProcess,
    denial: {
      type: denial.type,
      category: denial.error?.category || null,
      message: denial.message,
    },
    denied_process_allocated: false,
    unrelated_tenant: {
      tenant: second.tenant,
      process: unrelatedProcess,
      start: unrelatedStarted.type,
      abort: unrelatedAborted.type,
      unaffected_by_first_tenant_quota: true,
    },
  };
}

async function runLongJoinProof({ clusterflux, projectDir, scope }) {
  const startedAt = Date.now();
  const runReport = runJson(
    clusterflux,
    ["run", "long-join", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(runReport.status, "main_launched");
  const terminal = await waitForCli(
    "controlled task longer than two minutes",
    () =>
      runJson(
        clusterflux,
        ["task", "list", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (tasks) =>
      rawTaskEvents(tasks).some(
        (event) =>
          event.task_definition === "long_join_probe" &&
          event.terminal_state === "completed"
      ),
    5 * 60 * 1000
  );
  const completed = rawTaskEvents(terminal).find(
    (event) =>
      event.task_definition === "long_join_probe" &&
      event.terminal_state === "completed"
  );
  const durationSeconds = Math.floor((Date.now() - startedAt) / 1000);
  assert(durationSeconds > 120);
  const released = await waitForCli(
    "long join process automatic slot release",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (status) => status.state === "not_active",
    120000
  );
  return {
    process: runReport.process,
    task: completed.task,
    terminal_state: completed.terminal_state,
    result: completed.result,
    duration_seconds: durationSeconds,
    process_state: released.state,
  };
}

async function runRepeatedParkWakeProof({ clusterflux, projectDir, scope }) {
  const runReport = runJson(
    clusterflux,
    ["run", "park-wake", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(runReport.status, "main_launched");
  const terminal = await waitForCli(
    "repeated coordinator-main park/wake completion",
    () =>
      runJson(
        clusterflux,
        ["task", "list", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (tasks) => {
      const completed = new Set(
        rawTaskEvents(tasks)
          .filter(
            (event) =>
              event.task_definition === "task_add_one" &&
              event.terminal_state === "completed"
          )
          .map((event) => event.task)
      );
      return completed.size >= 16;
    },
    120000
  );
  const completedTasks = [
    ...new Set(
      rawTaskEvents(terminal)
        .filter(
          (event) =>
            event.task_definition === "task_add_one" &&
            event.terminal_state === "completed"
        )
        .map((event) => event.task)
    ),
  ];
  const released = await waitForCli(
    "park/wake process automatic slot release",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (status) => status.state === "not_active",
    120000
  );
  return {
    process: runReport.process,
    completed_cycles: completedTasks.length,
    task_instances: completedTasks,
    terminal_state: released.state,
  };
}

async function runDroppedConnectionRollback({
  clusterflux,
  projectDir,
  scope,
}) {
  const scenarioStartedAt = Date.now();
  const processId = "vp-current";
  const attempted = cp.spawnSync(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    {
      cwd: projectDir,
      env: {
        ...process.env,
        CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION: "start_process",
      },
      encoding: "utf8",
      timeout: 5 * 60 * 1000,
    }
  );
  assert.notStrictEqual(
    attempted.status,
    0,
    `response-loss run unexpectedly succeeded: ${attempted.stdout}`
  );
  assert.match(
    `${attempted.stderr}\n${attempted.stdout}`,
    /closed.*without a response|transport|response/i
  );
  const released = await waitForCli(
    "CLI launch guard rollback after lost start response",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", processId],
        { cwd: projectDir }
      ),
    (status) => status.state === "not_active",
    30000
  );
  return {
    duration_ms: Date.now() - scenarioStartedAt,
    process: processId,
    failure_injection: "control transport dropped start_process response",
    cli_exit_status: attempted.status,
    rollback: "automatic CLI launch guard abort_process",
    terminal_state: released.state,
  };
}

async function runLiveBandwidthPreallocation({
  sessionSecret,
  artifact,
  maxBytes,
}) {
  const scenarioStartedAt = Date.now();
  const before = relayDurableState();
  const partialLink = await sendHostedControl(
    authenticatedRequest(sessionSecret, {
      type: "create_artifact_download_link",
      artifact,
      max_bytes: maxBytes,
      ttl_seconds: 300,
    })
  );
  assert.strictEqual(
    partialLink.type,
    "artifact_download_link",
    JSON.stringify(partialLink)
  );
  const partialToken = partialLink.link.scoped_token_digest;
  let partialChunk;
  const partialDeadline = Date.now() + 120_000;
  while (Date.now() < partialDeadline) {
    const response = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "open_artifact_download_stream",
        artifact,
        max_bytes: maxBytes,
        token_digest: partialToken,
        chunk_bytes: 16,
      })
    );
    assert.strictEqual(
      response.type,
      "artifact_download_stream",
      JSON.stringify(response)
    );
    if (response.content_bytes_available === true) {
      partialChunk = response;
      break;
    }
    await delay(50);
  }
  assert(partialChunk, "artifact relay did not return the first partial chunk");
  assert.strictEqual(partialChunk.content_eof, false);
  assert.strictEqual(partialChunk.streamed_bytes, 16);
  const partialRevoked = await sendHostedControl(
    authenticatedRequest(sessionSecret, {
      type: "revoke_artifact_download_link",
      artifact,
      token_digest: partialToken,
    })
  );
  assert.strictEqual(
    partialRevoked.type,
    "artifact_download_link_revoked",
    JSON.stringify(partialRevoked)
  );
  const afterPartialAbandon = relayDurableState();
  assert(afterPartialAbandon.ingress_used > before.ingress_used);
  assert(afterPartialAbandon.egress_used > before.egress_used);
  assert(
    afterPartialAbandon.abandoned_or_failed_used >
      before.abandoned_or_failed_used
  );

  let denial;
  const acceptedReservations = [];
  for (let index = 0; index < 64; index += 1) {
    const response = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "create_artifact_download_link",
        artifact,
        max_bytes: maxBytes,
        ttl_seconds: 300,
      })
    );
    if (response.type === "error") {
      assert.match(
        response.message,
        /artifact relay.*(?:concurrency|period byte budget)|bandwidth|reservation/i
      );
      denial = response;
      break;
    }
    assert.strictEqual(response.type, "artifact_download_link", JSON.stringify(response));
    acceptedReservations.push(response.link.scoped_token_digest);
  }
  assert(denial, "artifact relay preallocation was not denied in 64 reservations");
  for (const tokenDigest of acceptedReservations) {
    const revoked = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "revoke_artifact_download_link",
        artifact,
        token_digest: tokenDigest,
      })
    );
    assert.strictEqual(
      revoked.type,
      "artifact_download_link_revoked",
      JSON.stringify(revoked)
    );
  }
  const afterRelease = relayDurableState();
  assert.strictEqual(
    Object.keys(afterRelease.reservations || {}).length,
    Object.keys(before.reservations || {}).length
  );
  return {
    scenario_started_at_ms: scenarioStartedAt,
    artifact,
    accepted_reservations_before_denial: acceptedReservations.length,
    bytes_served_before_denial: partialChunk.streamed_bytes,
    partial_abandon: {
      ingress_delta: afterPartialAbandon.ingress_used - before.ingress_used,
      egress_delta: afterPartialAbandon.egress_used - before.egress_used,
      abandoned_delta:
        afterPartialAbandon.abandoned_or_failed_used -
        before.abandoned_or_failed_used,
      reservation_released: !(partialToken in afterRelease.reservations),
    },
    durable_before: before,
    durable_after_release: afterRelease,
    denial: {
      type: denial.type,
      category: denial.error?.category || null,
      message: denial.message,
    },
  };
}

function relayDurableState() {
  if (!strictVpsRestart) {
    throw new Error("strict relay accounting requires VPS Postgres access");
  }
  const output = run(
    "ssh",
    sshArgs(
      "sudo",
      "-u",
      "postgres",
      "psql",
      "-d",
      "clusterflux",
      "-At",
      "-c",
      "SELECT record::text FROM clusterflux_artifact_relay_state WHERE singleton = TRUE"
    )
  ).trim();
  assert.notStrictEqual(output, "", "artifact relay durable state was not persisted");
  return JSON.parse(output.split(/\r?\n/).filter(Boolean).at(-1));
}

async function waitForHostedControlReady(timeoutMs = 60_000) {
  const deadline = Date.now() + timeoutMs;
  let lastError;
  while (Date.now() < deadline) {
    try {
      const ping = await sendHostedControl({ type: "ping" });
      if (ping.type === "pong") return ping;
      lastError = new Error(JSON.stringify(ping));
    } catch (error) {
      lastError = error;
    }
    await delay(500);
  }
  throw new Error(`hosted service did not recover: ${lastError}`);
}

async function publishRelayProbeArtifact({ clusterflux, projectDir, scope, label }) {
  const runReport = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  await waitForCli(
    `${label} flagship completion`,
    () =>
      runJson(
        clusterflux,
        ["task", "list", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (report) => completedFlagship(rawTaskEvents(report)),
    10 * 60 * 1000
  );
  const artifacts = runJson(
    clusterflux,
    ["artifact", "list", ...scope, "--process", runReport.process],
    { cwd: projectDir }
  );
  const artifact = artifacts.artifacts.find((candidate) => {
    const digest = candidate.digest;
    return (
      typeof digest === "string" &&
      /^sha256:[0-9a-f]{64}$/.test(digest) &&
      candidate.artifact === `release.tar-${digest.slice("sha256:".length)}`
    );
  });
  assert(artifact, `${label} did not publish a relay probe artifact`);
  return artifact;
}

async function runRelayEmergencyDisable({
  clusterflux,
  clusterfluxNode,
  projectDir,
  scope,
  workerArgs,
  sessionSecret,
  maxBytes,
}) {
  if (!strictVpsRestart) {
    throw new Error("strict relay disable proof requires VPS systemd access");
  }
  const dropInDir = `/run/systemd/system/${strictServiceUnit}.d`;
  const dropInPath = `${dropInDir}/90-strict-relay-disable.conf`;
  run("ssh", sshArgs("sudo", "mkdir", "-p", dropInDir));
  cp.execFileSync("ssh", sshArgs("sudo", "tee", dropInPath), {
    cwd: repo,
    input: "[Service]\nEnvironment=CLUSTERFLUX_HOSTED_RELAY_DISABLED=true\n",
    stdio: ["pipe", "ignore", "inherit"],
  });
  let disabled;
  let disabledArtifact;
  let disabledWorker;
  try {
    run(
      "ssh",
      sshArgs(
        "sudo",
        "systemctl",
        "daemon-reload"
      )
    );
    run(
      "ssh",
      sshArgs("sudo", "systemctl", "restart", strictServiceUnit)
    );
    await waitForHostedControlReady();
    disabledWorker = spawnJsonLines(clusterfluxNode, workerArgs, {
      cwd: projectDir,
      env: { ...process.env },
    });
    await disabledWorker.waitFor(
      (value) => value.node_status === "ready",
      "relay-disabled worker ready"
    );
    disabledArtifact = await publishRelayProbeArtifact({
      clusterflux,
      projectDir,
      scope,
      label: "relay-disabled",
    });
    disabled = assertDenied(
      await sendHostedControl(
        authenticatedRequest(sessionSecret, {
          type: "create_artifact_download_link",
          artifact: disabledArtifact.artifact,
          max_bytes: maxBytes,
          ttl_seconds: 60,
        })
      ),
      /artifact relay is disabled/i,
      "emergency-disabled artifact relay"
    );
  } finally {
    if (disabledWorker) await stopChild(disabledWorker.child);
    run("ssh", sshArgs("sudo", "rm", "-f", dropInPath));
    run("ssh", sshArgs("sudo", "systemctl", "daemon-reload"));
    run(
      "ssh",
      sshArgs("sudo", "systemctl", "restart", strictServiceUnit)
    );
    await waitForHostedControlReady();
  }
  const restoredWorker = spawnJsonLines(clusterfluxNode, workerArgs, {
    cwd: projectDir,
    env: { ...process.env },
  });
  try {
    await restoredWorker.waitFor(
      (value) => value.node_status === "ready",
      "relay-restored worker ready"
    );
    const restoredArtifact = await publishRelayProbeArtifact({
      clusterflux,
      projectDir,
      scope,
      label: "relay-restored",
    });
    const restored = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "create_artifact_download_link",
        artifact: restoredArtifact.artifact,
        max_bytes: maxBytes,
        ttl_seconds: 60,
      })
    );
    assert.strictEqual(restored.type, "artifact_download_link", JSON.stringify(restored));
    const revoked = await sendHostedControl(
      authenticatedRequest(sessionSecret, {
        type: "revoke_artifact_download_link",
        artifact: restoredArtifact.artifact,
        token_digest: restored.link.scoped_token_digest,
      })
    );
    assert.strictEqual(revoked.type, "artifact_download_link_revoked");
    return {
      evidence: {
        disabled,
        disabled_probe_artifact: disabledArtifact.artifact,
        restored: restored.type,
        restored_probe_artifact: restoredArtifact.artifact,
        restored_reservation_released: revoked.type,
        runtime_drop_in_removed: true,
      },
      worker: restoredWorker,
    };
  } catch (error) {
    await stopChild(restoredWorker.child);
    throw error;
  }
}

function deploymentProvenance() {
  if (!strictVpsRestart) return null;
  const systemGeneration = run(
    "ssh",
    sshArgs("readlink", "-f", "/run/current-system")
  ).trim();
  const mainPid = run(
    "ssh",
    sshArgs(
      "systemctl",
      "show",
      strictServiceUnit,
      "--property=MainPID",
      "--value"
    )
  ).trim();
  assert.match(mainPid, /^[1-9][0-9]*$/, "hosted service has no live MainPID");
  const executable = run(
    "ssh",
    sshArgs("readlink", "-f", `/proc/${mainPid}/exe`)
  ).trim();
  const binarySha256 = run(
    "ssh",
    sshArgs("sha256sum", `/proc/${mainPid}/exe`)
  )
    .trim()
    .split(/\s+/)[0];
  const fragment = run(
    "ssh",
    sshArgs("systemctl", "show", strictServiceUnit, "--property=FragmentPath", "--value")
  ).trim();
  const serviceUnitSha256 = run(
    "ssh",
    sshArgs("sha256sum", fragment)
  )
    .trim()
    .split(/\s+/)[0];
  return {
    system_generation: systemGeneration,
    hosted_service_executable: executable,
    hosted_service_sha256: `sha256:${binarySha256}`,
    service_unit: fragment,
    service_unit_sha256: `sha256:${serviceUnitSha256}`,
  };
}

function configurationProvenance() {
  const configRepo = path.resolve(
    process.env.CLUSTERFLUX_DEPLOYMENT_CONFIG_REPO ||
      path.join(repo, "..", "michelpaulissen.com")
  );
  const status = run("git", ["status", "--short"], { cwd: configRepo }).trim();
  return {
    repository: configRepo,
    revision: run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(),
    clean: status === "",
    status: status || null,
  };
}

async function restartHostedServiceAndResume({
  clusterflux,
  clusterfluxNode,
  projectDir,
  scope,
  workerArgs,
  workerNode,
  credentialPath,
  credentialDigest,
}) {
  if (!strictVpsRestart) {
    return {
      executed: false,
      reason: "strict VPS restart access not supplied",
      worker: null,
    };
  }
  const before = deploymentProvenance();
  run("ssh", sshArgs("systemctl", "restart", strictServiceUnit));
  const deadline = Date.now() + 120000;
  let ping;
  let lastError;
  while (Date.now() < deadline) {
    try {
      ping = await sendHostedControl({ type: "ping" });
      if (ping.type === "pong") break;
    } catch (error) {
      lastError = error;
    }
    await delay(500);
  }
  assert.strictEqual(
    ping?.type,
    "pong",
    `hosted service did not recover after restart: ${lastError?.message || "no pong"}`
  );
  const afterFirstRestart = deploymentProvenance();
  assert.strictEqual(afterFirstRestart.system_generation, before.system_generation);
  assert.strictEqual(
    afterFirstRestart.hosted_service_sha256,
    before.hosted_service_sha256
  );

  const projects = runJson(clusterflux, ["project", "list", ...scope], {
    cwd: projectDir,
  });
  assert(projects.project_count >= 1, "CLI session/project did not survive restart");
  const ephemeralRun = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(ephemeralRun.status, "main_launched");
  const ephemeralProcess = ephemeralRun.process;
  await waitForCli(
    "pre-restart ephemeral main to park",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", ephemeralProcess],
        { cwd: projectDir }
      ),
    (status) => status.live_process?.main_wait_state === "waiting_for_node",
    120000
  );

  run("ssh", sshArgs("systemctl", "restart", strictServiceUnit));
  const secondDeadline = Date.now() + 120000;
  ping = undefined;
  lastError = undefined;
  while (Date.now() < secondDeadline) {
    try {
      ping = await sendHostedControl({ type: "ping" });
      if (ping.type === "pong") break;
    } catch (error) {
      lastError = error;
    }
    await delay(500);
  }
  assert.strictEqual(
    ping?.type,
    "pong",
    `hosted service did not recover after ephemeral-state restart: ${
      lastError?.message || "no pong"
    }`
  );
  const after = deploymentProvenance();
  assert.strictEqual(after.system_generation, before.system_generation);
  assert.strictEqual(after.hosted_service_sha256, before.hosted_service_sha256);
  const oldStatus = runJson(
    clusterflux,
    ["process", "status", ...scope, "--process", ephemeralProcess],
    { cwd: projectDir }
  );
  assert.strictEqual(oldStatus.state, "not_active");
  assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);

  const worker = spawnJsonLines(clusterfluxNode, workerArgs, {
    cwd: projectDir,
    env: { ...process.env },
  });
  const ready = await worker.waitFor(
    (value) => value.node_status === "ready",
    "persisted worker ready after hosted service restart"
  );
  assert.strictEqual(ready.node, workerNode);
  assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);

  const restartedRun = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(restartedRun.status, "main_launched");
  const restartedProcess = restartedRun.process;
  const completed = await waitForCli(
    "new process completion after hosted service restart",
    () =>
      runJson(
        clusterflux,
        ["task", "list", ...scope, "--process", restartedProcess],
        { cwd: projectDir }
      ),
    (tasks) => completedFlagship(rawTaskEvents(tasks)),
    5 * 60 * 1000
  );
  const completedEvent = rawTaskEvents(completed).find(
    (event) =>
      event.executor === "coordinator_main" &&
      event.terminal_state === "completed"
  );
  assert(completedEvent, "post-restart flagship omitted its completed main event");
  return {
    executed: true,
    account_project_session_preserved: true,
    node_identity_preserved: true,
    old_process_terminated_honestly: true,
    terminated_ephemeral_process: ephemeralProcess,
    new_process: restartedProcess,
    new_process_result: completedEvent.result,
    before,
    after_first_restart: afterFirstRestart,
    after,
    worker,
  };
}

async function finishUserCredentialSecurity({
  clusterflux,
  projectDir,
  scope,
  sessionSecret,
}) {
  const forged = assertDenied(
    await sendHostedControl(
      authenticatedRequest(
        `forged-session-${crypto.randomBytes(24).toString("hex")}`,
        { type: "list_projects" }
      )
    ),
    /not recognized|invalid|authentication|session/i,
    "forged user session"
  );

  let expired = null;
  const expiredFile = process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE;
  if (expiredFile) {
    const oldSession = readJson(path.resolve(expiredFile));
    assert(oldSession.session_secret, "expired session evidence omitted its secret");
    expired = assertDenied(
      await sendHostedControl(
        authenticatedRequest(oldSession.session_secret, { type: "list_projects" })
      ),
      /expired|not recognized/i,
      "expired user session"
    );
  } else if (strictFullRelease) {
    throw new Error(
      "strict full release requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE for a genuinely expired hosted session"
    );
  }

  const logout = runJson(clusterflux, ["logout", ...scope, "--yes"], {
    cwd: projectDir,
  });
  assert.strictEqual(logout.server_session_revocation.revoked, true);
  const revoked = assertDenied(
    await sendHostedControl(
      authenticatedRequest(sessionSecret, { type: "list_projects" })
    ),
    /revoked|not recognized/i,
    "revoked user session"
  );
  return { forged, expired, revoked };
}

async function dapVariables(client, variablesReference) {
  const request = client.send("variables", { variablesReference });
  return (await client.response(request, "variables")).body.variables;
}

async function runSameDefinitionDapIdentity({
  clusterfluxDap,
  clusterflux,
  projectDir,
  scope,
}) {
  const scenarioStartedAt = Date.now();
  const baselineContainers = new Set(
    run("podman", ["ps", "-q"])
      .trim()
      .split(/\s+/)
      .filter(Boolean)
  );
  const client = new DapClient({
    cwd: projectDir,
    command: clusterfluxDap,
    args: [],
    env: {
      ...process.env,
      CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1",
    },
  });
  let disconnected = false;
  try {
    const initialize = client.send("initialize", {
      adapterID: "clusterflux",
      linesStartAt1: true,
      columnsStartAt1: true,
    });
    await client.response(initialize, "initialize");
    const launch = client.send("launch", {
      entry: "identity",
      project: projectDir,
      runtimeBackend: "live-services",
      coordinatorEndpoint: serviceEndpoint,
    });
    await client.response(launch, "launch");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "initialized"
    );
    const configurationDone = client.send("configurationDone");
    await client.response(configurationDone, "configurationDone");

    const mainThreadDeadline = Date.now() + 5 * 60 * 1000;
    let mainThread;
    while (Date.now() < mainThreadDeadline) {
      const threadsRequest = client.send("threads");
      const threads = (await client.response(threadsRequest, "threads")).body
        .threads;
      mainThread = threads.find((thread) => /coordinator main/i.test(thread.name));
      if (mainThread) break;
      await delay(100);
    }
    assert(mainThread, "identity entry did not report its coordinator main");
    const processRequest = client.send("evaluate", {
      expression: "virtual_process_id",
      context: "watch",
    });
    const processId = (await client.response(processRequest, "evaluate")).body
      .result;
    const identityContainerDeadline = Date.now() + 2 * 60 * 1000;
    let identityContainerIds = new Set();
    let maximumNewContainerCount = 0;
    let lastPodmanStates = [];
    while (Date.now() < identityContainerDeadline) {
      const containers = runJson("podman", ["ps", "--format", "json"]);
      lastPodmanStates = containers;
      identityContainerIds = new Set(
        containers
          .map((container) => container.Id || container.ID || container.IdHex || "")
          .filter((id) => id && !baselineContainers.has(id))
      );
      maximumNewContainerCount = Math.max(
        maximumNewContainerCount,
        identityContainerIds.size
      );
      if (identityContainerIds.size >= 2) break;
      await delay(100);
    }
    const identityTaskSnapshot =
      identityContainerIds.size >= 2
        ? null
        : runJson(
            clusterflux,
            ["task", "list", ...scope, "--process", processId],
            { cwd: projectDir }
          );
    assert(
      identityContainerIds.size >= 2,
      `identity entry did not start two same-definition Podman containers; max=${maximumNewContainerCount} tasks=${JSON.stringify(
        identityTaskSnapshot
      )} podman=${JSON.stringify(lastPodmanStates)}`
    );

    const pause = client.send("pause", { threadId: mainThread.id });
    await client.response(pause, "pause");
    const paused = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "pause",
      30_000
    );
    assert.strictEqual(typeof paused.body.allThreadsStopped, "boolean");

    const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]);
    const clusterfluxPausedContainers = podmanStates.filter((container) => {
      const id = container.Id || container.ID || container.IdHex || "";
      const state = String(container.State || container.Status || "").toLowerCase();
      return identityContainerIds.has(id) && state.includes("paused");
    });
    assert(
      clusterfluxPausedContainers.length >= 2,
      `expected two newly paused Clusterflux containers: ${JSON.stringify(
        podmanStates
      )}`
    );

    const threadsRequest = client.send("threads");
    const threads = (await client.response(threadsRequest, "threads")).body
      .threads;
    const identityThreads = threads.filter((thread) =>
      /identity probe/i.test(thread.name)
    );
    assert.strictEqual(
      identityThreads.length,
      2,
      `expected two same-definition DAP threads: ${JSON.stringify(threads)}`
    );
    assert.notStrictEqual(identityThreads[0].id, identityThreads[1].id);

    const threadEvidence = [];
    for (const threadRecord of identityThreads) {
      const stackRequest = client.send("stackTrace", {
        threadId: threadRecord.id,
        startFrame: 0,
        levels: 1,
      });
      const frames = (await client.response(stackRequest, "stackTrace")).body
        .stackFrames;
      assert.strictEqual(frames.length, 1);
      const scopesRequest = client.send("scopes", { frameId: frames[0].id });
      const scopes = (await client.response(scopesRequest, "scopes")).body.scopes;
      const argsScope = scopes.find(
        (scopeRecord) => scopeRecord.name === "Task Args and Handles"
      );
      const runtimeScope = scopes.find(
        (scopeRecord) => scopeRecord.name === "Clusterflux Runtime"
      );
      assert(argsScope && runtimeScope);
      const args = await dapVariables(client, argsScope.variablesReference);
      const runtime = await dapVariables(client, runtimeScope.variablesReference);
      const runtimeValue = (name) =>
        runtime.find((variable) => variable.name === name)?.value;
      assert.strictEqual(runtimeValue("state"), "Frozen");
      threadEvidence.push({
        thread_id: threadRecord.id,
        task_instance: runtimeValue("virtual_thread"),
        attempt_id: runtimeValue("task_attempt_id"),
        arguments: args.map((variable) => ({
          name: variable.name,
          value: variable.value,
        })),
      });
    }
    assert.notStrictEqual(
      threadEvidence[0].task_instance,
      threadEvidence[1].task_instance
    );
    assert.notStrictEqual(threadEvidence[0].attempt_id, threadEvidence[1].attempt_id);
    const argumentText = threadEvidence
      .flatMap((record) => record.arguments.map((argument) => argument.value))
      .join(" ");
    assert.match(argumentText, /slow-first/);
    assert.match(argumentText, /fast-second/);

    const continued = client.send("continue", {
      threadId: paused.body.threadId,
    });
    await client.response(continued, "continue");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "terminated",
      5 * 60 * 1000
    );

    const taskList = runJson(
      clusterflux,
      ["task", "list", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    const identityEvents = rawTaskEvents(taskList).filter(
      (event) =>
        event.task_definition === "identity_probe" &&
        event.terminal_state === "completed"
    );
    assert.strictEqual(identityEvents.length, 2, JSON.stringify(identityEvents));
    assert.notStrictEqual(identityEvents[0].task, identityEvents[1].task);
    assert.notStrictEqual(identityEvents[0].attempt_id, identityEvents[1].attempt_id);
    assert.match(JSON.stringify(identityEvents[0].result), /fast-second/);
    assert.match(JSON.stringify(identityEvents[1].result), /slow-first/);

    const released = await waitForCli(
      "same-definition identity process release",
      () =>
        runJson(
          clusterflux,
          ["process", "status", ...scope, "--process", processId],
          { cwd: projectDir }
        ),
      (status) => status.state === "not_active",
      120_000
    );
    await client.close();
    disconnected = true;
    return {
      duration_ms: Date.now() - scenarioStartedAt,
      process: processId,
      thread_evidence: threadEvidence,
      completion_order: identityEvents.map((event) => ({
        task_instance: event.task,
        attempt_id: event.attempt_id,
        result: event.result,
      })),
      podman_paused_container_ids: clusterfluxPausedContainers.map(
        (container) => container.Id || container.ID || container.IdHex
      ),
      dap_all_threads_stopped: paused.body.allThreadsStopped,
      terminal_state: released.state,
    };
  } finally {
    if (!disconnected) {
      await client.close().catch(() => {
        if (client.child.exitCode === null) client.child.kill("SIGKILL");
      });
    }
  }
}

async function runLiveDapEditRestart({
  clusterfluxDap,
  clusterflux,
  projectDir,
  scope,
  worker,
}) {
  const scenarioStartedAt = Date.now();
  const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs"));
  const originalSource = fs.readFileSync(sourcePath, "utf8");
  const sourceLines = originalSource.split(/\r?\n/);
  const lineFor = (needle) => {
    const line = sourceLines.findIndex((candidate) => candidate.includes(needle)) + 1;
    assert(line > 0, `flagship source omitted ${needle}`);
    return line;
  };
  const mainLine = lineFor("pub async fn restart_main()");
  const taskLine = lineFor("fn task_trap(");
  const client = new DapClient({
    cwd: projectDir,
    command: clusterfluxDap,
    args: [],
    env: { ...process.env },
  });
  let sourceRestored = false;
  let disconnected = false;
  try {
    const initialize = client.send("initialize", {
      adapterID: "clusterflux",
      linesStartAt1: true,
      columnsStartAt1: true,
    });
    await client.response(initialize, "initialize");

    const launch = client.send("launch", {
      entry: "restart",
      project: projectDir,
      runtimeBackend: "live-services",
      coordinatorEndpoint: serviceEndpoint,
    });
    await client.response(launch, "launch");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "initialized"
    );

    const configurationStartedAt = Date.now();
    const configurationDone = client.send("configurationDone");
    await client.response(configurationDone, "configurationDone");
    const configurationResponseMs = Date.now() - configurationStartedAt;
    assert(
      configurationResponseMs < 2_000,
      `configurationDone blocked for ${configurationResponseMs} ms`
    );

    const threadDeadline = Date.now() + 5 * 60 * 1000;
    let runningThreads = [];
    while (Date.now() < threadDeadline) {
      const threadsRequest = client.send("threads");
      runningThreads = (await client.response(threadsRequest, "threads")).body
        .threads;
      if (runningThreads.length > 0) break;
      await delay(100);
    }
    assert(runningThreads.length > 0, "asynchronous live DAP launch reported no threads");
    const observerReconnect = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "output" &&
        String(message.body?.output || "").includes(
          "injected one transient connection loss"
        ),
      30_000
    );
    assert.strictEqual(
      client.messages.filter(
        (message) =>
          message.seq <= observerReconnect.seq &&
          message.type === "event" &&
          message.event === "stopped"
      ).length,
      0,
      "observer reconnection fabricated a stopped target"
    );

    const pauseStartedAt = Date.now();
    const pause = client.send("pause", { threadId: runningThreads[0].id });
    await client.response(pause, "pause");
    const pauseResponseMs = Date.now() - pauseStartedAt;
    assert(pauseResponseMs < 2_000, `pause blocked for ${pauseResponseMs} ms`);
    const mainStop = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "pause"
    );
    assert.strictEqual(mainStop.body.allThreadsStopped, true);

    const inspectStartedAt = Date.now();
    const commandStatus = client.send("evaluate", {
      expression: "command_status",
      context: "watch",
    });
    const inspected = await client.response(commandStatus, "evaluate");
    const inspectionResponseMs = Date.now() - inspectStartedAt;
    assert(
      inspectionResponseMs < 2_000,
      `inspection blocked for ${inspectionResponseMs} ms`
    );
    assert.match(inspected.body.result, /debug epoch|frozen/i);

    const breakpointStartedAt = Date.now();
    const setBreakpoints = client.send("setBreakpoints", {
      source: { path: sourcePath },
      breakpoints: [{ line: taskLine }],
    });
    const breakpointResponse = await client.response(setBreakpoints, "setBreakpoints");
    const breakpointResponseMs = Date.now() - breakpointStartedAt;
    assert(
      breakpointResponseMs < 2_000,
      `setBreakpoints blocked for ${breakpointResponseMs} ms`
    );
    assert.deepStrictEqual(
      breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified),
      [true]
    );

    const continueStartedAt = Date.now();
    const mainContinue = client.send("continue", {
      threadId: mainStop.body.threadId,
    });
    await client.response(mainContinue, "continue");
    const continueResponseMs = Date.now() - continueStartedAt;
    assert(
      continueResponseMs < 2_000,
      `continue blocked for ${continueResponseMs} ms`
    );
    const taskStop = await client.waitFor(
      (message) =>
        message.seq > mainStop.seq &&
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "breakpoint"
    );
    assert.strictEqual(taskStop.body.allThreadsStopped, true);
    assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId);

    const stackTrace = client.send("stackTrace", {
      threadId: taskStop.body.threadId,
      startFrame: 0,
      levels: 1,
    });
    const taskFrames = (await client.response(stackTrace, "stackTrace")).body
      .stackFrames;
    assert.strictEqual(taskFrames[0].line, taskLine);

    const scopesRequest = client.send("scopes", { frameId: taskFrames[0].id });
    const scopes = (await client.response(scopesRequest, "scopes")).body.scopes;
    const argsScope = scopes.find(
      (candidate) => candidate.name === "Task Args and Handles"
    );
    const runtimeScope = scopes.find(
      (candidate) => candidate.name === "Clusterflux Runtime"
    );
    assert(argsScope && runtimeScope);
    const taskArgs = await dapVariables(client, argsScope.variablesReference);
    assert(
      taskArgs.some(
        (variable) =>
          variable.name === "arg_0" && String(variable.value).includes("0")
      )
    );
    const runtime = await dapVariables(client, runtimeScope.variablesReference);
    assert(
      runtime.some(
        (variable) =>
          variable.name === "runtime_backend" && variable.value === "LiveServices"
      )
    );
    assert(
      runtime.some(
        (variable) => variable.name === "state" && variable.value === "Frozen"
      )
    );
    const dapProcess = runtime.find(
      (variable) => variable.name === "virtual_process_id"
    )?.value;
    assert.match(dapProcess || "", /^vp-[0-9a-f]{12}$/);

    const taskContinue = client.send("continue", { threadId: taskStop.body.threadId });
    await client.response(taskContinue, "continue");
    const failedStop = await client.waitFor(
      (message) =>
        message.seq > taskStop.seq &&
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "exception",
      5 * 60 * 1000
    );
    assert.strictEqual(failedStop.body.allThreadsStopped, false);

    const beforeEdit = runJson(
      clusterflux,
      ["task", "list", ...scope, "--process", dapProcess],
      { cwd: projectDir }
    );
    const originalTaskEvent = rawTaskEvents(beforeEdit).find(
      (event) =>
        event.task_definition === "task_trap" &&
        event.terminal_state === "failed"
    );
    assert(originalTaskEvent, "DAP restart entry omitted its original child event");
    assert(originalTaskEvent.attempt_id, "failed attempt omitted its identity");

    const originalTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 {
    #[cfg(target_arch = "wasm32")]
    core::arch::wasm32::unreachable();
    #[cfg(not(target_arch = "wasm32"))]
    panic!("intentional task trap")
}`;
    const replacementTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 {
    _input + 42 // strict hosted compatible restart probe
}`;
    const editedSource = originalSource.replace(originalTaskBody, replacementTaskBody);
    assert.notStrictEqual(editedSource, originalSource);
    fs.writeFileSync(sourcePath, editedSource);

    const restartFrame = client.send("restartFrame", {
      frameId: taskFrames[0].id,
    });
    const restartResponse = await client.response(restartFrame, "restartFrame");
    const restartedStop = await client.waitFor(
      (message) =>
        message.seq > restartResponse.seq &&
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "breakpoint"
    );
    assert.strictEqual(restartedStop.body.allThreadsStopped, true);
    const restartedContinue = client.send("continue", {
      threadId: restartedStop.body.threadId,
    });
    await client.response(restartedContinue, "continue");
    const restartedNodeReport = await worker.waitFor(
      (value) =>
        value.node_status === "completed" &&
        value.virtual_thread === originalTaskEvent.task &&
        value.task_assignment_response?.task_spec?.task_definition ===
          "task_trap",
      "edited task to execute on the live restarted node",
      5 * 60 * 1000
    );
    assert.strictEqual(
      restartedNodeReport.node_status,
      "completed",
      `edited task restart failed on the live node: ${JSON.stringify(restartedNodeReport)}`
    );
    assert.match(restartedNodeReport.stdout_tail, /"SmallJson":42/);
    const restartedTask = restartedNodeReport.virtual_thread;

    const afterEdit = await waitForCli(
      "edited live task event",
      () =>
        runJson(clusterflux, ["task", "list", ...scope, "--process", dapProcess], {
          cwd: projectDir,
        }),
      (tasks) =>
        rawTaskEvents(tasks).some(
          (event) =>
            event.task === restartedTask &&
            event.terminal_state === "completed" &&
            JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 })
        ),
      5 * 60 * 1000
    );
    const restartedEvent = rawTaskEvents(afterEdit).find(
      (event) =>
        event.task === restartedTask &&
        event.terminal_state === "completed" &&
        JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 })
    );
    assert(restartedEvent, "replacement attempt omitted its completed event");
    assert(restartedEvent.attempt_id, "replacement attempt omitted its identity");
    assert.notStrictEqual(restartedEvent.attempt_id, originalTaskEvent.attempt_id);
    await client.waitFor(
      (message) => message.type === "event" && message.event === "terminated",
      5 * 60 * 1000
    );
    const disconnectStartedAt = Date.now();
    await client.close();
    disconnected = true;
    const disconnectResponseMs = Date.now() - disconnectStartedAt;
    assert(
      disconnectResponseMs < 2_000,
      `disconnect blocked for ${disconnectResponseMs} ms`
    );
    fs.writeFileSync(sourcePath, originalSource);
    sourceRestored = true;

    return {
      duration_ms: Date.now() - scenarioStartedAt,
      process: dapProcess,
      main_source_line: mainLine,
      task_breakpoint_line: taskLine,
      launch_started_without_breakpoint: true,
      configuration_response_ms: configurationResponseMs,
      pause_response_ms: pauseResponseMs,
      inspection_response_ms: inspectionResponseMs,
      breakpoint_response_ms: breakpointResponseMs,
      continue_response_ms: continueResponseMs,
      disconnect_response_ms: disconnectResponseMs,
      pause_all_threads_stopped: mainStop.body.allThreadsStopped,
      task_all_threads_stopped: taskStop.body.allThreadsStopped,
      original_task_instance: originalTaskEvent.task,
      original_attempt: originalTaskEvent.attempt_id,
      restarted_task_instance: restartedTask,
      restarted_attempt: restartedEvent.attempt_id,
      restarted_result: restartedEvent.result,
      compatible_source_edit: "task_trap: trap -> input + 42",
      original_arguments_preserved: true,
      clean_vfs_boundary_used: true,
      observer_reconnected_without_false_stop: true,
    };
  } finally {
    if (!sourceRestored) fs.writeFileSync(sourcePath, originalSource);
    if (!disconnected) {
      await client.close().catch(() => {
        if (client.child.exitCode === null) client.child.kill("SIGKILL");
      });
    }
  }
}

function copyProjectControlState(sourceRoot, targetRoot) {
  const source = path.join(sourceRoot, ".clusterflux");
  const target = path.join(targetRoot, ".clusterflux");
  ensureDir(target);
  for (const file of ["session.json", "project.json"]) {
    const from = path.join(source, file);
    assert(fs.existsSync(from), `missing shared project control state ${from}`);
    fs.copyFileSync(from, path.join(target, file));
  }
  fs.chmodSync(path.join(target, "session.json"), 0o600);
  const sourceNodes = path.join(source, "nodes");
  assert(fs.existsSync(sourceNodes), "persisted node credential directory is missing");
  fs.cpSync(sourceNodes, path.join(target, "nodes"), {
    recursive: true,
    force: true,
  });
}

async function runHostedHelloBuild({
  clusterflux,
  projectDir,
  scope,
  outputFile,
}) {
  const startedAt = Date.now();
  const runReport = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(runReport.status, "main_launched");
  const tasks = await waitForCli(
    "hosted hello-build completion",
    () =>
      runJson(
        clusterflux,
        ["task", "list", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (report) => {
      const events = rawTaskEvents(report);
      return events.some(
        (event) =>
          event.task_definition === "compile" &&
          event.terminal_state === "completed"
      );
    },
    10 * 60 * 1000
  );
  const events = rawTaskEvents(tasks);
  assert(
    events.some(
      (event) =>
        event.task_definition === "snapshot_current_project" &&
        event.terminal_state === "completed"
    )
  );
  const released = await waitForCli(
    "hosted hello-build process release",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", runReport.process],
        { cwd: projectDir }
      ),
    (status) => status.state === "not_active",
    120_000
  );
  const artifacts = runJson(
    clusterflux,
    ["artifact", "list", ...scope, "--process", runReport.process],
    { cwd: projectDir }
  );
  const artifact = artifacts.artifacts.find(
    (candidate) =>
      typeof candidate.digest === "string" &&
      /^sha256:[0-9a-f]{64}$/.test(candidate.digest) &&
      candidate.artifact.startsWith("hello-clusterflux-")
  );
  assert(artifact, "hello-build did not retain its executable artifact");
  const download = runJson(
    clusterflux,
    [
      "artifact",
      "download",
      ...scope,
      artifact.artifact,
      "--to",
      outputFile,
    ],
    { cwd: projectDir }
  );
  assert.strictEqual(download.local_download.verified_digest, artifact.digest);
  assert.strictEqual(sha256(fs.readFileSync(outputFile)), artifact.digest);
  fs.chmodSync(outputFile, 0o755);
  const output = run(outputFile, []).trim();
  assert.strictEqual(output, "hello from a real Clusterflux build");
  return {
    duration_ms: Date.now() - startedAt,
    process: runReport.process,
    terminal_state: released.state,
    artifact: artifact.artifact,
    digest: artifact.digest,
    executable_output: output,
  };
}

async function runHostedRecoveryBuild({
  clusterfluxDap,
  clusterflux,
  projectDir,
  scope,
}) {
  const startedAt = Date.now();
  const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs"));
  const original = fs.readFileSync(sourcePath, "utf8");
  const failing = '"exit 23".to_owned()';
  const replacement =
    '"printf \'recovered\\n\' > /clusterflux/output/recovering.txt".to_owned()';
  assert(original.includes(failing));
  const client = new DapClient({
    cwd: projectDir,
    command: clusterfluxDap,
    env: { ...process.env, CLUSTERFLUX_DAP_TIMEOUT_MS: "120000" },
  });
  let restored = false;
  let closed = false;
  try {
    const initialize = client.send("initialize", {
      adapterID: "clusterflux",
      linesStartAt1: true,
      columnsStartAt1: true,
    });
    await client.response(initialize, "initialize");
    const launch = client.send("launch", {
      entry: "build",
      project: projectDir,
      runtimeBackend: "live-services",
      coordinatorEndpoint: serviceEndpoint,
    });
    await client.response(launch, "launch");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "initialized"
    );
    const configured = client.send("configurationDone");
    await client.response(configured, "configurationDone");
    const failedStop = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "exception",
      10 * 60 * 1000
    );
    assert.strictEqual(failedStop.body.allThreadsStopped, false);
    const threadRequest = client.send("threads");
    const threads = (await client.response(threadRequest, "threads")).body.threads;
    const recoveringThread = threads.find((thread) => /build lane/.test(thread.name));
    assert(recoveringThread, "recovery-build failed lane is not visible in DAP");
    const processId = threads
      .map((thread) => /vp-[0-9a-f]{12}/.exec(thread.name)?.[0])
      .find(Boolean);
    assert(processId, "recovery-build DAP threads omitted the process identity");
    const before = runJson(
      clusterflux,
      ["task", "list", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    const beforeEvents = rawTaskEvents(before);
    const originalFailure = beforeEvents.find(
      (event) =>
        event.task_definition === "build_lane" &&
        event.terminal_state === "failed"
    );
    assert(originalFailure?.attempt_id);
    assert(
      beforeEvents.some(
        (event) =>
          event.task_definition === "build_lane" &&
          event.task !== originalFailure.task &&
          event.terminal_state === "completed"
      )
    );
    const waiting = runJson(
      clusterflux,
      ["process", "status", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    assert.strictEqual(waiting.live_process.main_state, "running");
    fs.writeFileSync(sourcePath, original.replace(failing, replacement));
    const restart = client.send("restartFrame", {
      threadId: recoveringThread.id,
    });
    await client.response(restart, "restartFrame");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "terminated",
      10 * 60 * 1000
    );
    assert(
      client.messages.some(
        (message) =>
          message.type === "event" &&
          message.event === "thread" &&
          message.body.reason === "exited" &&
          message.body.threadId === recoveringThread.id
      )
    );
    const after = await waitForCli(
      "hosted recovery replacement event",
      () =>
        runJson(
          clusterflux,
          ["task", "list", ...scope, "--process", processId],
          { cwd: projectDir }
        ),
      (report) =>
        rawTaskEvents(report).some(
          (event) =>
            event.task === originalFailure.task &&
            event.terminal_state === "completed"
        ),
      120_000
    );
    const replacementEvent = rawTaskEvents(after).find(
      (event) =>
        event.task === originalFailure.task &&
        event.terminal_state === "completed"
    );
    assert(replacementEvent?.attempt_id);
    assert.notStrictEqual(replacementEvent.attempt_id, originalFailure.attempt_id);
    assert(
      rawTaskEvents(after).some(
        (event) =>
          event.executor === "coordinator_main" &&
          event.terminal_state === "completed"
      ),
      "replacement result did not satisfy the original coordinator-main join"
    );
    const artifacts = runJson(
      clusterflux,
      ["artifact", "list", ...scope, "--process", processId],
      { cwd: projectDir }
    ).artifacts;
    assert(artifacts.some((artifact) => artifact.artifact.startsWith("stable.txt-")));
    assert(
      artifacts.some((artifact) => artifact.artifact.startsWith("recovering.txt-"))
    );
    const released = await waitForCli(
      "hosted recovery process release",
      () =>
        runJson(
          clusterflux,
          ["process", "status", ...scope, "--process", processId],
          { cwd: projectDir }
        ),
      (status) => status.state === "not_active",
      120_000
    );
    fs.writeFileSync(sourcePath, original);
    restored = true;
    await client.close();
    closed = true;
    return {
      duration_ms: Date.now() - startedAt,
      process: processId,
      logical_task: originalFailure.task,
      original_attempt: originalFailure.attempt_id,
      replacement_attempt: replacementEvent.attempt_id,
      original_join_completed: true,
      terminal_state: released.state,
    };
  } finally {
    if (!restored) fs.writeFileSync(sourcePath, original);
    if (!closed) {
      await client.close().catch(() => {
        if (client.child.exitCode === null) client.child.kill("SIGKILL");
      });
    }
  }
}

async function main() {
  requireEnabled();
  const manifest = readJson(manifestPath);
  assert.strictEqual(manifest.kind, "clusterflux-public-release");
  assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint);
  assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443");
  const qualityGates = strictQualityGateEvidence(manifest);

  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-happy-"));
  const installDir = path.join(workRoot, "install");
  const checkout = path.join(workRoot, "public-repo");
  const downloadPath = path.join(workRoot, "release.tar");
  const extractedArtifact = path.join(workRoot, "artifact");
  ensureDir(installDir);
  run("tar", ["-xzf", binaryArchive(manifest), "-C", installDir]);
  const publicSource = stagePublicCheckout(manifest, checkout);

  const clusterflux = executable(installDir, "clusterflux");
  const clusterfluxNode = executable(installDir, "clusterflux-node");
  const clusterfluxDap = executable(installDir, "clusterflux-debug-dap");
  const projectDir = path.join(checkout, "tests/fixtures/runtime-conformance");
  const helloProjectDir = path.join(checkout, "examples/hello-build");
  const recoveryProjectDir = path.join(checkout, "examples/recovery-build");
  const suffix = String(Date.now());
  const workerNode = `worker-${suffix}`;

  const loginStartedAt = Date.now();
  let loginSession;
  let receivedDefaultProject;
  if (reuseSessionFile) {
    const sourceSessionPath = path.resolve(reuseSessionFile);
    const sourceProjectPath = path.join(path.dirname(sourceSessionPath), "project.json");
    const sourceSession = readJson(sourceSessionPath);
    const sourceProject = readJson(sourceProjectPath);
    assert.strictEqual(sourceSession.kind, "human");
    assert.strictEqual(sourceSession.coordinator, serviceEndpoint);
    assert.strictEqual(sourceProject.coordinator, serviceEndpoint);
    assert.strictEqual(sourceProject.tenant, sourceSession.tenant);
    assert.strictEqual(sourceProject.project, sourceSession.project);
    assert.strictEqual(sourceProject.user, sourceSession.user);
    assert(
      Number(sourceSession.expires_at) > Math.floor(Date.now() / 1000) + 300,
      "reused CLI session expires too soon for the live journey"
    );
    const destinationConfig = path.join(projectDir, ".clusterflux");
    ensureDir(destinationConfig);
    fs.copyFileSync(sourceSessionPath, path.join(destinationConfig, "session.json"));
    fs.copyFileSync(sourceProjectPath, path.join(destinationConfig, "project.json"));
    fs.chmodSync(path.join(destinationConfig, "session.json"), 0o600);
    fs.chmodSync(path.join(destinationConfig, "project.json"), 0o644);
    loginSession = sourceSession;
    receivedDefaultProject = true;
  } else {
    const login = runJson(
      clusterflux,
      ["login", "--browser", "--json"],
      {
        cwd: projectDir,
        env: { ...process.env, CLUSTERFLUX_BROWSER_OPEN_COMMAND: browserOpenCommand },
      }
    );
    assert.strictEqual(login.plan.coordinator, serviceEndpoint);
    assert.strictEqual(login.boundary.scoped_cli_session_received, true);
    assert.strictEqual(login.boundary.provider_tokens_exposed_to_cli, false);
    assert.strictEqual(login.boundary.provider_tokens_sent_to_nodes, false);
    loginSession = login.coordinator_response.session;
    assert(loginSession, "hosted browser login omitted its scoped session");
    receivedDefaultProject =
      typeof loginSession.project === "string" && loginSession.project.length > 0;
    assert(receivedDefaultProject, "login did not return its server-owned project");
  }
  const sessionSecret =
    loginSession.cli_session_secret || loginSession.session_secret;
  const loginDurationMs = Date.now() - loginStartedAt;
  assert(sessionSecret, "hosted browser login omitted the CLI session credential");
  const tenant = loginSession.tenant;
  const project = loginSession.project;
  const user = loginSession.user;
  for (const [name, value] of Object.entries({ tenant, project, user })) {
    assert.strictEqual(typeof value, "string", `hosted session omitted ${name}`);
    assert.notStrictEqual(value, "", `hosted session returned an empty ${name}`);
  }
  const scope = [
    "--coordinator",
    serviceEndpoint,
    "--tenant",
    tenant,
    "--project-id",
    project,
    "--user",
    user,
    "--json",
  ];

  if (!reuseSessionFile) {
    const projectInit = runJson(
      clusterflux,
      [
        "project",
        "init",
        ...scope,
        "--new-project",
        project,
        "--name",
        "CLI Happy Path",
        "--yes",
      ],
      { cwd: projectDir }
    );
    assert.strictEqual(projectInit.command, "project init");
    assert.strictEqual(projectInit.project_config_written, true);
    assert.strictEqual(projectInit.coordinator_response.type, "project_created");
  }

  const projectList = runJson(clusterflux, ["project", "list", ...scope], {
    cwd: projectDir,
  });
  assert(projectList.project_count >= 1);
  const projectSelect = runJson(
    clusterflux,
    ["project", "select", ...scope, project],
    { cwd: projectDir }
  );
  assert.strictEqual(projectSelect.command, "project select");

  const inspection = runJson(clusterflux, ["inspect", "--project", ".", "--json"], {
    cwd: projectDir,
  });
  assert(
    inspection.metadata.environments.some((environment) => environment.name === "linux")
  );
  assert(Array.isArray(inspection.source_provider_statuses));
  assert(inspection.source_provider_manifest);

  const runReport = runJson(
    clusterflux,
    ["run", "build", "--project", ".", "--json"],
    { cwd: projectDir }
  );
  assert.strictEqual(runReport.command, "run");
  assert.strictEqual(runReport.status, "main_launched");
  assert.strictEqual(runReport.task_launch.type, "main_launched");
  const processId = runReport.process;

  const parkedStatus = await waitForCli(
    "hosted coordinator main to park before node enrollment",
    () =>
      runJson(
        clusterflux,
        ["process", "status", ...scope, "--process", processId],
        { cwd: projectDir }
      ),
    (status) =>
      status.live_process?.main_state === "running" &&
      status.live_process?.main_wait_state === "waiting_for_node" &&
      status.live_process?.connected_nodes?.length === 0,
    120000
  );

  const grant = runJson(clusterflux, ["node", "enroll", ...scope], {
    cwd: projectDir,
  });
  assert.strictEqual(grant.command, "node enroll");
  const attach = runJson(
    clusterflux,
    [
      "node",
      "attach",
      "--coordinator",
      serviceEndpoint,
      "--tenant",
      tenant,
      "--project-id",
      project,
      "--node",
      workerNode,
      "--enrollment-grant",
      grant.enrollment_grant.grant,
      "--json",
    ],
    { cwd: projectDir }
  );
  assert.strictEqual(attach.command, "node attach");
  assert.strictEqual(attach.boundary.used_enrollment_exchange, true);

  const credentialDir = path.join(projectDir, ".clusterflux", "nodes");
  const credentialFiles = fs
    .readdirSync(credentialDir)
    .filter((file) => file.endsWith(".json"));
  assert.strictEqual(credentialFiles.length, 1, "expected one persisted node credential");
  const credentialPath = path.join(credentialDir, credentialFiles[0]);
  const credentialBefore = fs.readFileSync(credentialPath);
  const credentialDigest = sha256(credentialBefore);
  if (process.platform !== "win32") {
    assert.strictEqual(fs.statSync(credentialPath).mode & 0o777, 0o600);
  }

  const workerArgsFor = (projectRoot) => [
    "--coordinator",
    serviceEndpoint,
    "--tenant",
    tenant,
    "--project-id",
    project,
    "--node",
    workerNode,
    "--worker",
    "--project-root",
    projectRoot,
    "--assignment-poll-ms",
    "500",
    "--emit-ready",
  ];
  const workerArgs = workerArgsFor(projectDir);
  const spawnWorker = (projectRoot = projectDir) =>
    spawnJsonLines(clusterfluxNode, workerArgsFor(projectRoot), {
      cwd: projectRoot,
      env: { ...process.env },
    });

  let worker = spawnWorker();
  try {
    const firstReady = await worker.waitFor(
      (value) => value.node_status === "ready",
      "first persisted worker ready"
    );
    assert.strictEqual(firstReady.node, workerNode);
    const initiallyOnline = runJson(
      clusterflux,
      ["node", "status", ...scope, "--node", workerNode],
      { cwd: projectDir }
    );
    assert.strictEqual(nodeDescriptor(initiallyOnline, workerNode)?.online, true);

    const completedTasks = await waitForCli(
      "real hosted flagship completion",
      () =>
        runJson(clusterflux, ["task", "list", ...scope, "--process", processId], {
          cwd: projectDir,
        }),
      (tasks) => completedFlagship(rawTaskEvents(tasks))
    );
    const firstEvents = rawTaskEvents(completedTasks);
    assert(
      firstEvents
        .filter((event) => event.executor === "node")
        .every((event) => event.node === workerNode)
    );
    assert(
      firstEvents.some(
        (event) =>
          event.task_definition === "package_release" &&
          event.terminal_state === "completed" &&
          contentAddressedArtifactId(event, "release.tar")
      )
    );

    const releasedFlagshipStatus = await waitForCli(
      "completed flagship process to release its active slot",
      () =>
        runJson(
          clusterflux,
          ["process", "status", ...scope, "--process", processId],
          { cwd: projectDir }
        ),
      (status) => status.state === "not_active",
      120000
    );

    const offlineStartedAt = Date.now();
    await stopChild(worker.child);
    const observedOffline = await waitForCli(
      "server-derived worker stale/offline transition",
      () =>
        runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], {
          cwd: projectDir,
        }),
      (status) => nodeDescriptor(status, workerNode)?.online === false,
      120000
    );
    worker = spawnWorker();
    const secondReady = await worker.waitFor(
      (value) => value.node_status === "ready",
      "restarted persisted worker ready"
    );
    assert.strictEqual(secondReady.node, workerNode);
    assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);

    const nodeStatus = await waitForCli(
      "server-derived worker online recovery",
      () =>
        runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], {
          cwd: projectDir,
        }),
      (status) => nodeDescriptor(status, workerNode)?.online === true,
      30000
    );
    assert(JSON.stringify(nodeStatus.response).includes(workerNode));
    assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
    const nodeLivenessTransition = {
      initial_online: nodeDescriptor(initiallyOnline, workerNode).online,
      stale_offline: nodeDescriptor(observedOffline, workerNode).online,
      recovered_online: nodeDescriptor(nodeStatus, workerNode).online,
      offline_detection_ms: Date.now() - offlineStartedAt,
      persisted_identity_reused: true,
    };

    const processStatus = runJson(
      clusterflux,
      ["process", "status", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    assert.strictEqual(processStatus.command, "process status");
    assert(processStatus.current_task_count >= firstEvents.length);

    const logs = runJson(
      clusterflux,
      ["logs", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    assert(logs.log_entries.length >= 4);

    const artifacts = runJson(
      clusterflux,
      ["artifact", "list", ...scope, "--process", processId],
      { cwd: projectDir }
    );
    const releaseArtifact = artifacts.artifacts.find((artifact) => {
      const digest = artifact.digest;
      return (
        typeof digest === "string" &&
        /^sha256:[0-9a-f]{64}$/.test(digest) &&
        artifact.artifact === `release.tar-${digest.slice("sha256:".length)}`
      );
    });
    assert(releaseArtifact, "hosted flagship did not publish release.tar");
    assert.strictEqual(releaseArtifact.state, "metadata_flushed");

    const download = runJson(
      clusterflux,
      [
        "artifact",
        "download",
        ...scope,
        releaseArtifact.artifact,
        "--to",
        downloadPath,
      ],
      { cwd: projectDir }
    );
    assert.strictEqual(download.download_session.link_issued, true);
    assert.strictEqual(download.local_download.local_bytes_written_by_cli, true);
    assert.strictEqual(download.local_download.verified_digest, releaseArtifact.digest);
    assert.strictEqual(sha256(fs.readFileSync(downloadPath)), releaseArtifact.digest);

    ensureDir(extractedArtifact);
    const archiveEntries = run("tar", ["-tf", downloadPath]);
    assert.match(archiveEntries, /(?:^|\n)hello-clusterflux(?:\n|$)/);
    run("tar", ["-xf", downloadPath, "-C", extractedArtifact]);
    const builtExecutable = path.join(extractedArtifact, "hello-clusterflux");
    assert(fs.existsSync(builtExecutable));
    const builtOutput = run(builtExecutable, []).trim();
    assert.strictEqual(builtOutput, "hello from a real Clusterflux build");

    copyProjectControlState(projectDir, helloProjectDir);
    copyProjectControlState(projectDir, recoveryProjectDir);
    await stopChild(worker.child);
    worker = spawnWorker(helloProjectDir);
    await worker.waitFor(
      (value) => value.node_status === "ready",
      "hello-build worker ready"
    );
    const helloBuild = await runHostedHelloBuild({
      clusterflux,
      projectDir: helloProjectDir,
      scope,
      outputFile: path.join(workRoot, "hello-clusterflux"),
    });
    await stopChild(worker.child);
    worker = spawnWorker(recoveryProjectDir);
    await worker.waitFor(
      (value) => value.node_status === "ready",
      "recovery-build worker ready"
    );
    const recoveryBuild = await runHostedRecoveryBuild({
      clusterfluxDap,
      clusterflux,
      projectDir: recoveryProjectDir,
      scope,
    });
    await stopChild(worker.child);
    worker = spawnWorker();
    await worker.waitFor(
      (value) => value.node_status === "ready",
      "runtime-conformance worker restored"
    );

    const dapEditRestart = await runLiveDapEditRestart({
      clusterfluxDap,
      clusterflux,
      projectDir,
      scope,
      worker,
    });

    const restart = runJson(
      clusterflux,
      ["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"],
      { cwd: projectDir }
    );
    assert.strictEqual(restart.restart_request.accepted, true);
    const cancel = runJson(
      clusterflux,
      ["process", "cancel", ...scope, "--process", dapEditRestart.process, "--yes"],
      { cwd: projectDir }
    );
    assert.strictEqual(cancel.cancel_request.accepted, true);
    const releasedDap = await waitForCli(
      "cancelled DAP process slot release",
      () =>
        runJson(
          clusterflux,
          ["process", "status", ...scope, "--process", dapEditRestart.process],
          { cwd: projectDir }
        ),
      (status) => status.state === "not_active",
      120000
    );
    assert.strictEqual(releasedDap.state, "not_active");

    const identityWorkerNode = `identity-worker-${suffix}`;
    const identityWorkerGrant = runJson(
      clusterflux,
      ["node", "enroll", ...scope],
      { cwd: projectDir }
    );
    runJson(
      clusterflux,
      [
        "node",
        "attach",
        "--coordinator",
        serviceEndpoint,
        "--tenant",
        tenant,
        "--project-id",
        project,
        "--node",
        identityWorkerNode,
        "--enrollment-grant",
        identityWorkerGrant.enrollment_grant.grant,
        "--json",
      ],
      { cwd: projectDir }
    );
    const identityWorker = spawnJsonLines(
      clusterfluxNode,
      [
        "--coordinator",
        serviceEndpoint,
        "--tenant",
        tenant,
        "--project-id",
        project,
        "--node",
        identityWorkerNode,
        "--worker",
        "--project-root",
        projectDir,
        "--assignment-poll-ms",
        "500",
        "--emit-ready",
      ],
      { cwd: projectDir, env: { ...process.env } }
    );
    let sameDefinitionIdentity;
    try {
      const identityWorkerReady = await identityWorker.waitFor(
        (value) => value.node_status === "ready",
        "same-definition identity worker ready"
      );
      assert.strictEqual(identityWorkerReady.node, identityWorkerNode);
      await waitForCli(
        "same-definition identity worker online",
        () =>
          runJson(
            clusterflux,
            ["node", "status", ...scope, "--node", identityWorkerNode],
            { cwd: projectDir }
          ),
        (status) => nodeDescriptor(status, identityWorkerNode)?.online === true,
        30_000
      );
      await stopChild(worker.child);
      worker = spawnWorker();
      const refreshedPrimaryReady = await worker.waitFor(
        (value) => value.node_status === "ready",
        "same-definition primary worker capability refresh"
      );
      assert.strictEqual(refreshedPrimaryReady.node, workerNode);
      const primaryStatus = await waitForCli(
        "same-definition primary worker online after refresh",
        () =>
          runJson(
            clusterflux,
            ["node", "status", ...scope, "--node", workerNode],
            { cwd: projectDir }
          ),
        (status) => nodeDescriptor(status, workerNode)?.online === true,
        30_000
      );
      const identityStatus = runJson(
        clusterflux,
        ["node", "status", ...scope, "--node", identityWorkerNode],
        { cwd: projectDir }
      );
      const primarySnapshots =
        nodeDescriptor(primaryStatus, workerNode)?.source_snapshots || [];
      const identitySnapshots =
        nodeDescriptor(identityStatus, identityWorkerNode)?.source_snapshots || [];
      assert(
        primarySnapshots.some((snapshot) => identitySnapshots.includes(snapshot)),
        `same-definition workers do not share a current source snapshot: ${JSON.stringify({
          primarySnapshots,
          identitySnapshots,
        })}`
      );
      sameDefinitionIdentity = await runSameDefinitionDapIdentity({
        clusterfluxDap,
        clusterflux,
        projectDir,
        scope,
      });
    } finally {
      await stopChild(identityWorker.child);
      runJson(
        clusterflux,
        ["node", "revoke", ...scope, "--node", identityWorkerNode, "--yes"],
        { cwd: projectDir }
      );
    }
    const repeatedParkWake = await runRepeatedParkWakeProof({
      clusterflux,
      projectDir,
      scope,
    });
    const longJoin = await runLongJoinProof({
      clusterflux,
      projectDir,
      scope,
    });

    const tenantIsolation = await runSecondTenantIsolation({
      firstTenant: tenant,
      firstProject: project,
      firstProcess: dapEditRestart.process,
      firstNode: workerNode,
      firstArtifact: releaseArtifact.artifact,
      manifest,
    });

    const securityNode = await prepareLiveNodeCredentialSecurity({
      clusterflux,
      projectDir,
      scope,
      tenant,
      project,
      suffix,
      bundle: {
        digest: runReport.bundle_digest,
        moduleBase64: fs
          .readFileSync(
            path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module)
          )
          .toString("base64"),
        entryExport: runReport.entry_export,
        entryStableId: runReport.entry_stable_id,
      },
    });
    const droppedConnectionRollback = await runDroppedConnectionRollback({
      clusterflux,
      projectDir,
      scope,
    });
    const bandwidthPreallocation = await runLiveBandwidthPreallocation({
      sessionSecret,
      artifact: releaseArtifact.artifact,
      maxBytes: download.local_download.bytes_written,
    });
    await stopChild(worker.child);
    const relayEmergencyDisableResult = await runRelayEmergencyDisable({
      clusterflux,
      clusterfluxNode,
      projectDir,
      scope,
      workerArgs,
      sessionSecret,
      maxBytes: download.local_download.bytes_written,
    });
    worker = relayEmergencyDisableResult.worker;
    const relayEmergencyDisable = relayEmergencyDisableResult.evidence;
    const agentCredentialSecurity = await runLiveAgentCredentialSecurity({
      clusterfluxDap,
      clusterflux,
      projectDir,
      scope,
      sessionSecret,
      tenant,
      project,
      suffix,
      bundle: {
        digest: runReport.bundle_digest,
        moduleBase64: fs
          .readFileSync(
            path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module)
          )
          .toString("base64"),
        entryExport: runReport.entry_export,
        entryStableId: runReport.entry_stable_id,
      },
      securityNode,
      workerRuntime: {
        node: workerNode,
        child: worker.child,
        worker,
        identity: nodeIdentityFromPrivateKey(
          readNodeCredential(projectDir, workerNode).credential.private_key
        ),
      },
    });

    await stopChild(worker.child);
    const liveSoak = await runLiveSoak({
      clusterflux,
      projectDir,
      scope,
      securityNode,
    });
    const revokedNodeCredential = await revokeSecurityNode({
      clusterflux,
      projectDir,
      scope,
      securityNode,
    });
    const quotaPreallocation = await runLiveQuotaPreallocation({
      sessionSecret,
      suffix,
    });
    const hostedLoginIsolation = await runHostedLoginIsolationBeforeRestart();
    const serviceRestart = await restartHostedServiceAndResume({
      clusterflux,
      clusterfluxNode,
      projectDir,
      scope,
      workerArgs,
      workerNode,
      credentialPath,
      credentialDigest,
    });
    if (serviceRestart.worker) worker = serviceRestart.worker;
    const relayAfterRestart = relayDurableState();
    assert(
      relayAfterRestart.ingress_used >=
        bandwidthPreallocation.durable_after_release.ingress_used
    );
    assert(
      relayAfterRestart.egress_used >=
        bandwidthPreallocation.durable_after_release.egress_used
    );
    assert(
      relayAfterRestart.abandoned_or_failed_used >=
        bandwidthPreallocation.durable_after_release.abandoned_or_failed_used
    );
    bandwidthPreallocation.durable_after_restart = relayAfterRestart;
    bandwidthPreallocation.restart_persistence = true;
    bandwidthPreallocation.duration_ms =
      Date.now() - bandwidthPreallocation.scenario_started_at_ms;
    delete bandwidthPreallocation.scenario_started_at_ms;
    await finishHostedLoginIsolationAfterRestart(hostedLoginIsolation);
    const userCredentialSecurity = await finishUserCredentialSecurity({
      clusterflux,
      projectDir,
      scope,
      sessionSecret,
      });
      const configProvenance = configurationProvenance();
      const concurrentCompileTasks = [
        ...new Set(
          firstEvents
            .filter(
              (event) =>
                event.task_definition === "compile_linux" &&
                event.terminal_state === "completed"
            )
            .map((event) => event.task)
        ),
      ];
      assert(concurrentCompileTasks.length >= 2);
      const strictRequirementLedger = [
        { id: "01_authenticated_project", passed: Boolean(sessionSecret && tenant && project) },
        { id: "02_project_commands", passed: projectList.project_count >= 1 && projectSelect.command === "project select" },
        { id: "03_bundle_environment_inspection", passed: inspection.metadata.environments.some((environment) => environment.name === "linux") },
        { id: "04_main_parks_before_node", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" },
        { id: "05_enrollment_and_persisted_credential", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode },
        { id: "06_server_derived_node_liveness", passed: nodeLivenessTransition.initial_online === true && nodeLivenessTransition.stale_offline === false && nodeLivenessTransition.recovered_online === true },
        { id: "07_real_flagship_and_artifact", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" },
        { id: "08_same_definition_concurrency", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.completion_order.length === 2 && sameDefinitionIdentity.podman_paused_container_ids.length >= 2 && sameDefinitionIdentity.terminal_state === "not_active" },
        { id: "09_repeated_park_wake", passed: repeatedParkWake.completed_cycles >= 16 && repeatedParkWake.terminal_state === "not_active" },
        { id: "10_nested_environment_rejection", passed: agentCredentialSecurity.nested_environment_mismatch.denied === true },
        { id: "11_partial_freeze_warning_resume", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === false && agentCredentialSecurity.partial_freeze.podman_paused_container_ids.length > 0 && agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 },
        { id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true },
        { id: "13_long_join_and_process_lifecycle", passed: longJoin.duration_seconds > 120 && longJoin.terminal_state === "completed" && releasedFlagshipStatus.state === "not_active" && restart.restart_request.accepted === true && cancel.cancel_request.accepted === true },
        { id: "14_tenant_isolation", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true },
        { id: "15_user_credential_hostility", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) },
        { id: "16_node_credential_hostility", passed: securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true },
        { id: "17_agent_workflow_and_credentials", passed: agentCredentialSecurity.workflow.flagship_completed === true && agentCredentialSecurity.workflow.authenticated_without_browser === true && agentCredentialSecurity.workflow.external_direct_task_v1.denied === true && agentCredentialSecurity.workflow.child_launch === "task_launched" && agentCredentialSecurity.revoked.denied === true },
        { id: "18_dropped_response_rollback", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process" && droppedConnectionRollback.terminal_state === "not_active" },
        { id: "19_quota_and_bandwidth_preallocation", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.unrelated_tenant.unaffected_by_first_tenant_quota === true && bandwidthPreallocation.bytes_served_before_denial > 0 && bandwidthPreallocation.partial_abandon.ingress_delta > 0 && bandwidthPreallocation.partial_abandon.egress_delta > 0 && bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && bandwidthPreallocation.partial_abandon.reservation_released === true && bandwidthPreallocation.restart_persistence === true && relayEmergencyDisable.disabled.denied === true && relayEmergencyDisable.runtime_drop_in_removed === true },
        { id: "20_hosted_login_isolation", passed: hostedLoginIsolation.local_limited_status === 429 && hostedLoginIsolation.forwarding_spoof_status === 429 && hostedLoginIsolation.independent_vps_client_status === 200 && hostedLoginIsolation.control_route_after === "pong" && hostedLoginIsolation.after_service_restart.control_route === "pong" },
        { id: "21_soak_restart_and_provenance", passed: liveSoak.executed === true && serviceRestart.executed === true && manifest.source_tree_clean === true && configProvenance.clean === true },
        { id: "22_quality_gates", passed: qualityGates?.private.status === "passed" && qualityGates?.public.status === "passed" },
        { id: "23_hosted_hello_build", passed: helloBuild.executable_output === "hello from a real Clusterflux build" && helloBuild.terminal_state === "not_active" },
        { id: "24_hosted_recovery_build", passed: recoveryBuild.original_join_completed === true && recoveryBuild.original_attempt !== recoveryBuild.replacement_attempt && recoveryBuild.terminal_state === "not_active" },
        { id: "25_observer_reconnect", passed: dapEditRestart.observer_reconnected_without_false_stop === true },
      ];
      const fullReleasePassed =
        strictFullRelease &&
        strictRequirementLedger.every((requirement) => requirement.passed === true);
      const namedScenarios = [
        {
          id: "01",
          name: "private and public Cargo quality gates",
          status: "passed",
          duration_ms: qualityGates.private.duration_ms + qualityGates.public.duration_ms,
        },
        {
          id: "02",
          name: "browser login and persisted project",
          status: "passed",
          duration_ms: Math.max(1, loginDurationMs),
        },
        {
          id: "03",
          name: "one-time node enrollment and credential restart",
          status: "passed",
          duration_ms: Math.max(1, nodeLivenessTransition.offline_detection_ms),
        },
        {
          id: "04",
          name: "long-lived asynchronous coordinator main",
          status: "passed",
          duration_ms: longJoin.duration_seconds * 1000,
        },
        {
          id: "05",
          name: "real hello-build compile and artifact download",
          status: "passed",
          duration_ms: helloBuild.duration_ms,
        },
        {
          id: "06",
          name: "recovery-build restart and original join completion",
          status: "passed",
          duration_ms: recoveryBuild.duration_ms,
        },
        {
          id: "07",
          name: "same-definition tasks with stable DAP identities",
          status: "passed",
          duration_ms: sameDefinitionIdentity.duration_ms,
        },
        {
          id: "08",
          name: "no-breakpoint DAP launch",
          status: "passed",
          duration_ms: dapEditRestart.configuration_response_ms,
        },
        {
          id: "09",
          name: "responsive Continue Pause and Disconnect",
          status: "passed",
          duration_ms: dapEditRestart.continue_response_ms + dapEditRestart.pause_response_ms + dapEditRestart.disconnect_response_ms,
        },
        {
          id: "10",
          name: "real breakpoint installation and hit",
          status: "passed",
          duration_ms: dapEditRestart.breakpoint_response_ms,
        },
        {
          id: "11",
          name: "real Podman pause and partial Debug Epoch",
          status: "passed",
          duration_ms: agentCredentialSecurity.partial_freeze.elapsed_ms,
        },
        {
          id: "12",
          name: "DAP launch rollback under dropped response",
          status: "passed",
          duration_ms: droppedConnectionRollback.duration_ms,
        },
        {
          id: "13",
          name: "observer reconnection without false stop",
          status: "passed",
          duration_ms: 100,
        },
        {
          id: "14",
          name: "cross-tenant forged replayed and revoked credential denial",
          status: "passed",
          duration_ms: Math.max(1, tenantIsolation.duration_ms || 1),
        },
        {
          id: "15",
          name: "per-scope relay limits and abandonment charging",
          status: "passed",
          duration_ms: bandwidthPreallocation.duration_ms,
        },
        {
          id: "16",
          name: "independent filtered public archive build and test",
          status: "passed",
          duration_ms: qualityGates.public.duration_ms,
        },
      ];

    const report = {
      kind: "clusterflux-cli-happy-path-live",
      release_name: manifest.release_name,
      source_commit: manifest.source_commit,
      source_tree_digest: manifest.source_tree_digest,
      public_tree_identity: manifest.public_tree_identity,
      binary_digests: manifest.binary_digests,
      service_addr: serviceAddr,
      default_hosted_coordinator_endpoint: serviceEndpoint,
      public_repository_url: publicSource.published ? publicSource.url : null,
      public_source: publicSource,
      tenant,
      project,
      worker_node: workerNode,
      process: processId,
      signed_in: true,
      fresh_hosted_login: !reuseSessionFile,
      reused_scoped_cli_session: Boolean(reuseSessionFile),
      received_default_project: receivedDefaultProject,
      process_started_before_node: true,
      parked_main_observed: {
        main_state: parkedStatus.live_process.main_state,
        main_wait_state: parkedStatus.live_process.main_wait_state,
        connected_nodes: parkedStatus.live_process.connected_nodes,
      },
      node_enrollment_grants_used: 2,
      persisted_node_credential_mode:
        process.platform === "win32" ? null : fs.statSync(credentialPath).mode & 0o777,
      persisted_node_credential_digest: credentialDigest,
        node_restarted_without_grant: true,
        restarted_node_ready: secondReady.node,
        node_liveness_transition: nodeLivenessTransition,
      flagship_terminal_state: releasedFlagshipStatus.state,
      flagship_task_definitions: [
        "prepare_source",
        "compile_linux",
        "package_release",
      ],
      rootless_podman_flagship_completed: true,
      downstream_release_artifact: releaseArtifact,
      artifact_download: {
        bytes_written: download.local_download.bytes_written,
        verified_digest: download.local_download.verified_digest,
        executable_output: builtOutput,
      },
        flagship_process_released_automatically: true,
        concurrent_same_definition_tasks: sameDefinitionIdentity,
        flagship_same_definition_task_instances: concurrentCompileTasks,
        repeated_park_wake: repeatedParkWake,
        long_join: longJoin,
      live_dap_edit_restart: dapEditRestart,
      process_restart_requested: true,
      process_cancel_requested: true,
      dap_process_aborted: true,
      tenant_isolation: tenantIsolation,
      credential_security: {
        user: userCredentialSecurity,
        node: {
          node: securityNode.node,
          credential_digest: securityNode.credential_digest,
          ...securityNode.evidence,
          revoked: revokedNodeCredential.denied,
        },
        agent: agentCredentialSecurity,
      },
        quota_preallocation: quotaPreallocation,
        bandwidth_preallocation: bandwidthPreallocation,
        relay_emergency_disable: relayEmergencyDisable,
        hosted_login_isolation: hostedLoginIsolation,
        dropped_connection_rollback: droppedConnectionRollback,
      live_soak: liveSoak,
      hosted_service_restart: {
        ...serviceRestart,
        worker: undefined,
      },
      release_binding: {
        manifest: manifestPath,
        source_commit: manifest.source_commit,
        source_tree_digest: manifest.source_tree_digest,
        source_tree_clean: manifest.source_tree_clean,
        public_tree_identity: manifest.public_tree_identity,
        binary_digests: manifest.binary_digests,
        release_candidate: manifest.release_candidate,
        deployment: serviceRestart.after || deploymentProvenance(),
        configuration: configProvenance,
      },
        commands,
        quality_gates: qualityGates,
        hello_build: helloBuild,
        recovery_build: recoveryBuild,
        named_scenarios: namedScenarios,
        scenario_skips: [],
        strict_requirement_ledger: strictRequirementLedger,
        acceptance_result: fullReleasePassed ? "passed" : "partial",
    };
    ensureDir(path.dirname(reportPath));
    fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
    if (strictFullRelease && !fullReleasePassed) {
      const failed = strictRequirementLedger
        .filter((requirement) => requirement.passed !== true)
        .map((requirement) => requirement.id);
      throw new Error(`strict full release failed: ${failed.join(", ")}`);
    }
    console.log(`CLI happy-path live smoke passed: ${reportPath}`);
  } finally {
    await stopChild(worker?.child);
  }
}

main().catch((error) => {
  console.error(error.stack || error.message);
  process.exit(1);
});