clusterflux-public/scripts/recovery-build-smoke.js

#!/usr/bin/env node

const assert = require("assert");
const fs = require("fs");
const path = require("path");
const { DapClient } = require("./dap-client");
const { configurePodmanTestEnvironment } = require("./podman-test-env");

const repo = path.resolve(__dirname, "..");
const project = path.join(repo, "examples/recovery-build");
const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs"));
const original = fs.readFileSync(sourcePath, "utf8");
const failing = "\"exit 23\".to_owned()";
const replacement =
  "\"printf 'recovered\\n' > /clusterflux/output/recovering.txt\".to_owned()";
assert(original.includes(failing), "recovery source must contain the real failing command");
configurePodmanTestEnvironment(repo);

(async () => {
  const client = new DapClient({
    cwd: repo,
    env: {
      ...process.env,
      CLUSTERFLUX_DAP_TIMEOUT_MS:
        process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || "120000",
    },
  });
  let restored = false;
  try {
    const initialize = client.send("initialize", {
      adapterID: "clusterflux",
      linesStartAt1: true,
      columnsStartAt1: true,
    });
    await client.response(initialize, "initialize");

    const launch = client.send("launch", {
      entry: "build",
      project,
      runtimeBackend: "local-services",
    });
    await client.response(launch, "launch");
    await client.waitFor(
      (message) => message.type === "event" && message.event === "initialized"
    );

    const configured = client.send("configurationDone");
    await client.response(configured, "configurationDone");
    const failed = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "stopped" &&
        message.body.reason === "exception"
    );
    assert.strictEqual(failed.body.allThreadsStopped, false);

    const threadRequest = client.send("threads");
    const threads = (await client.response(threadRequest, "threads")).body.threads;
    const laneThreads = threads
      .filter((thread) => /build[_ ]lane/.test(thread.name))
      .sort((left, right) => left.id - right.id);
    assert.strictEqual(
      laneThreads.length,
      1,
      "only the failed recovering lane should remain a live DAP thread"
    );
    const recoveringThread = laneThreads[0];
    const views = JSON.parse(
      fs.readFileSync(path.join(project, ".clusterflux/views.json"), "utf8")
    );
    const nodeReport = JSON.parse(
      views.inspector.find((item) => item.label === "Node report").value
    );
    const laneSnapshots = nodeReport.task_snapshots.snapshots.filter(
      (snapshot) => snapshot.task_definition === "build_lane"
    );
    assert.strictEqual(laneSnapshots.length, 2);
    assert.notStrictEqual(laneSnapshots[0].task, laneSnapshots[1].task);
    assert(laneSnapshots.some((snapshot) => snapshot.state === "completed"));
    assert(
      laneSnapshots.some(
        (snapshot) => snapshot.state === "failed_awaiting_action"
      )
    );
    const startedIds = new Set(
      client.messages
        .filter(
          (message) =>
            message.type === "event" &&
            message.event === "thread" &&
            message.body.reason === "started"
        )
        .map((message) => message.body.threadId)
    );
    const laneStartedIds = [...startedIds]
      .filter((id) => id !== 1)
      .sort((left, right) => left - right)
      .slice(-2);
    assert.strictEqual(laneStartedIds.length, 2);
    assert.notStrictEqual(laneStartedIds[0], laneStartedIds[1]);
    assert(startedIds.has(recoveringThread.id));

    fs.writeFileSync(sourcePath, original.replace(failing, replacement));
    const restart = client.send("restartFrame", {
      threadId: recoveringThread.id,
    });
    await client.response(restart, "restartFrame");
    const terminated = await client.waitFor(
      (message) =>
        message.type === "event" &&
        message.event === "terminated"
    );
    assert(terminated);

    const exitedIds = new Set(
      client.messages
        .filter(
          (message) =>
            message.type === "event" &&
            message.event === "thread" &&
            message.body.reason === "exited"
        )
        .map((message) => message.body.threadId)
    );
    assert(exitedIds.has(laneStartedIds[0]));
    assert(exitedIds.has(recoveringThread.id));

    fs.writeFileSync(sourcePath, original);
    restored = true;
    await client.close();
    console.log("Recovery build DAP restart smoke passed");
  } finally {
    if (!restored) fs.writeFileSync(sourcePath, original);
    if (client.child.exitCode === null) client.child.kill("SIGKILL");
  }
})().catch((error) => {
  console.error(error.stack || error.message);
  process.exit(1);
});