clusterflux-public/scripts/real-flagship-harness.js

const assert = require("assert");
const cp = require("child_process");
const net = require("net");
const path = require("path");
const { coordinatorWireRequest } = require("./coordinator-wire");
const { configurePodmanTestEnvironment } = require("./podman-test-env");

const repo = path.resolve(__dirname, "..");
const project = path.join(repo, "examples/launch-build-demo");

function waitForJsonLine(child) {
  return new Promise((resolve, reject) => {
    let buffer = "";
    let stderr = "";
    child.stdout.on("data", (chunk) => {
      buffer += chunk.toString();
      const newline = buffer.indexOf("\n");
      if (newline < 0) return;
      try {
        resolve(JSON.parse(buffer.slice(0, newline).trim()));
      } catch (error) {
        reject(error);
      }
    });
    child.stderr?.on("data", (chunk) => {
      stderr += chunk.toString();
      if (stderr.length > 4096) stderr = stderr.slice(-4096);
    });
    child.once("exit", (code) => {
      const detail = stderr.trim();
      reject(new Error(
        `process exited before JSON line with code ${code}${detail ? `: ${detail}` : ""}`
      ));
    });
  });
}

function waitForNodeStatus(child, expectedStatus) {
  return new Promise((resolve, reject) => {
    let buffer = "";
    let stderr = "";
    const cleanup = () => {
      child.stdout.off("data", onData);
      child.stderr?.off("data", onStderr);
      child.off("exit", onExit);
    };
    const onData = (chunk) => {
      buffer += chunk.toString();
      while (buffer.includes("\n")) {
        const newline = buffer.indexOf("\n");
        const line = buffer.slice(0, newline).trim();
        buffer = buffer.slice(newline + 1);
        if (!line) continue;
        let event;
        try {
          event = JSON.parse(line);
        } catch (error) {
          cleanup();
          reject(error);
          return;
        }
        if (event.node_status === expectedStatus) {
          cleanup();
          resolve(event);
          return;
        }
      }
    };
    const onStderr = (chunk) => {
      stderr += chunk.toString();
      if (stderr.length > 4096) stderr = stderr.slice(-4096);
    };
    const onExit = (code) => {
      cleanup();
      const detail = stderr.trim();
      reject(
        new Error(
          `worker exited before node status ${expectedStatus} with code ${code}${
            detail ? `: ${detail}` : ""
          }`
        )
      );
    };
    child.stdout.on("data", onData);
    child.stderr?.on("data", onStderr);
    child.once("exit", onExit);
  });
}

function send(addr, message) {
  return new Promise((resolve, reject) => {
    const socket = net.connect(addr.port, addr.host, () => {
      socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`);
    });
    let buffer = "";
    socket.on("data", (chunk) => {
      buffer += chunk.toString();
      const newline = buffer.indexOf("\n");
      if (newline < 0) return;
      socket.end();
      try {
        resolve(JSON.parse(buffer.slice(0, newline)));
      } catch (error) {
        reject(error);
      }
    });
    socket.on("error", reject);
  });
}

function configureContainerPolicyHome() {
  configurePodmanTestEnvironment(repo);
}

function commandWithPodman(program, args) {
  if (cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status === 0) {
    return { program, args };
  }
  if (cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0) {
    return {
      program: "nix",
      args: ["shell", "nixpkgs#podman", "--command", program, ...args],
    };
  }
  throw new Error(
    "real flagship smoke requires rootless Podman (or Nix to provide it)"
  );
}

function ensureRootlessPodman() {
  configureContainerPolicyHome();
  const invocation = commandWithPodman("podman", [
    "info",
    "--format",
    "{{.Host.Security.Rootless}}",
  ]);
  const rootless = cp.execFileSync(invocation.program, invocation.args, {
    cwd: repo,
    env: process.env,
    encoding: "utf8",
  }).trim();
  assert.strictEqual(rootless, "true", "flagship worker must use rootless Podman");
}

async function runFlagshipWorker(addr, node, identity) {
  const enrollment = await send(addr, {
    type: "create_node_enrollment_grant",
    tenant: "tenant",
    project: "project",
    actor_user: "user",
    ttl_seconds: 60,
  });
  assert.strictEqual(enrollment.type, "node_enrollment_grant_created");
  const cargoArgs = [
    "run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node", "--",
    "--coordinator", `${addr.host}:${addr.port}`,
    "--tenant", "tenant",
    "--project-id", "project",
    "--node", node,
    "--enrollment-grant", enrollment.grant,
    "--worker",
    "--emit-ready",
    "--project-root", project,
    "--assignment-poll-ms", "25",
  ];
  const invocation = commandWithPodman("cargo", cargoArgs);
  const child = cp.spawn(invocation.program, invocation.args, {
    cwd: repo,
    env: {
      ...process.env,
      CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey,
    },
  });
  return { child, ready: waitForJsonLine(child) };
}

const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));

async function waitForTaskEvent(addr, process, predicate, description) {
  for (let attempt = 0; attempt < 2400; attempt += 1) {
    const response = await send(addr, {
      type: "list_task_events",
      tenant: "tenant",
      project: "project",
      actor_user: "user",
      process,
    });
    assert.strictEqual(response.type, "task_events", JSON.stringify(response));
    const event = response.events.find(predicate);
    if (event) return event;
    await delay(25);
  }
  throw new Error(`timed out waiting for real Wasm task ${description}`);
}

function startFlagship(addr) {
  const report = JSON.parse(
    cp.execFileSync(
      "cargo",
      [
        "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--",
        "run", "build",
        "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`,
        "--project", project,
        "--json",
      ],
      { cwd: repo, env: process.env, encoding: "utf8" }
    )
  );
  assert.strictEqual(report.command, "run");
  assert.strictEqual(report.status, "main_launched", JSON.stringify(report));
  assert.strictEqual(report.entry, "build");
  assert.strictEqual(report.task_launch.type, "main_launched");
  assert.strictEqual(report.task_launch.task_instance, report.task_instance);
  assert.strictEqual(report.task_launch.task_definition, report.task_definition);
  assert.strictEqual(report.worker_placement_requested, true);
  assert.match(report.bundle_digest, /^sha256:[0-9a-f]{64}$/);
  assert.match(report.entry_export, /^clusterflux_entry_v1_/);
  const virtualProcess = report.process;
  return { report, process: virtualProcess };
}

async function launchFlagship(addr) {
  const { report, process: virtualProcess } = startFlagship(addr);
  const compileEvent = await waitForTaskEvent(
    addr,
    virtualProcess,
    (event) => event.task_definition === "compile_linux",
    "compile_linux"
  );
  const packageEvent = await waitForTaskEvent(
    addr,
    virtualProcess,
    (event) => event.task_definition === "package_release",
    "package_release"
  );
  const buildEvent = await waitForTaskEvent(
    addr,
    virtualProcess,
    (event) => event.task === report.task_instance && event.executor === "coordinator_main",
    "coordinator build main"
  );
  for (const event of [compileEvent, packageEvent, buildEvent]) {
    assert.strictEqual(event.terminal_state, "completed", JSON.stringify(event));
  }
  return {
    report,
    process: virtualProcess,
    compileEvent,
    packageEvent,
    buildEvent,
  };
}

function flagshipNodeCapabilities() {
  return {
    os: "Linux",
    arch: process.arch,
    capabilities: [
      "Command",
      "Containers",
      "RootlessPodman",
      "SourceFilesystem",
      "VfsArtifacts",
    ],
    environment_backends: ["Container"],
    source_providers: ["filesystem"],
  };
}

module.exports = {
  ensureRootlessPodman,
  flagshipNodeCapabilities,
  launchFlagship,
  project,
  repo,
  runFlagshipWorker,
  send,
  startFlagship,
  waitForTaskEvent,
  waitForJsonLine,
  waitForNodeStatus,
};