70319cde15
Source commit: ff7f847f143dc67864bed68a86cf259114384bd5 Public tree identity: sha256:bb7dcd2ac78bdbad2f4eba0f49be649d446d67f96f4eb2796941c026412fc032
3985 lines
129 KiB
JavaScript
3985 lines
129 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const assert = require("assert");
|
|
const cp = require("child_process");
|
|
const crypto = require("crypto");
|
|
const fs = require("fs");
|
|
const https = require("https");
|
|
const os = require("os");
|
|
const path = require("path");
|
|
const { DapClient } = require("./dap-client");
|
|
const { agentIdentity, signedAgentWorkflowRequest } = require("./agent-signing");
|
|
const { coordinatorWireRequest } = require("./coordinator-wire");
|
|
const {
|
|
nodeIdentity,
|
|
nodeIdentityFromPrivateKey,
|
|
signedNodeHeartbeat,
|
|
signedNodeRequest,
|
|
} = require("./node-signing");
|
|
const { configurePodmanTestEnvironment } = require("./podman-test-env");
|
|
|
|
const repo = path.resolve(__dirname, "..");
|
|
configurePodmanTestEnvironment(repo);
|
|
if (
|
|
!process.env.CLUSTERFLUX_PODMAN_NIX_SHELL &&
|
|
cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 &&
|
|
cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0
|
|
) {
|
|
cp.execFileSync(
|
|
"nix",
|
|
["shell", "nixpkgs#podman", "--command", "node", __filename],
|
|
{
|
|
cwd: repo,
|
|
env: { ...process.env, CLUSTERFLUX_PODMAN_NIX_SHELL: "1" },
|
|
stdio: "inherit",
|
|
}
|
|
);
|
|
process.exit(0);
|
|
}
|
|
|
|
const releaseRoot = path.resolve(
|
|
process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR ||
|
|
path.join(repo, "target/public-release")
|
|
);
|
|
const acceptanceRoot = path.join(repo, "target/acceptance");
|
|
const manifestPath = path.join(releaseRoot, "public-release-manifest.json");
|
|
const reportPath = path.join(acceptanceRoot, "cli-happy-path-live.json");
|
|
const serviceEndpoint = "https://clusterflux.michelpaulissen.com";
|
|
const serviceAddr =
|
|
process.env.CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR ||
|
|
"clusterflux.michelpaulissen.com:443";
|
|
const browserOpenCommand =
|
|
process.env.CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND;
|
|
const reuseSessionFile =
|
|
process.env.CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE;
|
|
const enabled = process.env.CLUSTERFLUX_CLI_HAPPY_PATH_LIVE === "1";
|
|
const strictFullRelease = process.env.CLUSTERFLUX_STRICT_FULL_RELEASE === "1";
|
|
const strictVpsRestart = process.env.CLUSTERFLUX_STRICT_VPS_RESTART === "1";
|
|
const strictVpsHost = process.env.CLUSTERFLUX_STRICT_VPS_HOST;
|
|
const strictVpsIdentity = process.env.CLUSTERFLUX_STRICT_VPS_IDENTITY;
|
|
const strictServiceUnit =
|
|
process.env.CLUSTERFLUX_STRICT_SERVICE_UNIT ||
|
|
"clusterflux-hosted.service";
|
|
const strictSoakSeconds = Number(
|
|
process.env.CLUSTERFLUX_STRICT_SOAK_SECONDS || "300"
|
|
);
|
|
const commands = [];
|
|
|
|
function requireEnabled() {
|
|
if (!enabled) {
|
|
throw new Error(
|
|
"CLUSTERFLUX_CLI_HAPPY_PATH_LIVE=1 is required because this runs the live CLI happy path"
|
|
);
|
|
}
|
|
if (!browserOpenCommand && !reuseSessionFile) {
|
|
throw new Error(
|
|
"CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND or CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE is required"
|
|
);
|
|
}
|
|
if (
|
|
strictFullRelease &&
|
|
!process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE
|
|
) {
|
|
throw new Error(
|
|
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE for live isolation and quota independence"
|
|
);
|
|
}
|
|
if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) {
|
|
throw new Error(
|
|
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE"
|
|
);
|
|
}
|
|
if (strictFullRelease && !strictVpsRestart) {
|
|
throw new Error(
|
|
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_STRICT_VPS_RESTART=1"
|
|
);
|
|
}
|
|
if (strictVpsRestart && (!strictVpsHost || !strictVpsIdentity)) {
|
|
throw new Error(
|
|
"strict VPS restart requires CLUSTERFLUX_STRICT_VPS_HOST and CLUSTERFLUX_STRICT_VPS_IDENTITY"
|
|
);
|
|
}
|
|
if (
|
|
!Number.isInteger(strictSoakSeconds) ||
|
|
strictSoakSeconds < 120 ||
|
|
strictSoakSeconds > 1800
|
|
) {
|
|
throw new Error("CLUSTERFLUX_STRICT_SOAK_SECONDS must be an integer from 120 to 1800");
|
|
}
|
|
}
|
|
|
|
function readJson(file) {
|
|
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
}
|
|
|
|
function ensureDir(dir) {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
function sha256(bytes) {
|
|
return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`;
|
|
}
|
|
|
|
function nonInteractiveGitEnv(extra = {}) {
|
|
return {
|
|
...process.env,
|
|
GIT_TERMINAL_PROMPT: "0",
|
|
GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false",
|
|
SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false",
|
|
GIT_SSH_COMMAND:
|
|
process.env.GIT_SSH_COMMAND ||
|
|
"ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0",
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function commandEnv(command, env) {
|
|
if (command !== "git") return env;
|
|
return nonInteractiveGitEnv(env);
|
|
}
|
|
|
|
function recordCommand(command, args) {
|
|
const redacted = [];
|
|
let hideNext = false;
|
|
for (const argument of args) {
|
|
if (hideNext) {
|
|
redacted.push("<redacted>");
|
|
hideNext = false;
|
|
} else {
|
|
redacted.push(argument);
|
|
hideNext = ["--enrollment-grant", "--admin-token"].includes(argument);
|
|
}
|
|
}
|
|
commands.push([command, ...redacted].join(" "));
|
|
}
|
|
|
|
function run(command, args, options = {}) {
|
|
recordCommand(command, args);
|
|
return cp.execFileSync(command, args, {
|
|
cwd: repo,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
timeout: 15 * 60 * 1000,
|
|
...options,
|
|
env: commandEnv(command, options.env),
|
|
});
|
|
}
|
|
|
|
function parseJsonOutput(output) {
|
|
const trimmed = output.trim();
|
|
if (!trimmed) throw new Error("expected JSON output, got empty stdout");
|
|
try {
|
|
return JSON.parse(trimmed);
|
|
} catch (_) {
|
|
// Commands may print progress before their final JSON report.
|
|
}
|
|
const lines = trimmed.split(/\r?\n/).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
try {
|
|
return JSON.parse(lines.slice(index).join("\n"));
|
|
} catch (_) {
|
|
// Keep scanning for the trailing JSON value.
|
|
}
|
|
}
|
|
throw new Error(`could not parse JSON output:\n${trimmed}`);
|
|
}
|
|
|
|
function runJson(command, args, options = {}) {
|
|
return parseJsonOutput(run(command, args, options));
|
|
}
|
|
|
|
function authenticatedRequest(sessionSecret, request) {
|
|
return {
|
|
type: "authenticated",
|
|
session_secret: sessionSecret,
|
|
request,
|
|
};
|
|
}
|
|
|
|
function sendHostedControl(payload) {
|
|
const body = Buffer.from(
|
|
JSON.stringify(coordinatorWireRequest(payload, "strict-live"))
|
|
);
|
|
const url = new URL("/api/v1/control", serviceEndpoint);
|
|
return new Promise((resolve, reject) => {
|
|
const request = https.request(
|
|
url,
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
"content-length": body.length,
|
|
},
|
|
timeout: 30_000,
|
|
},
|
|
(response) => {
|
|
const chunks = [];
|
|
response.on("data", (chunk) => chunks.push(chunk));
|
|
response.on("end", () => {
|
|
const responseBody = Buffer.concat(chunks).toString("utf8");
|
|
if (response.statusCode < 200 || response.statusCode >= 300) {
|
|
reject(
|
|
new Error(
|
|
`hosted control HTTP ${response.statusCode}: ${responseBody}`
|
|
)
|
|
);
|
|
return;
|
|
}
|
|
try {
|
|
resolve(JSON.parse(responseBody));
|
|
} catch (error) {
|
|
reject(
|
|
new Error(`hosted control returned invalid JSON: ${error.message}`)
|
|
);
|
|
}
|
|
});
|
|
}
|
|
);
|
|
request.on("timeout", () =>
|
|
request.destroy(new Error("hosted control request timed out"))
|
|
);
|
|
request.on("error", reject);
|
|
request.end(body);
|
|
});
|
|
}
|
|
|
|
function sendHostedControlDroppingResponse(payload) {
|
|
const body = Buffer.from(
|
|
JSON.stringify(coordinatorWireRequest(payload, "strict-live-dropped-response"))
|
|
);
|
|
const url = new URL("/api/v1/control", serviceEndpoint);
|
|
return new Promise((resolve, reject) => {
|
|
let settled = false;
|
|
const request = https.request(
|
|
url,
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
"content-length": body.length,
|
|
},
|
|
timeout: 30_000,
|
|
},
|
|
(response) => {
|
|
settled = true;
|
|
const statusCode = response.statusCode;
|
|
response.destroy();
|
|
resolve({ status_code: statusCode, response_body_consumed: false });
|
|
}
|
|
);
|
|
request.on("timeout", () =>
|
|
request.destroy(new Error("hosted dropped-response request timed out"))
|
|
);
|
|
request.on("error", (error) => {
|
|
if (!settled) reject(error);
|
|
});
|
|
request.end(body);
|
|
});
|
|
}
|
|
|
|
function sendHostedLoginStatus(extraHeaders = {}) {
|
|
const body = Buffer.from(
|
|
JSON.stringify(
|
|
coordinatorWireRequest(
|
|
{ type: "begin_oidc_browser_login" },
|
|
`strict-live-login-${Date.now()}-${Math.random()}`
|
|
)
|
|
)
|
|
);
|
|
const url = new URL("/api/v1/login", serviceEndpoint);
|
|
return new Promise((resolve, reject) => {
|
|
const request = https.request(
|
|
url,
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
"content-length": body.length,
|
|
...extraHeaders,
|
|
},
|
|
timeout: 30_000,
|
|
},
|
|
(response) => {
|
|
const chunks = [];
|
|
response.on("data", (chunk) => chunks.push(chunk));
|
|
response.on("end", () =>
|
|
resolve({
|
|
status_code: response.statusCode,
|
|
body: Buffer.concat(chunks).toString("utf8"),
|
|
})
|
|
);
|
|
}
|
|
);
|
|
request.on("timeout", () =>
|
|
request.destroy(new Error("hosted login request timed out"))
|
|
);
|
|
request.on("error", reject);
|
|
request.end(body);
|
|
});
|
|
}
|
|
|
|
async function runHostedLoginIsolationBeforeRestart() {
|
|
const scenarioStartedAt = Date.now();
|
|
const controlBefore = await sendHostedControl({ type: "ping" });
|
|
assert.strictEqual(controlBefore.type, "pong");
|
|
const statuses = [];
|
|
let limited;
|
|
for (let index = 0; index < 128; index += 1) {
|
|
const response = await sendHostedLoginStatus();
|
|
statuses.push(response.status_code);
|
|
if (response.status_code === 429) {
|
|
limited = response;
|
|
break;
|
|
}
|
|
assert.strictEqual(response.status_code, 200, response.body);
|
|
}
|
|
assert(limited, `hosted login route was not rate limited: ${statuses.join(",")}`);
|
|
const spoofed = await sendHostedLoginStatus({
|
|
forwarded: "for=203.0.113.99",
|
|
"x-forwarded-for": "203.0.113.99",
|
|
"x-real-ip": "203.0.113.99",
|
|
});
|
|
assert.strictEqual(
|
|
spoofed.status_code,
|
|
429,
|
|
"caller-supplied forwarding headers bypassed the client rate limit"
|
|
);
|
|
|
|
const remoteBody = JSON.stringify(
|
|
coordinatorWireRequest(
|
|
{ type: "begin_oidc_browser_login" },
|
|
`strict-vps-login-${Date.now()}`
|
|
)
|
|
);
|
|
const remoteStatus = Number(
|
|
run(
|
|
"ssh",
|
|
sshArgs(
|
|
"curl",
|
|
"--silent",
|
|
"--show-error",
|
|
"--output",
|
|
"/dev/null",
|
|
"--write-out=%{http_code}",
|
|
"-HContent-Type:application/json",
|
|
`--data-binary=${remoteBody}`,
|
|
`${serviceEndpoint}/api/v1/login`
|
|
)
|
|
).trim()
|
|
);
|
|
assert.strictEqual(
|
|
remoteStatus,
|
|
200,
|
|
"one client exhausted the browser-login allowance for a different client"
|
|
);
|
|
const controlAfter = await sendHostedControl({ type: "ping" });
|
|
assert.strictEqual(controlAfter.type, "pong");
|
|
return {
|
|
scenario_started_at_ms: scenarioStartedAt,
|
|
local_statuses: statuses,
|
|
local_limited_status: limited.status_code,
|
|
forwarding_spoof_status: spoofed.status_code,
|
|
independent_vps_client_status: remoteStatus,
|
|
control_route_before: controlBefore.type,
|
|
control_route_after: controlAfter.type,
|
|
};
|
|
}
|
|
|
|
async function finishHostedLoginIsolationAfterRestart(evidence) {
|
|
const control = await sendHostedControl({ type: "ping" });
|
|
assert.strictEqual(control.type, "pong");
|
|
const response = await sendHostedLoginStatus();
|
|
assert(
|
|
[200, 429].includes(response.status_code),
|
|
`login route did not recover after service restart: ${response.status_code} ${response.body}`
|
|
);
|
|
evidence.after_service_restart = {
|
|
control_route: control.type,
|
|
login_route_status: response.status_code,
|
|
};
|
|
evidence.duration_ms = Date.now() - evidence.scenario_started_at_ms;
|
|
delete evidence.scenario_started_at_ms;
|
|
return evidence;
|
|
}
|
|
|
|
function assertDenied(response, pattern, label) {
|
|
assert.strictEqual(response.type, "error", `${label}: ${JSON.stringify(response)}`);
|
|
assert.match(response.message, pattern, `${label}: ${JSON.stringify(response)}`);
|
|
return {
|
|
denied: true,
|
|
category: response.error?.category || null,
|
|
message: response.message,
|
|
};
|
|
}
|
|
|
|
function readNodeCredential(projectDir, node) {
|
|
const directory = path.join(projectDir, ".clusterflux", "nodes");
|
|
for (const file of fs.readdirSync(directory)) {
|
|
if (!file.endsWith(".json")) continue;
|
|
const absolute = path.join(directory, file);
|
|
const credential = readJson(absolute);
|
|
if (credential.node === node) return { absolute, credential };
|
|
}
|
|
throw new Error(`persisted credential for ${node} was not found`);
|
|
}
|
|
|
|
function directoryStats(root) {
|
|
if (!fs.existsSync(root)) return { files: 0, bytes: 0 };
|
|
let files = 0;
|
|
let bytes = 0;
|
|
const visit = (directory) => {
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
const absolute = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) visit(absolute);
|
|
else if (entry.isFile()) {
|
|
files += 1;
|
|
bytes += fs.statSync(absolute).size;
|
|
}
|
|
}
|
|
};
|
|
visit(root);
|
|
return { files, bytes };
|
|
}
|
|
|
|
function shellQuote(argument) {
|
|
return `'${String(argument).replaceAll("'", "'\"'\"'")}'`;
|
|
}
|
|
|
|
function sshArgs(...remoteArgs) {
|
|
assert(strictVpsHost && strictVpsIdentity);
|
|
return [
|
|
"-i",
|
|
path.resolve(strictVpsIdentity.replace(/^~(?=\/)/, os.homedir())),
|
|
"-o",
|
|
"BatchMode=yes",
|
|
strictVpsHost,
|
|
remoteArgs.map(shellQuote).join(" "),
|
|
];
|
|
}
|
|
|
|
function strictInfrastructureSample(projectDir) {
|
|
const memory = Object.fromEntries(
|
|
run(
|
|
"ssh",
|
|
sshArgs(
|
|
"systemctl",
|
|
"show",
|
|
strictServiceUnit,
|
|
"--property=MemoryCurrent",
|
|
"--property=MemoryPeak"
|
|
)
|
|
)
|
|
.trim()
|
|
.split(/\r?\n/)
|
|
.map((line) => line.split("="))
|
|
);
|
|
const serviceDisk = Number(
|
|
run(
|
|
"ssh",
|
|
sshArgs("du", "-sb", "/var/lib/clusterflux-public-release")
|
|
)
|
|
.trim()
|
|
.split(/\s+/)[0]
|
|
);
|
|
const database = run(
|
|
"ssh",
|
|
sshArgs(
|
|
"runuser",
|
|
"-u",
|
|
"postgres",
|
|
"--",
|
|
"psql",
|
|
"-d",
|
|
"clusterflux",
|
|
"-AtF,"
|
|
),
|
|
{
|
|
input:
|
|
"SELECT pg_database_size('clusterflux'), (SELECT count(*) FROM clusterflux_tenants), (SELECT count(*) FROM clusterflux_projects), (SELECT count(*) FROM clusterflux_node_identities), (SELECT count(*) FROM clusterflux_cli_sessions);\n",
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
}
|
|
)
|
|
.trim()
|
|
.split(",")
|
|
.map(Number);
|
|
assert(Number.isFinite(Number(memory.MemoryCurrent)));
|
|
assert(Number.isFinite(Number(memory.MemoryPeak)));
|
|
assert.strictEqual(database.length, 5);
|
|
return {
|
|
sampled_at_epoch_ms: Date.now(),
|
|
service_memory_current_bytes: Number(memory.MemoryCurrent),
|
|
service_memory_peak_bytes: Number(memory.MemoryPeak),
|
|
service_state_disk_bytes: serviceDisk,
|
|
postgres_database_bytes: database[0],
|
|
durable_rows: {
|
|
tenants: database[1],
|
|
projects: database[2],
|
|
node_identities: database[3],
|
|
cli_sessions: database[4],
|
|
},
|
|
local_node_state: directoryStats(path.join(projectDir, ".clusterflux")),
|
|
};
|
|
}
|
|
|
|
function executable(root, name) {
|
|
return path.join(root, "bin", process.platform === "win32" ? `${name}.exe` : name);
|
|
}
|
|
|
|
function binaryArchive(manifest) {
|
|
const platform = `${os.platform()}-${os.arch()}`;
|
|
const asset = manifest.assets.find(
|
|
(candidate) =>
|
|
candidate.name.startsWith("clusterflux-public-binaries-") &&
|
|
candidate.name.includes(platform)
|
|
);
|
|
assert(asset, `missing released binary archive for ${platform}`);
|
|
assert(fs.existsSync(asset.file), `missing released binary archive ${asset.file}`);
|
|
return asset.file;
|
|
}
|
|
|
|
function publicRepoUrl(manifest) {
|
|
const url =
|
|
process.env.CLUSTERFLUX_PUBLIC_REPO_URL ||
|
|
process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE ||
|
|
manifest.public_repo_url ||
|
|
manifest.public_repo_remote;
|
|
assert(url, "public repository URL is required");
|
|
assert(url.includes("git.michelpaulissen.com"));
|
|
return url;
|
|
}
|
|
|
|
function stagePublicCheckout(manifest, checkout) {
|
|
const candidateTree = process.env.CLUSTERFLUX_LIVE_PUBLIC_TREE;
|
|
if (candidateTree) {
|
|
const source = path.resolve(candidateTree);
|
|
assert.strictEqual(
|
|
source,
|
|
path.resolve(manifest.public_tree),
|
|
"explicit live candidate tree must be the tree recorded by the release manifest"
|
|
);
|
|
fs.cpSync(source, checkout, { recursive: true });
|
|
return {
|
|
kind: "local_filtered_release_candidate",
|
|
path: source,
|
|
published: false,
|
|
};
|
|
}
|
|
run("git", ["clone", "--depth", "1", publicRepoUrl(manifest), checkout]);
|
|
return {
|
|
kind: "published_forgejo_checkout",
|
|
url: publicRepoUrl(manifest),
|
|
published: true,
|
|
};
|
|
}
|
|
|
|
function delay(milliseconds) {
|
|
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|
|
|
|
async function waitForCli(label, action, predicate, timeoutMs = 10 * 60 * 1000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastValue;
|
|
let lastError;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
lastValue = action();
|
|
if (predicate(lastValue)) return lastValue;
|
|
lastError = undefined;
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
await delay(250);
|
|
}
|
|
throw new Error(
|
|
`${label} timed out; last value=${JSON.stringify(lastValue)}${
|
|
lastError ? `; last error=${lastError.message}` : ""
|
|
}`
|
|
);
|
|
}
|
|
|
|
function spawnJsonLines(command, args, options) {
|
|
recordCommand(command, args);
|
|
const child = cp.spawn(command, args, {
|
|
...options,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
const values = [];
|
|
const waiters = [];
|
|
|
|
function deliver(value) {
|
|
values.push(value);
|
|
for (let index = waiters.length - 1; index >= 0; index -= 1) {
|
|
const waiter = waiters[index];
|
|
if (waiter.predicate(value)) {
|
|
waiters.splice(index, 1);
|
|
clearTimeout(waiter.timer);
|
|
waiter.resolve(value);
|
|
}
|
|
}
|
|
}
|
|
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk.toString();
|
|
while (stdout.includes("\n")) {
|
|
const newline = stdout.indexOf("\n");
|
|
const line = stdout.slice(0, newline).trim();
|
|
stdout = stdout.slice(newline + 1);
|
|
if (!line) continue;
|
|
try {
|
|
deliver(JSON.parse(line));
|
|
} catch (_) {
|
|
// Non-JSON progress stays out of the evidence report.
|
|
}
|
|
}
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk.toString();
|
|
});
|
|
|
|
function waitFor(predicate, label, timeoutMs = 120000) {
|
|
const existing = values.find(predicate);
|
|
if (existing) return Promise.resolve(existing);
|
|
return new Promise((resolve, reject) => {
|
|
const waiter = { predicate, resolve, reject, timer: undefined };
|
|
waiter.timer = setTimeout(() => {
|
|
const index = waiters.indexOf(waiter);
|
|
if (index >= 0) waiters.splice(index, 1);
|
|
reject(new Error(`${label} timed out\n${stderr}`));
|
|
}, timeoutMs);
|
|
waiters.push(waiter);
|
|
});
|
|
}
|
|
|
|
child.once("exit", (code) => {
|
|
for (const waiter of waiters.splice(0)) {
|
|
clearTimeout(waiter.timer);
|
|
waiter.reject(new Error(`${waiter.label || "child"} exited with ${code}\n${stderr}`));
|
|
}
|
|
});
|
|
return { child, values, waitFor, stderr: () => stderr };
|
|
}
|
|
|
|
async function stopChild(child) {
|
|
if (!child || child.exitCode !== null) return;
|
|
child.kill("SIGTERM");
|
|
const exited = new Promise((resolve) => child.once("exit", resolve));
|
|
const forced = delay(5000).then(() => {
|
|
if (child.exitCode === null) child.kill("SIGKILL");
|
|
});
|
|
await Promise.race([exited, forced]);
|
|
if (child.exitCode === null) await exited;
|
|
}
|
|
|
|
function rawTaskEvents(report) {
|
|
return report?.events?.response?.events || [];
|
|
}
|
|
|
|
function nodeDescriptor(status, node) {
|
|
return status?.response?.descriptors?.find(
|
|
(descriptor) => descriptor.id === node || descriptor.node === node
|
|
);
|
|
}
|
|
|
|
function completedFlagship(events) {
|
|
const completedDefinitions = new Set(
|
|
events
|
|
.filter((event) => event.terminal_state === "completed")
|
|
.map((event) => event.task_definition)
|
|
);
|
|
return (
|
|
completedDefinitions.has("prepare_source") &&
|
|
completedDefinitions.has("compile_linux") &&
|
|
completedDefinitions.has("package_release") &&
|
|
events.some(
|
|
(event) =>
|
|
event.executor === "coordinator_main" && event.terminal_state === "completed"
|
|
)
|
|
);
|
|
}
|
|
|
|
function contentAddressedArtifactId(event, logicalName) {
|
|
const digest = event?.artifact_digest;
|
|
if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) {
|
|
return null;
|
|
}
|
|
const artifactId = `${logicalName}-${digest.slice("sha256:".length)}`;
|
|
return event.artifact_path?.endsWith(`/vfs/artifacts/${artifactId}`)
|
|
? artifactId
|
|
: null;
|
|
}
|
|
|
|
async function prepareLiveNodeCredentialSecurity({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
tenant,
|
|
project,
|
|
suffix,
|
|
}) {
|
|
const node = `security-node-${suffix}`;
|
|
const grant = runJson(clusterflux, ["node", "enroll", ...scope], {
|
|
cwd: projectDir,
|
|
});
|
|
const attach = runJson(
|
|
clusterflux,
|
|
[
|
|
"node",
|
|
"attach",
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--node",
|
|
node,
|
|
"--enrollment-grant",
|
|
grant.enrollment_grant.grant,
|
|
"--json",
|
|
],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(attach.boundary.used_enrollment_exchange, true);
|
|
const stored = readNodeCredential(projectDir, node);
|
|
const identity = nodeIdentityFromPrivateKey(stored.credential.private_key);
|
|
assert.strictEqual(identity.publicKey, stored.credential.public_key);
|
|
|
|
const heartbeat = {
|
|
type: "node_heartbeat",
|
|
node,
|
|
node_signature: signedNodeHeartbeat(node, identity, {
|
|
nonce: `strict-valid-${suffix}`,
|
|
}),
|
|
};
|
|
const valid = await sendHostedControl(heartbeat);
|
|
assert.strictEqual(valid.type, "node_heartbeat");
|
|
const replay = assertDenied(
|
|
await sendHostedControl(heartbeat),
|
|
/nonce.*already.*used|replay/i,
|
|
"replayed node credential"
|
|
);
|
|
|
|
const expired = assertDenied(
|
|
await sendHostedControl({
|
|
type: "node_heartbeat",
|
|
node,
|
|
node_signature: signedNodeHeartbeat(node, identity, {
|
|
nonce: `strict-expired-${suffix}`,
|
|
issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
|
|
}),
|
|
}),
|
|
/expired|clock skew/i,
|
|
"expired node credential"
|
|
);
|
|
|
|
const forgedIdentity = nodeIdentity("strict-forged-node", node);
|
|
const forged = assertDenied(
|
|
await sendHostedControl({
|
|
type: "node_heartbeat",
|
|
node,
|
|
node_signature: signedNodeHeartbeat(node, forgedIdentity, {
|
|
nonce: `strict-forged-${suffix}`,
|
|
}),
|
|
}),
|
|
/signature|public key/i,
|
|
"forged node credential"
|
|
);
|
|
|
|
const originalCapabilityBody = {
|
|
type: "report_node_capabilities",
|
|
tenant,
|
|
project,
|
|
node,
|
|
capabilities: {
|
|
os: "Linux",
|
|
arch: process.arch === "x64" ? "x86_64" : process.arch,
|
|
capabilities: [],
|
|
environment_backends: [],
|
|
source_providers: [],
|
|
},
|
|
cached_environment_digests: [],
|
|
dependency_cache_digests: [],
|
|
source_snapshots: [],
|
|
artifact_locations: [],
|
|
direct_connectivity: false,
|
|
online: true,
|
|
};
|
|
const modifiedEnvelope = signedNodeRequest(
|
|
node,
|
|
identity,
|
|
"report_node_capabilities",
|
|
originalCapabilityBody,
|
|
{ nonce: `strict-modified-${suffix}` }
|
|
);
|
|
modifiedEnvelope.request.online = false;
|
|
const bodyModified = assertDenied(
|
|
await sendHostedControl(modifiedEnvelope),
|
|
/signature/i,
|
|
"body-modified node credential"
|
|
);
|
|
|
|
return {
|
|
node,
|
|
identity,
|
|
credential_path: stored.absolute,
|
|
credential_digest: sha256(fs.readFileSync(stored.absolute)),
|
|
capability_body: originalCapabilityBody,
|
|
evidence: {
|
|
valid_request: valid.type,
|
|
replay,
|
|
expired,
|
|
forged,
|
|
body_modified: bodyModified,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runLiveAgentCredentialSecurity({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
sessionSecret,
|
|
tenant,
|
|
project,
|
|
suffix,
|
|
bundle,
|
|
securityNode,
|
|
workerRuntime,
|
|
}) {
|
|
const agent = `security-agent-${suffix}`;
|
|
const identity = agentIdentity("strict-live-agent", agent);
|
|
const added = runJson(
|
|
clusterflux,
|
|
["key", "add", ...scope, "--agent", agent, "--public-key", identity.publicKey],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(added.command, "key add");
|
|
|
|
const processId = `vp-agent-security-${suffix}`;
|
|
const unsigned = {
|
|
type: "start_process",
|
|
tenant,
|
|
project,
|
|
actor_agent: agent,
|
|
process: processId,
|
|
restart: false,
|
|
};
|
|
const validRequest = signedAgentWorkflowRequest(identity, unsigned, {
|
|
nonce: `strict-agent-valid-${suffix}`,
|
|
});
|
|
const valid = await sendHostedControl(validRequest);
|
|
assert.strictEqual(valid.type, "process_started", JSON.stringify(valid));
|
|
const replay = assertDenied(
|
|
await sendHostedControl(validRequest),
|
|
/nonce.*already.*used|replay/i,
|
|
"replayed agent credential"
|
|
);
|
|
|
|
const expired = assertDenied(
|
|
await sendHostedControl(
|
|
signedAgentWorkflowRequest(
|
|
identity,
|
|
{ ...unsigned, process: `${processId}-expired` },
|
|
{
|
|
nonce: `strict-agent-expired-${suffix}`,
|
|
issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
|
|
}
|
|
)
|
|
),
|
|
/expired|clock skew/i,
|
|
"expired agent credential"
|
|
);
|
|
|
|
const modifiedRequest = signedAgentWorkflowRequest(
|
|
identity,
|
|
{ ...unsigned, process: `${processId}-modified` },
|
|
{ nonce: `strict-agent-modified-${suffix}` }
|
|
);
|
|
modifiedRequest.restart = true;
|
|
const bodyModified = assertDenied(
|
|
await sendHostedControl(modifiedRequest),
|
|
/signature/i,
|
|
"body-modified agent credential"
|
|
);
|
|
|
|
const forgedIdentity = agentIdentity("strict-live-forged-agent", agent);
|
|
const forged = assertDenied(
|
|
await sendHostedControl(
|
|
signedAgentWorkflowRequest(
|
|
forgedIdentity,
|
|
{ ...unsigned, process: `${processId}-forged` },
|
|
{
|
|
nonce: `strict-agent-forged-${suffix}`,
|
|
publicKeyFingerprint: identity.publicKeyFingerprint,
|
|
}
|
|
)
|
|
),
|
|
/signature|public key/i,
|
|
"forged agent credential"
|
|
);
|
|
|
|
const fakeEnvironmentDigest = sha256(
|
|
Buffer.from(`strict-debug-missing-participant-${suffix}`)
|
|
);
|
|
const fakeCapabilityBody = {
|
|
...securityNode.capability_body,
|
|
cached_environment_digests: [fakeEnvironmentDigest],
|
|
online: true,
|
|
};
|
|
const mainTask = `ti:${processId}:main`;
|
|
const fakeTask = `missing-debug-participant-${suffix}`;
|
|
const mainLaunchBody = {
|
|
type: "launch_task",
|
|
tenant,
|
|
project,
|
|
actor_agent: agent,
|
|
task_spec: {
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
task_definition: bundle.entryStableId,
|
|
task_instance: mainTask,
|
|
dispatch: {
|
|
kind: "coordinator_node_wasm",
|
|
export: bundle.entryExport,
|
|
abi: "entrypoint_v1",
|
|
},
|
|
environment_id: null,
|
|
environment: null,
|
|
environment_digest: null,
|
|
required_capabilities: [],
|
|
dependency_cache: null,
|
|
source_snapshot: null,
|
|
required_artifacts: [],
|
|
args: [],
|
|
vfs_epoch: valid.epoch,
|
|
bundle_digest: bundle.digest,
|
|
},
|
|
wait_for_node: true,
|
|
artifact_path: `/vfs/artifacts/${mainTask}-output.txt`,
|
|
wasm_module_base64: bundle.moduleBase64,
|
|
};
|
|
let mainLaunch;
|
|
let directTaskV1;
|
|
let fakeLaunch;
|
|
let liveParentTask;
|
|
let liveParentAssignment;
|
|
let realDebugLaunch;
|
|
const realDebugTask = `real-debug-participant-${suffix}`;
|
|
let workerPaused = false;
|
|
const baselineContainers = new Set(
|
|
run("podman", ["ps", "-q"])
|
|
.trim()
|
|
.split(/\s+/)
|
|
.filter(Boolean)
|
|
);
|
|
try {
|
|
mainLaunch = await sendHostedControl(
|
|
signedAgentWorkflowRequest(identity, mainLaunchBody, {
|
|
nonce: `strict-agent-main-${suffix}`,
|
|
})
|
|
);
|
|
assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch));
|
|
assert.strictEqual(mainLaunch.task_instance, mainTask);
|
|
assert.strictEqual(mainLaunch.state, "running");
|
|
assert.strictEqual(mainLaunch.actor.kind, "agent");
|
|
assert.strictEqual(mainLaunch.actor.authenticated_without_browser, true);
|
|
|
|
liveParentAssignment = await workerRuntime.worker.waitFor(
|
|
(value) => {
|
|
const taskSpec = value.task_assignment_response?.task_spec;
|
|
return (
|
|
value.node_status === "assignment_started" &&
|
|
value.node === workerRuntime.node &&
|
|
value.process === processId &&
|
|
taskSpec?.dispatch?.abi === "task_v1" &&
|
|
taskSpec.task_definition === "compile_linux" &&
|
|
taskSpec.environment_id === "linux" &&
|
|
typeof taskSpec.environment_digest === "string" &&
|
|
typeof taskSpec.source_snapshot === "string"
|
|
);
|
|
},
|
|
"agent main to dispatch an environment-bound live child to the real worker",
|
|
120000
|
|
);
|
|
liveParentTask = liveParentAssignment.virtual_thread;
|
|
assert.strictEqual(typeof liveParentTask, "string");
|
|
assert.notStrictEqual(liveParentTask, "");
|
|
assert.strictEqual(
|
|
workerRuntime.child.kill("SIGSTOP"),
|
|
true,
|
|
"failed to pause the real worker after observing its live child"
|
|
);
|
|
workerPaused = true;
|
|
|
|
const fakeCapabilities = await sendHostedControl(
|
|
signedNodeRequest(
|
|
securityNode.node,
|
|
securityNode.identity,
|
|
"report_node_capabilities",
|
|
fakeCapabilityBody,
|
|
{ nonce: `strict-agent-fake-capabilities-${suffix}` }
|
|
)
|
|
);
|
|
assert.strictEqual(
|
|
fakeCapabilities.type,
|
|
"node_capabilities_recorded",
|
|
JSON.stringify(fakeCapabilities)
|
|
);
|
|
|
|
const fakeTaskSpec = {
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
task_definition: "task_add_one",
|
|
task_instance: fakeTask,
|
|
dispatch: {
|
|
kind: "coordinator_node_wasm",
|
|
export: null,
|
|
abi: "task_v1",
|
|
},
|
|
environment_id: "strict-debug-missing-participant",
|
|
environment: null,
|
|
environment_digest: fakeEnvironmentDigest,
|
|
required_capabilities: [],
|
|
dependency_cache: null,
|
|
source_snapshot: null,
|
|
required_artifacts: [],
|
|
args: [{ SmallJson: 41 }],
|
|
vfs_epoch: valid.epoch,
|
|
bundle_digest: bundle.digest,
|
|
};
|
|
directTaskV1 = assertDenied(
|
|
await sendHostedControl(
|
|
signedAgentWorkflowRequest(
|
|
identity,
|
|
{
|
|
type: "launch_task",
|
|
tenant,
|
|
project,
|
|
actor_agent: agent,
|
|
task_spec: fakeTaskSpec,
|
|
wait_for_node: false,
|
|
artifact_path: `/vfs/artifacts/${fakeTask}.txt`,
|
|
wasm_module_base64: bundle.moduleBase64,
|
|
},
|
|
{ nonce: `strict-agent-direct-task-v1-${suffix}` }
|
|
)
|
|
),
|
|
/external callers.*EntrypointV1|TaskV1 requires an authenticated live parent/i,
|
|
"external direct TaskV1 launch"
|
|
);
|
|
|
|
const liveParentSpec =
|
|
liveParentAssignment.task_assignment_response.task_spec;
|
|
realDebugLaunch = await sendHostedControl(
|
|
signedNodeRequest(
|
|
workerRuntime.node,
|
|
workerRuntime.identity,
|
|
"launch_child_task",
|
|
{
|
|
type: "launch_child_task",
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
node: workerRuntime.node,
|
|
parent_task: liveParentTask,
|
|
task_spec: {
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
task_definition: "abort_probe",
|
|
task_instance: realDebugTask,
|
|
dispatch: {
|
|
kind: "coordinator_node_wasm",
|
|
export: null,
|
|
abi: "task_v1",
|
|
},
|
|
environment_id: liveParentSpec.environment_id,
|
|
environment: liveParentSpec.environment,
|
|
environment_digest: liveParentSpec.environment_digest,
|
|
required_capabilities: ["Command"],
|
|
dependency_cache: null,
|
|
source_snapshot: liveParentSpec.source_snapshot,
|
|
required_artifacts: [],
|
|
args: liveParentSpec.args,
|
|
vfs_epoch: valid.epoch,
|
|
bundle_digest: bundle.digest,
|
|
},
|
|
wait_for_node: false,
|
|
artifact_path: `/vfs/artifacts/${realDebugTask}.txt`,
|
|
wasm_module_base64: bundle.moduleBase64,
|
|
},
|
|
{ nonce: `strict-real-debug-participant-${suffix}` }
|
|
)
|
|
);
|
|
assert.strictEqual(
|
|
realDebugLaunch.type,
|
|
"task_launched",
|
|
JSON.stringify(realDebugLaunch)
|
|
);
|
|
|
|
const fakeLaunchBody = {
|
|
type: "launch_child_task",
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
node: workerRuntime.node,
|
|
parent_task: liveParentTask,
|
|
task_spec: fakeTaskSpec,
|
|
wait_for_node: false,
|
|
artifact_path: `/vfs/artifacts/${fakeTask}.txt`,
|
|
wasm_module_base64: bundle.moduleBase64,
|
|
};
|
|
fakeLaunch = await sendHostedControl(
|
|
signedNodeRequest(
|
|
workerRuntime.node,
|
|
workerRuntime.identity,
|
|
"launch_child_task",
|
|
fakeLaunchBody,
|
|
{ nonce: `strict-parent-runtime-fake-task-${suffix}` }
|
|
)
|
|
);
|
|
assert.strictEqual(fakeLaunch.type, "task_launched", JSON.stringify(fakeLaunch));
|
|
assert.strictEqual(fakeLaunch.placement.node, securityNode.node);
|
|
} finally {
|
|
if (workerPaused) {
|
|
assert.strictEqual(
|
|
workerRuntime.child.kill("SIGCONT"),
|
|
true,
|
|
"failed to resume the real worker after authenticated child launch"
|
|
);
|
|
}
|
|
}
|
|
await workerRuntime.worker.waitFor(
|
|
(value) =>
|
|
value.node_status === "assignment_started" &&
|
|
value.virtual_thread === realDebugTask,
|
|
"real Podman debug participant to start",
|
|
120_000
|
|
);
|
|
const realDebugContainerDeadline = Date.now() + 120_000;
|
|
let realDebugContainerIds = new Set();
|
|
let lastRealDebugPodmanStates = [];
|
|
while (Date.now() < realDebugContainerDeadline) {
|
|
const containers = runJson("podman", ["ps", "--format", "json"]);
|
|
lastRealDebugPodmanStates = containers;
|
|
realDebugContainerIds = new Set(
|
|
containers
|
|
.map((container) => container.Id || container.ID || container.IdHex || "")
|
|
.filter((id) => id && !baselineContainers.has(id))
|
|
);
|
|
if (realDebugContainerIds.size > 0) break;
|
|
await delay(100);
|
|
}
|
|
assert(
|
|
realDebugContainerIds.size > 0,
|
|
`real debug participant did not start a Podman container: ${JSON.stringify(
|
|
lastRealDebugPodmanStates
|
|
)}`
|
|
);
|
|
|
|
const debugClient = new DapClient({
|
|
cwd: projectDir,
|
|
command: clusterfluxDap,
|
|
args: [],
|
|
env: { ...process.env },
|
|
});
|
|
const initialize = debugClient.send("initialize", {
|
|
adapterID: "clusterflux",
|
|
linesStartAt1: true,
|
|
columnsStartAt1: true,
|
|
});
|
|
await debugClient.response(initialize, "initialize");
|
|
const attachRequest = debugClient.send("attach", {
|
|
entry: "build",
|
|
project: projectDir,
|
|
processId,
|
|
runtimeBackend: "live-services",
|
|
coordinatorEndpoint: serviceEndpoint,
|
|
});
|
|
await debugClient.response(attachRequest, "attach");
|
|
await debugClient.waitFor(
|
|
(message) => message.type === "event" && message.event === "initialized"
|
|
);
|
|
const configurationDone = debugClient.send("configurationDone");
|
|
await debugClient.response(configurationDone, "configurationDone");
|
|
const attachDeadline = Date.now() + 120_000;
|
|
let attachedThreads = [];
|
|
while (Date.now() < attachDeadline) {
|
|
const threadsRequest = debugClient.send("threads");
|
|
attachedThreads = (
|
|
await debugClient.response(threadsRequest, "threads")
|
|
).body.threads;
|
|
if (attachedThreads.length > 0) break;
|
|
await delay(100);
|
|
}
|
|
assert(attachedThreads.length > 0, "DAP attach reported no live threads");
|
|
|
|
const freezeStartedAt = Date.now();
|
|
const pauseRequest = debugClient.send("pause", {
|
|
threadId: attachedThreads[0].id,
|
|
});
|
|
await debugClient.response(pauseRequest, "pause");
|
|
const dapPartialStop = await debugClient.waitFor(
|
|
(message) =>
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "pause",
|
|
30_000
|
|
);
|
|
assert.strictEqual(dapPartialStop.body.allThreadsStopped, false);
|
|
const epochRequest = debugClient.send("evaluate", {
|
|
expression: "debug_epoch",
|
|
context: "watch",
|
|
});
|
|
const freeze = {
|
|
epoch: Number(
|
|
(await debugClient.response(epochRequest, "evaluate")).body.result
|
|
),
|
|
};
|
|
assert(Number.isSafeInteger(freeze.epoch) && freeze.epoch > 0);
|
|
const partialFreeze = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "inspect_debug_epoch",
|
|
process: processId,
|
|
epoch: freeze.epoch,
|
|
})
|
|
);
|
|
assert.strictEqual(partialFreeze.type, "debug_epoch_status");
|
|
assert.strictEqual(partialFreeze.partially_frozen, true, JSON.stringify(partialFreeze));
|
|
assert.strictEqual(partialFreeze.fully_frozen, false);
|
|
assert.strictEqual(partialFreeze.failed, true);
|
|
assert.match(
|
|
partialFreeze.failure_messages.join("; "),
|
|
/did not acknowledge frozen state within \d+ ms/
|
|
);
|
|
const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]);
|
|
const pausedContainers = podmanStates.filter((container) => {
|
|
const id = container.Id || container.ID || container.IdHex || "";
|
|
const state = String(container.State || container.Status || "").toLowerCase();
|
|
return realDebugContainerIds.has(id) && state.includes("paused");
|
|
});
|
|
assert(
|
|
pausedContainers.length > 0,
|
|
`partial Debug Epoch did not pause a real Podman container: ${JSON.stringify(
|
|
podmanStates
|
|
)}`
|
|
);
|
|
const pausedContainerIds = pausedContainers.map(
|
|
(container) => container.Id || container.ID || container.IdHex
|
|
);
|
|
|
|
const mismatchedDigest = sha256(Buffer.from(`strict-nested-mismatch-${suffix}`));
|
|
const mismatchedChild = await sendHostedControl(
|
|
signedNodeRequest(
|
|
securityNode.node,
|
|
securityNode.identity,
|
|
"launch_child_task",
|
|
{
|
|
type: "launch_child_task",
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
node: securityNode.node,
|
|
parent_task: fakeTask,
|
|
task_spec: {
|
|
tenant,
|
|
project,
|
|
process: processId,
|
|
task_definition: "task_add_one",
|
|
task_instance: `${fakeTask}:child:mismatched-environment`,
|
|
dispatch: {
|
|
kind: "coordinator_node_wasm",
|
|
export: null,
|
|
abi: "task_v1",
|
|
},
|
|
environment_id: "strict-nested-mismatch",
|
|
environment: null,
|
|
environment_digest: mismatchedDigest,
|
|
required_capabilities: [],
|
|
dependency_cache: null,
|
|
source_snapshot: null,
|
|
required_artifacts: [],
|
|
args: [{ SmallJson: 41 }],
|
|
vfs_epoch: valid.epoch,
|
|
bundle_digest: bundle.digest,
|
|
},
|
|
wait_for_node: false,
|
|
artifact_path: `/vfs/artifacts/${fakeTask}-mismatched-child.txt`,
|
|
wasm_module_base64: bundle.moduleBase64,
|
|
},
|
|
{ nonce: `strict-nested-mismatch-${suffix}` }
|
|
)
|
|
);
|
|
const nestedEnvironmentMismatch = assertDenied(
|
|
mismatchedChild,
|
|
/environment|digest|compatible|placement|no node/i,
|
|
"nested environment mismatch"
|
|
);
|
|
|
|
const continueStartedAt = Date.now();
|
|
const continueRequest = debugClient.send("continue", {
|
|
threadId: dapPartialStop.body.threadId,
|
|
});
|
|
await debugClient.response(continueRequest, "continue");
|
|
const continueResponseMs = Date.now() - continueStartedAt;
|
|
assert(
|
|
continueResponseMs < 2_000,
|
|
`partial-freeze continue blocked for ${continueResponseMs} ms`
|
|
);
|
|
await delay(1000);
|
|
const resumed = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "inspect_debug_epoch",
|
|
process: processId,
|
|
epoch: freeze.epoch,
|
|
})
|
|
);
|
|
assert.strictEqual(resumed.type, "debug_epoch_status");
|
|
const resumedAcknowledgements = resumed.acknowledgements.filter(
|
|
(acknowledgement) => acknowledgement.state === "running"
|
|
);
|
|
assert(resumedAcknowledgements.length > 0, JSON.stringify(resumed));
|
|
const resumedPodmanStates = runJson("podman", ["ps", "--format", "json"]);
|
|
assert(
|
|
!resumedPodmanStates.some((container) => {
|
|
const id = container.Id || container.ID || container.IdHex || "";
|
|
const state = String(container.State || container.Status || "").toLowerCase();
|
|
return pausedContainerIds.includes(id) && state.includes("paused");
|
|
}),
|
|
`DAP continue left a Clusterflux container paused: ${JSON.stringify(
|
|
resumedPodmanStates
|
|
)}`
|
|
);
|
|
await debugClient.close();
|
|
|
|
const completed = await waitForCli(
|
|
"agent-authenticated coordinator main and child workflow completion",
|
|
() =>
|
|
runJson(clusterflux, ["task", "list", ...scope, "--process", processId], {
|
|
cwd: projectDir,
|
|
}),
|
|
(tasks) => completedFlagship(rawTaskEvents(tasks)),
|
|
120000
|
|
);
|
|
const completedEvents = rawTaskEvents(completed);
|
|
const concurrentCompileTasks = [
|
|
...new Set(
|
|
completedEvents
|
|
.filter(
|
|
(event) =>
|
|
event.task_definition === "compile_linux" &&
|
|
event.terminal_state === "completed"
|
|
)
|
|
.map((event) => event.task)
|
|
),
|
|
];
|
|
assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents));
|
|
|
|
const revoked = runJson(
|
|
clusterflux,
|
|
["key", "revoke", ...scope, "--agent", agent, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(revoked.command, "key revoke");
|
|
const revokedUse = assertDenied(
|
|
await sendHostedControl(
|
|
signedAgentWorkflowRequest(
|
|
identity,
|
|
{ ...unsigned, process: `${processId}-revoked` },
|
|
{ nonce: `strict-agent-revoked-${suffix}` }
|
|
)
|
|
),
|
|
/revoked/i,
|
|
"revoked agent credential"
|
|
);
|
|
return {
|
|
agent,
|
|
valid_request: valid.type,
|
|
replay,
|
|
expired,
|
|
forged,
|
|
body_modified: bodyModified,
|
|
revoked: revokedUse,
|
|
workflow: {
|
|
process: processId,
|
|
main_launch: mainLaunch.type,
|
|
external_direct_task_v1: directTaskV1,
|
|
child_launch: fakeLaunch.type,
|
|
child_parent: liveParentTask,
|
|
real_debug_child_launch: realDebugLaunch.type,
|
|
real_debug_child: realDebugTask,
|
|
actor_kind: mainLaunch.actor.kind,
|
|
authenticated_without_browser: mainLaunch.actor.authenticated_without_browser,
|
|
flagship_completed: completedFlagship(completedEvents),
|
|
concurrent_compile_tasks: concurrentCompileTasks,
|
|
},
|
|
partial_freeze: {
|
|
epoch: freeze.epoch,
|
|
elapsed_ms: Date.now() - freezeStartedAt,
|
|
partially_frozen: partialFreeze.partially_frozen,
|
|
fully_frozen: partialFreeze.fully_frozen,
|
|
dap_all_threads_stopped: dapPartialStop.body.allThreadsStopped,
|
|
dap_continue_response_ms: continueResponseMs,
|
|
podman_paused_container_ids: pausedContainerIds,
|
|
warning: partialFreeze.failure_messages,
|
|
resumed_participants: resumedAcknowledgements.map(
|
|
(acknowledgement) => acknowledgement.task
|
|
),
|
|
},
|
|
nested_environment_mismatch: nestedEnvironmentMismatch,
|
|
};
|
|
}
|
|
|
|
async function runSecondTenantIsolation({
|
|
firstTenant,
|
|
firstProject,
|
|
firstProcess,
|
|
firstNode,
|
|
firstArtifact,
|
|
manifest,
|
|
}) {
|
|
const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE;
|
|
const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE;
|
|
if (!file && evidenceFile) {
|
|
const evidenceBytes = fs.readFileSync(path.resolve(evidenceFile));
|
|
const previous = JSON.parse(evidenceBytes);
|
|
const isolation = previous.tenant_isolation;
|
|
assert.strictEqual(isolation?.executed, true);
|
|
assert.notStrictEqual(isolation.second_tenant, firstTenant);
|
|
assert.strictEqual(isolation.project?.denied, true);
|
|
assert.strictEqual(isolation.process_hidden, true);
|
|
assert.strictEqual(isolation.node_hidden, true);
|
|
assert.strictEqual(isolation.tasks_and_logs?.denied, true);
|
|
assert.strictEqual(isolation.debug?.denied, true);
|
|
assert.strictEqual(isolation.debug?.audited, true);
|
|
assert.strictEqual(isolation.debug?.charged_debug_read_bytes, 0);
|
|
assert.strictEqual(isolation.artifact_and_download?.denied, true);
|
|
assert.strictEqual(isolation.process_control?.denied, true);
|
|
const previousDeployment = previous.release_binding?.deployment;
|
|
const currentDeployment = deploymentProvenance();
|
|
assert.strictEqual(
|
|
previousDeployment?.hosted_service_sha256,
|
|
currentDeployment.hosted_service_sha256,
|
|
"reused isolation evidence must target the exact deployed hosted binary"
|
|
);
|
|
assert.deepStrictEqual(
|
|
previous.release_binding?.binary_digests,
|
|
manifest.binary_digests,
|
|
"reused isolation evidence must target the exact public binaries"
|
|
);
|
|
return {
|
|
...isolation,
|
|
reused_from_immutable_evidence: {
|
|
report_sha256: sha256(evidenceBytes),
|
|
source_commit: previous.source_commit,
|
|
hosted_service_sha256: currentDeployment.hosted_service_sha256,
|
|
public_binary_digests: manifest.binary_digests,
|
|
},
|
|
};
|
|
}
|
|
if (!file) return { executed: false, reason: "second tenant session not supplied" };
|
|
const second = readJson(path.resolve(file));
|
|
assert.strictEqual(second.coordinator, serviceEndpoint);
|
|
assert(second.session_secret, "second tenant session omitted its session secret");
|
|
assert.notStrictEqual(
|
|
second.tenant,
|
|
firstTenant,
|
|
"isolation proof requires a genuinely distinct hosted tenant"
|
|
);
|
|
const call = (request) =>
|
|
sendHostedControl(authenticatedRequest(second.session_secret, request));
|
|
|
|
const project = assertDenied(
|
|
await call({ type: "select_project", project: firstProject }),
|
|
/outside|not visible|tenant|project/i,
|
|
"cross-tenant project selection"
|
|
);
|
|
const processes = await call({ type: "list_processes" });
|
|
assert.strictEqual(
|
|
processes.type,
|
|
"process_statuses",
|
|
JSON.stringify(processes)
|
|
);
|
|
assert(!JSON.stringify(processes).includes(firstProcess));
|
|
const nodes = await call({ type: "list_node_descriptors" });
|
|
assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes));
|
|
assert(!JSON.stringify(nodes).includes(firstNode));
|
|
const tasksAndLogs = assertDenied(
|
|
await call({ type: "list_task_events", process: firstProcess }),
|
|
/outside|scope|tenant|project|not active|requires an active|unknown/i,
|
|
"cross-tenant task and log listing"
|
|
);
|
|
const debugResponse = await call({
|
|
type: "debug_attach",
|
|
process: firstProcess,
|
|
});
|
|
assert.strictEqual(debugResponse.type, "debug_attach", JSON.stringify(debugResponse));
|
|
assert.strictEqual(debugResponse.authorization?.allowed, false);
|
|
assert.strictEqual(debugResponse.audit_event?.allowed, false);
|
|
assert.match(
|
|
debugResponse.authorization?.reason || "",
|
|
/outside|scope|permission|tenant|project|not active|unknown/i
|
|
);
|
|
assert.strictEqual(debugResponse.charged_debug_read_bytes, 0);
|
|
const debug = {
|
|
denied: true,
|
|
reason: debugResponse.authorization.reason,
|
|
audited: true,
|
|
charged_debug_read_bytes: debugResponse.charged_debug_read_bytes,
|
|
};
|
|
const artifact = assertDenied(
|
|
await call({
|
|
type: "create_artifact_download_link",
|
|
artifact: firstArtifact,
|
|
max_bytes: 1024,
|
|
ttl_seconds: 60,
|
|
}),
|
|
/outside|scope|tenant|project|not found|unavailable/i,
|
|
"cross-tenant artifact download"
|
|
);
|
|
const control = assertDenied(
|
|
await call({ type: "abort_process", process: firstProcess }),
|
|
/outside|scope|tenant|project|not active|requires an active|unknown/i,
|
|
"cross-tenant process control"
|
|
);
|
|
return {
|
|
executed: true,
|
|
second_tenant: second.tenant,
|
|
project,
|
|
process_hidden: true,
|
|
node_hidden: true,
|
|
tasks_and_logs: tasksAndLogs,
|
|
debug,
|
|
artifact_and_download: artifact,
|
|
process_control: control,
|
|
};
|
|
}
|
|
|
|
async function runLiveSoak({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
securityNode,
|
|
}) {
|
|
if (!strictVpsRestart) {
|
|
return { executed: false, reason: "strict VPS measurement access not supplied" };
|
|
}
|
|
const runReport = runJson(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(runReport.status, "main_launched");
|
|
const processId = runReport.process;
|
|
const parked = await waitForCli(
|
|
"soak coordinator main to park without a capable node",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) =>
|
|
status.live_process?.main_state === "running" &&
|
|
status.live_process?.main_wait_state === "waiting_for_task" &&
|
|
status.current_task_count === 0 &&
|
|
status.live_process?.connected_nodes?.length === 0,
|
|
120000
|
|
);
|
|
|
|
const pollSecurityNode = (nonce) =>
|
|
sendHostedControl(
|
|
signedNodeRequest(
|
|
securityNode.node,
|
|
securityNode.identity,
|
|
"poll_task_assignment",
|
|
{
|
|
type: "poll_task_assignment",
|
|
tenant: securityNode.capability_body.tenant,
|
|
project: securityNode.capability_body.project,
|
|
node: securityNode.node,
|
|
},
|
|
{ nonce }
|
|
)
|
|
);
|
|
const drainedPriorAssignments = [];
|
|
for (let drainIndex = 0; drainIndex < 16; drainIndex += 1) {
|
|
const poll = await pollSecurityNode(
|
|
`strict-soak-drain-${drainIndex}-${Date.now()}`
|
|
);
|
|
assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll));
|
|
if (poll.assignment === null) break;
|
|
assert.notStrictEqual(
|
|
poll.assignment.process,
|
|
processId,
|
|
"soak process unexpectedly received a task assignment"
|
|
);
|
|
drainedPriorAssignments.push({
|
|
process: poll.assignment.process,
|
|
task: poll.assignment.task,
|
|
});
|
|
assert(drainIndex < 15, "pre-soak assignment drain did not quiesce");
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const deadline = startedAt + strictSoakSeconds * 1000;
|
|
const samples = [];
|
|
let sampleIndex = 0;
|
|
while (true) {
|
|
const capabilities = await sendHostedControl(
|
|
signedNodeRequest(
|
|
securityNode.node,
|
|
securityNode.identity,
|
|
"report_node_capabilities",
|
|
securityNode.capability_body,
|
|
{ nonce: `strict-soak-capabilities-${sampleIndex}-${Date.now()}` }
|
|
)
|
|
);
|
|
assert.strictEqual(
|
|
capabilities.type,
|
|
"node_capabilities_recorded",
|
|
JSON.stringify(capabilities)
|
|
);
|
|
const poll = await pollSecurityNode(
|
|
`strict-soak-poll-${sampleIndex}-${Date.now()}`
|
|
);
|
|
assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll));
|
|
assert.strictEqual(poll.assignment, null);
|
|
const status = runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(status.live_process.main_wait_state, "waiting_for_task");
|
|
samples.push(strictInfrastructureSample(projectDir));
|
|
sampleIndex += 1;
|
|
if (Date.now() >= deadline) break;
|
|
await delay(Math.min(30_000, deadline - Date.now()));
|
|
}
|
|
|
|
const currentMemory = samples.map(
|
|
(sample) => sample.service_memory_current_bytes
|
|
);
|
|
const serviceDisk = samples.map((sample) => sample.service_state_disk_bytes);
|
|
const databaseDisk = samples.map((sample) => sample.postgres_database_bytes);
|
|
const localDisk = samples.map((sample) => sample.local_node_state.bytes);
|
|
const spread = (values) => Math.max(...values) - Math.min(...values);
|
|
assert(spread(currentMemory) <= 128 * 1024 * 1024);
|
|
assert(spread(serviceDisk) <= 16 * 1024 * 1024);
|
|
assert(spread(databaseDisk) <= 32 * 1024 * 1024);
|
|
assert(spread(localDisk) <= 16 * 1024 * 1024);
|
|
assert.deepStrictEqual(
|
|
samples.at(-1).durable_rows,
|
|
samples[0].durable_rows,
|
|
"durable object counts grew during the parked-process soak"
|
|
);
|
|
const aborted = runJson(
|
|
clusterflux,
|
|
["process", "abort", ...scope, "--process", processId, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(aborted.abort_request.process_slot_released, true);
|
|
return {
|
|
executed: true,
|
|
process: processId,
|
|
duration_seconds: Math.floor((Date.now() - startedAt) / 1000),
|
|
sample_interval_seconds: 30,
|
|
sample_count: samples.length,
|
|
parked_main: {
|
|
main_state: parked.live_process.main_state,
|
|
main_wait_state: parked.live_process.main_wait_state,
|
|
},
|
|
drained_prior_assignments: drainedPriorAssignments,
|
|
signed_node_poll_cycles: samples.length,
|
|
memory_spread_bytes: spread(currentMemory),
|
|
service_disk_spread_bytes: spread(serviceDisk),
|
|
postgres_disk_spread_bytes: spread(databaseDisk),
|
|
local_node_state_spread_bytes: spread(localDisk),
|
|
samples,
|
|
};
|
|
}
|
|
|
|
async function revokeSecurityNode({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
securityNode,
|
|
}) {
|
|
const revoked = runJson(
|
|
clusterflux,
|
|
["node", "revoke", ...scope, "--node", securityNode.node, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(revoked.command, "node revoke");
|
|
const denied = assertDenied(
|
|
await sendHostedControl({
|
|
type: "node_heartbeat",
|
|
node: securityNode.node,
|
|
node_signature: signedNodeHeartbeat(
|
|
securityNode.node,
|
|
securityNode.identity,
|
|
{ nonce: `strict-node-revoked-${Date.now()}` }
|
|
),
|
|
}),
|
|
/revoked|unknown node|not recognized|not enrolled/i,
|
|
"revoked node credential"
|
|
);
|
|
return { node: securityNode.node, denied };
|
|
}
|
|
|
|
async function runLiveQuotaPreallocation({ sessionSecret, suffix }) {
|
|
const readSpawnQuota = async () => {
|
|
const status = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, { type: "quota_status" })
|
|
);
|
|
assert.strictEqual(status.type, "quota_status", JSON.stringify(status));
|
|
const limit = status.limits?.limits?.Spawn;
|
|
const usage = status.usage?.Spawn;
|
|
const windowSeconds = status.window_seconds?.Spawn;
|
|
const windowStarted = status.window_started_epoch_seconds?.Spawn;
|
|
assert(Number.isInteger(limit) && limit > 0, JSON.stringify(status));
|
|
assert(Number.isInteger(usage) && usage >= 0, JSON.stringify(status));
|
|
assert(
|
|
Number.isInteger(windowSeconds) && windowSeconds > 0,
|
|
JSON.stringify(status)
|
|
);
|
|
assert(Number.isInteger(windowStarted), JSON.stringify(status));
|
|
return { limit, usage, windowSeconds, windowStarted };
|
|
};
|
|
|
|
let spawnQuota = await readSpawnQuota();
|
|
const windowElapsed = Math.max(
|
|
0,
|
|
Math.floor(Date.now() / 1000) - spawnQuota.windowStarted
|
|
);
|
|
if (spawnQuota.usage > 0 || windowElapsed > 1) {
|
|
await delay(
|
|
Math.max(1000, (spawnQuota.windowSeconds - windowElapsed + 1) * 1000)
|
|
);
|
|
spawnQuota = await readSpawnQuota();
|
|
}
|
|
let denial;
|
|
let deniedProcess;
|
|
let acceptedStarts = 0;
|
|
const maxAttempts = spawnQuota.limit - spawnQuota.usage + 2;
|
|
for (let index = 0; index < maxAttempts; index += 1) {
|
|
const processId = `vp-quota-${suffix}-${index}`;
|
|
const started = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "start_process",
|
|
process: processId,
|
|
restart: false,
|
|
})
|
|
);
|
|
if (started.type === "error") {
|
|
assert.match(started.message, /quota|spawn|limit|community tier/i);
|
|
denial = started;
|
|
deniedProcess = processId;
|
|
break;
|
|
}
|
|
assert.strictEqual(started.type, "process_started", JSON.stringify(started));
|
|
acceptedStarts += 1;
|
|
const aborted = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "abort_process",
|
|
process: processId,
|
|
})
|
|
);
|
|
assert.strictEqual(aborted.type, "process_aborted", JSON.stringify(aborted));
|
|
}
|
|
assert(
|
|
denial,
|
|
`live community-tier spawn quota was not reached in ${maxAttempts} attempts`
|
|
);
|
|
const processes = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, { type: "list_processes" })
|
|
);
|
|
assert.strictEqual(
|
|
processes.type,
|
|
"process_statuses",
|
|
JSON.stringify(processes)
|
|
);
|
|
assert(
|
|
!JSON.stringify(processes).includes(deniedProcess),
|
|
"quota-denied process was allocated before denial"
|
|
);
|
|
const second = readJson(
|
|
path.resolve(process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE)
|
|
);
|
|
assert.notStrictEqual(second.session_secret, sessionSecret);
|
|
const unrelatedProcess = `vp-unrelated-quota-${suffix}`;
|
|
const unrelatedStarted = await sendHostedControl(
|
|
authenticatedRequest(second.session_secret, {
|
|
type: "start_process",
|
|
process: unrelatedProcess,
|
|
restart: false,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
unrelatedStarted.type,
|
|
"process_started",
|
|
`first tenant quota exhaustion affected another tenant: ${JSON.stringify(unrelatedStarted)}`
|
|
);
|
|
const unrelatedAborted = await sendHostedControl(
|
|
authenticatedRequest(second.session_secret, {
|
|
type: "abort_process",
|
|
process: unrelatedProcess,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
unrelatedAborted.type,
|
|
"process_aborted",
|
|
JSON.stringify(unrelatedAborted)
|
|
);
|
|
return {
|
|
limit: spawnQuota.limit,
|
|
initial_usage: spawnQuota.usage,
|
|
window_seconds: spawnQuota.windowSeconds,
|
|
accepted_starts_before_denial: acceptedStarts,
|
|
denied_process: deniedProcess,
|
|
denial: {
|
|
type: denial.type,
|
|
category: denial.error?.category || null,
|
|
message: denial.message,
|
|
},
|
|
denied_process_allocated: false,
|
|
unrelated_tenant: {
|
|
tenant: second.tenant,
|
|
process: unrelatedProcess,
|
|
start: unrelatedStarted.type,
|
|
abort: unrelatedAborted.type,
|
|
unaffected_by_first_tenant_quota: true,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runLongJoinProof({ clusterflux, projectDir, scope }) {
|
|
const startedAt = Date.now();
|
|
const runReport = runJson(
|
|
clusterflux,
|
|
["run", "long-join", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(runReport.status, "main_launched");
|
|
const terminal = await waitForCli(
|
|
"controlled task longer than two minutes",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(tasks) =>
|
|
rawTaskEvents(tasks).some(
|
|
(event) =>
|
|
event.task_definition === "long_join_probe" &&
|
|
event.terminal_state === "completed"
|
|
),
|
|
5 * 60 * 1000
|
|
);
|
|
const completed = rawTaskEvents(terminal).find(
|
|
(event) =>
|
|
event.task_definition === "long_join_probe" &&
|
|
event.terminal_state === "completed"
|
|
);
|
|
const durationSeconds = Math.floor((Date.now() - startedAt) / 1000);
|
|
assert(durationSeconds > 120);
|
|
const released = await waitForCli(
|
|
"long join process automatic slot release",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
120000
|
|
);
|
|
return {
|
|
process: runReport.process,
|
|
task: completed.task,
|
|
terminal_state: completed.terminal_state,
|
|
result: completed.result,
|
|
duration_seconds: durationSeconds,
|
|
process_state: released.state,
|
|
};
|
|
}
|
|
|
|
async function runRepeatedParkWakeProof({ clusterflux, projectDir, scope }) {
|
|
const runReport = runJson(
|
|
clusterflux,
|
|
["run", "park-wake", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(runReport.status, "main_launched");
|
|
const terminal = await waitForCli(
|
|
"repeated coordinator-main park/wake completion",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(tasks) => {
|
|
const completed = new Set(
|
|
rawTaskEvents(tasks)
|
|
.filter(
|
|
(event) =>
|
|
event.task_definition === "task_add_one" &&
|
|
event.terminal_state === "completed"
|
|
)
|
|
.map((event) => event.task)
|
|
);
|
|
return completed.size >= 16;
|
|
},
|
|
120000
|
|
);
|
|
const completedTasks = [
|
|
...new Set(
|
|
rawTaskEvents(terminal)
|
|
.filter(
|
|
(event) =>
|
|
event.task_definition === "task_add_one" &&
|
|
event.terminal_state === "completed"
|
|
)
|
|
.map((event) => event.task)
|
|
),
|
|
];
|
|
const released = await waitForCli(
|
|
"park/wake process automatic slot release",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
120000
|
|
);
|
|
return {
|
|
process: runReport.process,
|
|
completed_cycles: completedTasks.length,
|
|
task_instances: completedTasks,
|
|
terminal_state: released.state,
|
|
};
|
|
}
|
|
|
|
async function runDroppedConnectionRollback({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
}) {
|
|
const scenarioStartedAt = Date.now();
|
|
const processId = "vp-current";
|
|
const attempted = cp.spawnSync(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{
|
|
cwd: projectDir,
|
|
env: {
|
|
...process.env,
|
|
CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION: "start_process",
|
|
},
|
|
encoding: "utf8",
|
|
timeout: 5 * 60 * 1000,
|
|
}
|
|
);
|
|
assert.notStrictEqual(
|
|
attempted.status,
|
|
0,
|
|
`response-loss run unexpectedly succeeded: ${attempted.stdout}`
|
|
);
|
|
assert.match(
|
|
`${attempted.stderr}\n${attempted.stdout}`,
|
|
/closed.*without a response|transport|response/i
|
|
);
|
|
const released = await waitForCli(
|
|
"CLI launch guard rollback after lost start response",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
30000
|
|
);
|
|
return {
|
|
duration_ms: Date.now() - scenarioStartedAt,
|
|
process: processId,
|
|
failure_injection: "control transport dropped start_process response",
|
|
cli_exit_status: attempted.status,
|
|
rollback: "automatic CLI launch guard abort_process",
|
|
terminal_state: released.state,
|
|
};
|
|
}
|
|
|
|
async function runLiveBandwidthPreallocation({
|
|
sessionSecret,
|
|
artifact,
|
|
maxBytes,
|
|
}) {
|
|
const scenarioStartedAt = Date.now();
|
|
const before = relayDurableState();
|
|
const partialLink = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "create_artifact_download_link",
|
|
artifact,
|
|
max_bytes: maxBytes,
|
|
ttl_seconds: 300,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
partialLink.type,
|
|
"artifact_download_link",
|
|
JSON.stringify(partialLink)
|
|
);
|
|
const partialToken = partialLink.link.scoped_token_digest;
|
|
let partialChunk;
|
|
const partialDeadline = Date.now() + 120_000;
|
|
while (Date.now() < partialDeadline) {
|
|
const response = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "open_artifact_download_stream",
|
|
artifact,
|
|
max_bytes: maxBytes,
|
|
token_digest: partialToken,
|
|
chunk_bytes: 16,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
response.type,
|
|
"artifact_download_stream",
|
|
JSON.stringify(response)
|
|
);
|
|
if (response.content_bytes_available === true) {
|
|
partialChunk = response;
|
|
break;
|
|
}
|
|
await delay(50);
|
|
}
|
|
assert(partialChunk, "artifact relay did not return the first partial chunk");
|
|
assert.strictEqual(partialChunk.content_eof, false);
|
|
assert.strictEqual(partialChunk.streamed_bytes, 16);
|
|
const partialRevoked = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "revoke_artifact_download_link",
|
|
artifact,
|
|
token_digest: partialToken,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
partialRevoked.type,
|
|
"artifact_download_link_revoked",
|
|
JSON.stringify(partialRevoked)
|
|
);
|
|
const afterPartialAbandon = relayDurableState();
|
|
assert(afterPartialAbandon.ingress_used > before.ingress_used);
|
|
assert(afterPartialAbandon.egress_used > before.egress_used);
|
|
assert(
|
|
afterPartialAbandon.abandoned_or_failed_used >
|
|
before.abandoned_or_failed_used
|
|
);
|
|
|
|
let denial;
|
|
const acceptedReservations = [];
|
|
for (let index = 0; index < 64; index += 1) {
|
|
const response = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "create_artifact_download_link",
|
|
artifact,
|
|
max_bytes: maxBytes,
|
|
ttl_seconds: 300,
|
|
})
|
|
);
|
|
if (response.type === "error") {
|
|
assert.match(
|
|
response.message,
|
|
/artifact relay.*(?:concurrency|period byte budget)|bandwidth|reservation/i
|
|
);
|
|
denial = response;
|
|
break;
|
|
}
|
|
assert.strictEqual(response.type, "artifact_download_link", JSON.stringify(response));
|
|
acceptedReservations.push(response.link.scoped_token_digest);
|
|
}
|
|
assert(denial, "artifact relay preallocation was not denied in 64 reservations");
|
|
for (const tokenDigest of acceptedReservations) {
|
|
const revoked = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "revoke_artifact_download_link",
|
|
artifact,
|
|
token_digest: tokenDigest,
|
|
})
|
|
);
|
|
assert.strictEqual(
|
|
revoked.type,
|
|
"artifact_download_link_revoked",
|
|
JSON.stringify(revoked)
|
|
);
|
|
}
|
|
const afterRelease = relayDurableState();
|
|
assert.strictEqual(
|
|
Object.keys(afterRelease.reservations || {}).length,
|
|
Object.keys(before.reservations || {}).length
|
|
);
|
|
return {
|
|
scenario_started_at_ms: scenarioStartedAt,
|
|
artifact,
|
|
accepted_reservations_before_denial: acceptedReservations.length,
|
|
bytes_served_before_denial: partialChunk.streamed_bytes,
|
|
partial_abandon: {
|
|
ingress_delta: afterPartialAbandon.ingress_used - before.ingress_used,
|
|
egress_delta: afterPartialAbandon.egress_used - before.egress_used,
|
|
abandoned_delta:
|
|
afterPartialAbandon.abandoned_or_failed_used -
|
|
before.abandoned_or_failed_used,
|
|
reservation_released: !(partialToken in afterRelease.reservations),
|
|
},
|
|
durable_before: before,
|
|
durable_after_release: afterRelease,
|
|
denial: {
|
|
type: denial.type,
|
|
category: denial.error?.category || null,
|
|
message: denial.message,
|
|
},
|
|
};
|
|
}
|
|
|
|
function relayDurableState() {
|
|
if (!strictVpsRestart) {
|
|
throw new Error("strict relay accounting requires VPS Postgres access");
|
|
}
|
|
const output = run(
|
|
"ssh",
|
|
sshArgs(
|
|
"sudo",
|
|
"-u",
|
|
"postgres",
|
|
"psql",
|
|
"-d",
|
|
"clusterflux",
|
|
"-At",
|
|
"-c",
|
|
"SELECT record::text FROM clusterflux_artifact_relay_state WHERE singleton = TRUE"
|
|
)
|
|
).trim();
|
|
assert.notStrictEqual(output, "", "artifact relay durable state was not persisted");
|
|
return JSON.parse(output.split(/\r?\n/).filter(Boolean).at(-1));
|
|
}
|
|
|
|
async function waitForHostedControlReady(timeoutMs = 60_000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastError;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const ping = await sendHostedControl({ type: "ping" });
|
|
if (ping.type === "pong") return ping;
|
|
lastError = new Error(JSON.stringify(ping));
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
await delay(500);
|
|
}
|
|
throw new Error(`hosted service did not recover: ${lastError}`);
|
|
}
|
|
|
|
async function publishRelayProbeArtifact({ clusterflux, projectDir, scope, label }) {
|
|
const runReport = runJson(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
await waitForCli(
|
|
`${label} flagship completion`,
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(report) => completedFlagship(rawTaskEvents(report)),
|
|
10 * 60 * 1000
|
|
);
|
|
const artifacts = runJson(
|
|
clusterflux,
|
|
["artifact", "list", ...scope, "--process", runReport.process],
|
|
{ cwd: projectDir }
|
|
);
|
|
const artifact = artifacts.artifacts.find((candidate) => {
|
|
const digest = candidate.digest;
|
|
return (
|
|
typeof digest === "string" &&
|
|
/^sha256:[0-9a-f]{64}$/.test(digest) &&
|
|
candidate.artifact === `release.tar-${digest.slice("sha256:".length)}`
|
|
);
|
|
});
|
|
assert(artifact, `${label} did not publish a relay probe artifact`);
|
|
return artifact;
|
|
}
|
|
|
|
async function runRelayEmergencyDisable({
|
|
clusterflux,
|
|
clusterfluxNode,
|
|
projectDir,
|
|
scope,
|
|
workerArgs,
|
|
sessionSecret,
|
|
maxBytes,
|
|
}) {
|
|
if (!strictVpsRestart) {
|
|
throw new Error("strict relay disable proof requires VPS systemd access");
|
|
}
|
|
const dropInDir = `/run/systemd/system/${strictServiceUnit}.d`;
|
|
const dropInPath = `${dropInDir}/90-strict-relay-disable.conf`;
|
|
run("ssh", sshArgs("sudo", "mkdir", "-p", dropInDir));
|
|
cp.execFileSync("ssh", sshArgs("sudo", "tee", dropInPath), {
|
|
cwd: repo,
|
|
input: "[Service]\nEnvironment=CLUSTERFLUX_HOSTED_RELAY_DISABLED=true\n",
|
|
stdio: ["pipe", "ignore", "inherit"],
|
|
});
|
|
let disabled;
|
|
let disabledArtifact;
|
|
let disabledWorker;
|
|
try {
|
|
run(
|
|
"ssh",
|
|
sshArgs(
|
|
"sudo",
|
|
"systemctl",
|
|
"daemon-reload"
|
|
)
|
|
);
|
|
run(
|
|
"ssh",
|
|
sshArgs("sudo", "systemctl", "restart", strictServiceUnit)
|
|
);
|
|
await waitForHostedControlReady();
|
|
disabledWorker = spawnJsonLines(clusterfluxNode, workerArgs, {
|
|
cwd: projectDir,
|
|
env: { ...process.env },
|
|
});
|
|
await disabledWorker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"relay-disabled worker ready"
|
|
);
|
|
disabledArtifact = await publishRelayProbeArtifact({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
label: "relay-disabled",
|
|
});
|
|
disabled = assertDenied(
|
|
await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "create_artifact_download_link",
|
|
artifact: disabledArtifact.artifact,
|
|
max_bytes: maxBytes,
|
|
ttl_seconds: 60,
|
|
})
|
|
),
|
|
/artifact relay is disabled/i,
|
|
"emergency-disabled artifact relay"
|
|
);
|
|
} finally {
|
|
if (disabledWorker) await stopChild(disabledWorker.child);
|
|
run("ssh", sshArgs("sudo", "rm", "-f", dropInPath));
|
|
run("ssh", sshArgs("sudo", "systemctl", "daemon-reload"));
|
|
run(
|
|
"ssh",
|
|
sshArgs("sudo", "systemctl", "restart", strictServiceUnit)
|
|
);
|
|
await waitForHostedControlReady();
|
|
}
|
|
const restoredWorker = spawnJsonLines(clusterfluxNode, workerArgs, {
|
|
cwd: projectDir,
|
|
env: { ...process.env },
|
|
});
|
|
try {
|
|
await restoredWorker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"relay-restored worker ready"
|
|
);
|
|
const restoredArtifact = await publishRelayProbeArtifact({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
label: "relay-restored",
|
|
});
|
|
const restored = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "create_artifact_download_link",
|
|
artifact: restoredArtifact.artifact,
|
|
max_bytes: maxBytes,
|
|
ttl_seconds: 60,
|
|
})
|
|
);
|
|
assert.strictEqual(restored.type, "artifact_download_link", JSON.stringify(restored));
|
|
const revoked = await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, {
|
|
type: "revoke_artifact_download_link",
|
|
artifact: restoredArtifact.artifact,
|
|
token_digest: restored.link.scoped_token_digest,
|
|
})
|
|
);
|
|
assert.strictEqual(revoked.type, "artifact_download_link_revoked");
|
|
return {
|
|
evidence: {
|
|
disabled,
|
|
disabled_probe_artifact: disabledArtifact.artifact,
|
|
restored: restored.type,
|
|
restored_probe_artifact: restoredArtifact.artifact,
|
|
restored_reservation_released: revoked.type,
|
|
runtime_drop_in_removed: true,
|
|
},
|
|
worker: restoredWorker,
|
|
};
|
|
} catch (error) {
|
|
await stopChild(restoredWorker.child);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function deploymentProvenance() {
|
|
if (!strictVpsRestart) return null;
|
|
const systemGeneration = run(
|
|
"ssh",
|
|
sshArgs("readlink", "-f", "/run/current-system")
|
|
).trim();
|
|
const mainPid = run(
|
|
"ssh",
|
|
sshArgs(
|
|
"systemctl",
|
|
"show",
|
|
strictServiceUnit,
|
|
"--property=MainPID",
|
|
"--value"
|
|
)
|
|
).trim();
|
|
assert.match(mainPid, /^[1-9][0-9]*$/, "hosted service has no live MainPID");
|
|
const executable = run(
|
|
"ssh",
|
|
sshArgs("readlink", "-f", `/proc/${mainPid}/exe`)
|
|
).trim();
|
|
const binarySha256 = run(
|
|
"ssh",
|
|
sshArgs("sha256sum", `/proc/${mainPid}/exe`)
|
|
)
|
|
.trim()
|
|
.split(/\s+/)[0];
|
|
const fragment = run(
|
|
"ssh",
|
|
sshArgs("systemctl", "show", strictServiceUnit, "--property=FragmentPath", "--value")
|
|
).trim();
|
|
const serviceUnitSha256 = run(
|
|
"ssh",
|
|
sshArgs("sha256sum", fragment)
|
|
)
|
|
.trim()
|
|
.split(/\s+/)[0];
|
|
return {
|
|
system_generation: systemGeneration,
|
|
hosted_service_executable: executable,
|
|
hosted_service_sha256: `sha256:${binarySha256}`,
|
|
service_unit: fragment,
|
|
service_unit_sha256: `sha256:${serviceUnitSha256}`,
|
|
};
|
|
}
|
|
|
|
function configurationProvenance() {
|
|
const configRepo = path.resolve(
|
|
process.env.CLUSTERFLUX_DEPLOYMENT_CONFIG_REPO ||
|
|
path.join(repo, "..", "michelpaulissen.com")
|
|
);
|
|
const status = run("git", ["status", "--short"], { cwd: configRepo }).trim();
|
|
return {
|
|
repository: configRepo,
|
|
revision: run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(),
|
|
clean: status === "",
|
|
status: status || null,
|
|
};
|
|
}
|
|
|
|
async function restartHostedServiceAndResume({
|
|
clusterflux,
|
|
clusterfluxNode,
|
|
projectDir,
|
|
scope,
|
|
workerArgs,
|
|
workerNode,
|
|
credentialPath,
|
|
credentialDigest,
|
|
}) {
|
|
if (!strictVpsRestart) {
|
|
return {
|
|
executed: false,
|
|
reason: "strict VPS restart access not supplied",
|
|
worker: null,
|
|
};
|
|
}
|
|
const before = deploymentProvenance();
|
|
run("ssh", sshArgs("systemctl", "restart", strictServiceUnit));
|
|
const deadline = Date.now() + 120000;
|
|
let ping;
|
|
let lastError;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
ping = await sendHostedControl({ type: "ping" });
|
|
if (ping.type === "pong") break;
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
await delay(500);
|
|
}
|
|
assert.strictEqual(
|
|
ping?.type,
|
|
"pong",
|
|
`hosted service did not recover after restart: ${lastError?.message || "no pong"}`
|
|
);
|
|
const afterFirstRestart = deploymentProvenance();
|
|
assert.strictEqual(afterFirstRestart.system_generation, before.system_generation);
|
|
assert.strictEqual(
|
|
afterFirstRestart.hosted_service_sha256,
|
|
before.hosted_service_sha256
|
|
);
|
|
|
|
const projects = runJson(clusterflux, ["project", "list", ...scope], {
|
|
cwd: projectDir,
|
|
});
|
|
assert(projects.project_count >= 1, "CLI session/project did not survive restart");
|
|
const ephemeralRun = runJson(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(ephemeralRun.status, "main_launched");
|
|
const ephemeralProcess = ephemeralRun.process;
|
|
await waitForCli(
|
|
"pre-restart ephemeral main to park",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", ephemeralProcess],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.live_process?.main_wait_state === "waiting_for_node",
|
|
120000
|
|
);
|
|
|
|
run("ssh", sshArgs("systemctl", "restart", strictServiceUnit));
|
|
const secondDeadline = Date.now() + 120000;
|
|
ping = undefined;
|
|
lastError = undefined;
|
|
while (Date.now() < secondDeadline) {
|
|
try {
|
|
ping = await sendHostedControl({ type: "ping" });
|
|
if (ping.type === "pong") break;
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
await delay(500);
|
|
}
|
|
assert.strictEqual(
|
|
ping?.type,
|
|
"pong",
|
|
`hosted service did not recover after ephemeral-state restart: ${
|
|
lastError?.message || "no pong"
|
|
}`
|
|
);
|
|
const after = deploymentProvenance();
|
|
assert.strictEqual(after.system_generation, before.system_generation);
|
|
assert.strictEqual(after.hosted_service_sha256, before.hosted_service_sha256);
|
|
const oldStatus = runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", ephemeralProcess],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(oldStatus.state, "not_active");
|
|
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
|
|
|
|
const worker = spawnJsonLines(clusterfluxNode, workerArgs, {
|
|
cwd: projectDir,
|
|
env: { ...process.env },
|
|
});
|
|
const ready = await worker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"persisted worker ready after hosted service restart"
|
|
);
|
|
assert.strictEqual(ready.node, workerNode);
|
|
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
|
|
|
|
const restartedRun = runJson(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(restartedRun.status, "main_launched");
|
|
const restartedProcess = restartedRun.process;
|
|
const completed = await waitForCli(
|
|
"new process completion after hosted service restart",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", restartedProcess],
|
|
{ cwd: projectDir }
|
|
),
|
|
(tasks) => completedFlagship(rawTaskEvents(tasks)),
|
|
5 * 60 * 1000
|
|
);
|
|
const completedEvent = rawTaskEvents(completed).find(
|
|
(event) =>
|
|
event.executor === "coordinator_main" &&
|
|
event.terminal_state === "completed"
|
|
);
|
|
assert(completedEvent, "post-restart flagship omitted its completed main event");
|
|
return {
|
|
executed: true,
|
|
account_project_session_preserved: true,
|
|
node_identity_preserved: true,
|
|
old_process_terminated_honestly: true,
|
|
terminated_ephemeral_process: ephemeralProcess,
|
|
new_process: restartedProcess,
|
|
new_process_result: completedEvent.result,
|
|
before,
|
|
after_first_restart: afterFirstRestart,
|
|
after,
|
|
worker,
|
|
};
|
|
}
|
|
|
|
async function finishUserCredentialSecurity({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
sessionSecret,
|
|
}) {
|
|
const forged = assertDenied(
|
|
await sendHostedControl(
|
|
authenticatedRequest(
|
|
`forged-session-${crypto.randomBytes(24).toString("hex")}`,
|
|
{ type: "list_projects" }
|
|
)
|
|
),
|
|
/not recognized|invalid|authentication|session/i,
|
|
"forged user session"
|
|
);
|
|
|
|
let expired = null;
|
|
const expiredFile = process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE;
|
|
if (expiredFile) {
|
|
const oldSession = readJson(path.resolve(expiredFile));
|
|
assert(oldSession.session_secret, "expired session evidence omitted its secret");
|
|
expired = assertDenied(
|
|
await sendHostedControl(
|
|
authenticatedRequest(oldSession.session_secret, { type: "list_projects" })
|
|
),
|
|
/expired|not recognized/i,
|
|
"expired user session"
|
|
);
|
|
} else if (strictFullRelease) {
|
|
throw new Error(
|
|
"strict full release requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE for a genuinely expired hosted session"
|
|
);
|
|
}
|
|
|
|
const logout = runJson(clusterflux, ["logout", ...scope, "--yes"], {
|
|
cwd: projectDir,
|
|
});
|
|
assert.strictEqual(logout.server_session_revocation.revoked, true);
|
|
const revoked = assertDenied(
|
|
await sendHostedControl(
|
|
authenticatedRequest(sessionSecret, { type: "list_projects" })
|
|
),
|
|
/revoked|not recognized/i,
|
|
"revoked user session"
|
|
);
|
|
return { forged, expired, revoked };
|
|
}
|
|
|
|
async function dapVariables(client, variablesReference) {
|
|
const request = client.send("variables", { variablesReference });
|
|
return (await client.response(request, "variables")).body.variables;
|
|
}
|
|
|
|
async function runSameDefinitionDapIdentity({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
}) {
|
|
const scenarioStartedAt = Date.now();
|
|
const baselineContainers = new Set(
|
|
run("podman", ["ps", "-q"])
|
|
.trim()
|
|
.split(/\s+/)
|
|
.filter(Boolean)
|
|
);
|
|
const client = new DapClient({
|
|
cwd: projectDir,
|
|
command: clusterfluxDap,
|
|
args: [],
|
|
env: { ...process.env },
|
|
});
|
|
let disconnected = false;
|
|
try {
|
|
const initialize = client.send("initialize", {
|
|
adapterID: "clusterflux",
|
|
linesStartAt1: true,
|
|
columnsStartAt1: true,
|
|
});
|
|
await client.response(initialize, "initialize");
|
|
const launch = client.send("launch", {
|
|
entry: "identity",
|
|
project: projectDir,
|
|
runtimeBackend: "live-services",
|
|
coordinatorEndpoint: serviceEndpoint,
|
|
});
|
|
await client.response(launch, "launch");
|
|
await client.waitFor(
|
|
(message) => message.type === "event" && message.event === "initialized"
|
|
);
|
|
const configurationDone = client.send("configurationDone");
|
|
await client.response(configurationDone, "configurationDone");
|
|
|
|
const mainThreadDeadline = Date.now() + 5 * 60 * 1000;
|
|
let mainThread;
|
|
while (Date.now() < mainThreadDeadline) {
|
|
const threadsRequest = client.send("threads");
|
|
const threads = (await client.response(threadsRequest, "threads")).body
|
|
.threads;
|
|
mainThread = threads.find((thread) => /coordinator main/i.test(thread.name));
|
|
if (mainThread) break;
|
|
await delay(100);
|
|
}
|
|
assert(mainThread, "identity entry did not report its coordinator main");
|
|
const processRequest = client.send("evaluate", {
|
|
expression: "virtual_process_id",
|
|
context: "watch",
|
|
});
|
|
const processId = (await client.response(processRequest, "evaluate")).body
|
|
.result;
|
|
const identityContainerDeadline = Date.now() + 2 * 60 * 1000;
|
|
let identityContainerIds = new Set();
|
|
let maximumNewContainerCount = 0;
|
|
let lastPodmanStates = [];
|
|
while (Date.now() < identityContainerDeadline) {
|
|
const containers = runJson("podman", ["ps", "--format", "json"]);
|
|
lastPodmanStates = containers;
|
|
identityContainerIds = new Set(
|
|
containers
|
|
.map((container) => container.Id || container.ID || container.IdHex || "")
|
|
.filter((id) => id && !baselineContainers.has(id))
|
|
);
|
|
maximumNewContainerCount = Math.max(
|
|
maximumNewContainerCount,
|
|
identityContainerIds.size
|
|
);
|
|
if (identityContainerIds.size >= 2) break;
|
|
await delay(100);
|
|
}
|
|
const identityTaskSnapshot =
|
|
identityContainerIds.size >= 2
|
|
? null
|
|
: runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert(
|
|
identityContainerIds.size >= 2,
|
|
`identity entry did not start two same-definition Podman containers; max=${maximumNewContainerCount} tasks=${JSON.stringify(
|
|
identityTaskSnapshot
|
|
)} podman=${JSON.stringify(lastPodmanStates)}`
|
|
);
|
|
|
|
const pause = client.send("pause", { threadId: mainThread.id });
|
|
await client.response(pause, "pause");
|
|
const paused = await client.waitFor(
|
|
(message) =>
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "pause",
|
|
30_000
|
|
);
|
|
assert.strictEqual(typeof paused.body.allThreadsStopped, "boolean");
|
|
|
|
const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]);
|
|
const clusterfluxPausedContainers = podmanStates.filter((container) => {
|
|
const id = container.Id || container.ID || container.IdHex || "";
|
|
const state = String(container.State || container.Status || "").toLowerCase();
|
|
return identityContainerIds.has(id) && state.includes("paused");
|
|
});
|
|
assert(
|
|
clusterfluxPausedContainers.length >= 2,
|
|
`expected two newly paused Clusterflux containers: ${JSON.stringify(
|
|
podmanStates
|
|
)}`
|
|
);
|
|
|
|
const threadsRequest = client.send("threads");
|
|
const threads = (await client.response(threadsRequest, "threads")).body
|
|
.threads;
|
|
const identityThreads = threads.filter((thread) =>
|
|
/identity probe/i.test(thread.name)
|
|
);
|
|
assert.strictEqual(
|
|
identityThreads.length,
|
|
2,
|
|
`expected two same-definition DAP threads: ${JSON.stringify(threads)}`
|
|
);
|
|
assert.notStrictEqual(identityThreads[0].id, identityThreads[1].id);
|
|
|
|
const threadEvidence = [];
|
|
for (const threadRecord of identityThreads) {
|
|
const stackRequest = client.send("stackTrace", {
|
|
threadId: threadRecord.id,
|
|
startFrame: 0,
|
|
levels: 1,
|
|
});
|
|
const frames = (await client.response(stackRequest, "stackTrace")).body
|
|
.stackFrames;
|
|
assert.strictEqual(frames.length, 1);
|
|
const scopesRequest = client.send("scopes", { frameId: frames[0].id });
|
|
const scopes = (await client.response(scopesRequest, "scopes")).body.scopes;
|
|
const argsScope = scopes.find(
|
|
(scopeRecord) => scopeRecord.name === "Task Args and Handles"
|
|
);
|
|
const runtimeScope = scopes.find(
|
|
(scopeRecord) => scopeRecord.name === "Clusterflux Runtime"
|
|
);
|
|
assert(argsScope && runtimeScope);
|
|
const args = await dapVariables(client, argsScope.variablesReference);
|
|
const runtime = await dapVariables(client, runtimeScope.variablesReference);
|
|
const runtimeValue = (name) =>
|
|
runtime.find((variable) => variable.name === name)?.value;
|
|
assert.strictEqual(runtimeValue("state"), "Frozen");
|
|
threadEvidence.push({
|
|
thread_id: threadRecord.id,
|
|
task_instance: runtimeValue("virtual_thread"),
|
|
attempt_id: runtimeValue("task_attempt_id"),
|
|
arguments: args.map((variable) => ({
|
|
name: variable.name,
|
|
value: variable.value,
|
|
})),
|
|
});
|
|
}
|
|
assert.notStrictEqual(
|
|
threadEvidence[0].task_instance,
|
|
threadEvidence[1].task_instance
|
|
);
|
|
assert.notStrictEqual(threadEvidence[0].attempt_id, threadEvidence[1].attempt_id);
|
|
const argumentText = threadEvidence
|
|
.flatMap((record) => record.arguments.map((argument) => argument.value))
|
|
.join(" ");
|
|
assert.match(argumentText, /slow-first/);
|
|
assert.match(argumentText, /fast-second/);
|
|
|
|
const continued = client.send("continue", {
|
|
threadId: paused.body.threadId,
|
|
});
|
|
await client.response(continued, "continue");
|
|
await client.waitFor(
|
|
(message) => message.type === "event" && message.event === "terminated",
|
|
5 * 60 * 1000
|
|
);
|
|
|
|
const taskList = runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
const identityEvents = rawTaskEvents(taskList).filter(
|
|
(event) =>
|
|
event.task_definition === "identity_probe" &&
|
|
event.terminal_state === "completed"
|
|
);
|
|
assert.strictEqual(identityEvents.length, 2, JSON.stringify(identityEvents));
|
|
assert.notStrictEqual(identityEvents[0].task, identityEvents[1].task);
|
|
assert.notStrictEqual(identityEvents[0].attempt_id, identityEvents[1].attempt_id);
|
|
assert.match(JSON.stringify(identityEvents[0].result), /fast-second/);
|
|
assert.match(JSON.stringify(identityEvents[1].result), /slow-first/);
|
|
|
|
const released = await waitForCli(
|
|
"same-definition identity process release",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
120_000
|
|
);
|
|
await client.close();
|
|
disconnected = true;
|
|
return {
|
|
duration_ms: Date.now() - scenarioStartedAt,
|
|
process: processId,
|
|
thread_evidence: threadEvidence,
|
|
completion_order: identityEvents.map((event) => ({
|
|
task_instance: event.task,
|
|
attempt_id: event.attempt_id,
|
|
result: event.result,
|
|
})),
|
|
podman_paused_container_ids: clusterfluxPausedContainers.map(
|
|
(container) => container.Id || container.ID || container.IdHex
|
|
),
|
|
dap_all_threads_stopped: paused.body.allThreadsStopped,
|
|
terminal_state: released.state,
|
|
};
|
|
} finally {
|
|
if (!disconnected) {
|
|
await client.close().catch(() => {
|
|
if (client.child.exitCode === null) client.child.kill("SIGKILL");
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runLiveDapEditRestart({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
worker,
|
|
}) {
|
|
const scenarioStartedAt = Date.now();
|
|
const sourcePath = fs.realpathSync(path.join(projectDir, "src/build.rs"));
|
|
const originalSource = fs.readFileSync(sourcePath, "utf8");
|
|
const sourceLines = originalSource.split(/\r?\n/);
|
|
const lineFor = (needle) => {
|
|
const line = sourceLines.findIndex((candidate) => candidate.includes(needle)) + 1;
|
|
assert(line > 0, `flagship source omitted ${needle}`);
|
|
return line;
|
|
};
|
|
const mainLine = lineFor("pub async fn restart_main()");
|
|
const taskLine = lineFor("fn task_trap(");
|
|
const client = new DapClient({
|
|
cwd: projectDir,
|
|
command: clusterfluxDap,
|
|
args: [],
|
|
env: { ...process.env },
|
|
});
|
|
let sourceRestored = false;
|
|
let disconnected = false;
|
|
try {
|
|
const initialize = client.send("initialize", {
|
|
adapterID: "clusterflux",
|
|
linesStartAt1: true,
|
|
columnsStartAt1: true,
|
|
});
|
|
await client.response(initialize, "initialize");
|
|
|
|
const launch = client.send("launch", {
|
|
entry: "restart",
|
|
project: projectDir,
|
|
runtimeBackend: "live-services",
|
|
coordinatorEndpoint: serviceEndpoint,
|
|
});
|
|
await client.response(launch, "launch");
|
|
await client.waitFor(
|
|
(message) => message.type === "event" && message.event === "initialized"
|
|
);
|
|
|
|
const configurationStartedAt = Date.now();
|
|
const configurationDone = client.send("configurationDone");
|
|
await client.response(configurationDone, "configurationDone");
|
|
const configurationResponseMs = Date.now() - configurationStartedAt;
|
|
assert(
|
|
configurationResponseMs < 2_000,
|
|
`configurationDone blocked for ${configurationResponseMs} ms`
|
|
);
|
|
|
|
const threadDeadline = Date.now() + 5 * 60 * 1000;
|
|
let runningThreads = [];
|
|
while (Date.now() < threadDeadline) {
|
|
const threadsRequest = client.send("threads");
|
|
runningThreads = (await client.response(threadsRequest, "threads")).body
|
|
.threads;
|
|
if (runningThreads.length > 0) break;
|
|
await delay(100);
|
|
}
|
|
assert(runningThreads.length > 0, "asynchronous live DAP launch reported no threads");
|
|
|
|
const pauseStartedAt = Date.now();
|
|
const pause = client.send("pause", { threadId: runningThreads[0].id });
|
|
await client.response(pause, "pause");
|
|
const pauseResponseMs = Date.now() - pauseStartedAt;
|
|
assert(pauseResponseMs < 2_000, `pause blocked for ${pauseResponseMs} ms`);
|
|
const mainStop = await client.waitFor(
|
|
(message) =>
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "pause"
|
|
);
|
|
assert.strictEqual(mainStop.body.allThreadsStopped, true);
|
|
|
|
const inspectStartedAt = Date.now();
|
|
const commandStatus = client.send("evaluate", {
|
|
expression: "command_status",
|
|
context: "watch",
|
|
});
|
|
const inspected = await client.response(commandStatus, "evaluate");
|
|
const inspectionResponseMs = Date.now() - inspectStartedAt;
|
|
assert(
|
|
inspectionResponseMs < 2_000,
|
|
`inspection blocked for ${inspectionResponseMs} ms`
|
|
);
|
|
assert.match(inspected.body.result, /debug epoch|frozen/i);
|
|
|
|
const breakpointStartedAt = Date.now();
|
|
const setBreakpoints = client.send("setBreakpoints", {
|
|
source: { path: sourcePath },
|
|
breakpoints: [{ line: taskLine }],
|
|
});
|
|
const breakpointResponse = await client.response(setBreakpoints, "setBreakpoints");
|
|
const breakpointResponseMs = Date.now() - breakpointStartedAt;
|
|
assert(
|
|
breakpointResponseMs < 2_000,
|
|
`setBreakpoints blocked for ${breakpointResponseMs} ms`
|
|
);
|
|
assert.deepStrictEqual(
|
|
breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified),
|
|
[true]
|
|
);
|
|
|
|
const continueStartedAt = Date.now();
|
|
const mainContinue = client.send("continue", {
|
|
threadId: mainStop.body.threadId,
|
|
});
|
|
await client.response(mainContinue, "continue");
|
|
const continueResponseMs = Date.now() - continueStartedAt;
|
|
assert(
|
|
continueResponseMs < 2_000,
|
|
`continue blocked for ${continueResponseMs} ms`
|
|
);
|
|
const taskStop = await client.waitFor(
|
|
(message) =>
|
|
message.seq > mainStop.seq &&
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "breakpoint"
|
|
);
|
|
assert.strictEqual(taskStop.body.allThreadsStopped, true);
|
|
assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId);
|
|
|
|
const stackTrace = client.send("stackTrace", {
|
|
threadId: taskStop.body.threadId,
|
|
startFrame: 0,
|
|
levels: 1,
|
|
});
|
|
const taskFrames = (await client.response(stackTrace, "stackTrace")).body
|
|
.stackFrames;
|
|
assert.strictEqual(taskFrames[0].line, taskLine);
|
|
|
|
const scopesRequest = client.send("scopes", { frameId: taskFrames[0].id });
|
|
const scopes = (await client.response(scopesRequest, "scopes")).body.scopes;
|
|
const argsScope = scopes.find(
|
|
(candidate) => candidate.name === "Task Args and Handles"
|
|
);
|
|
const runtimeScope = scopes.find(
|
|
(candidate) => candidate.name === "Clusterflux Runtime"
|
|
);
|
|
assert(argsScope && runtimeScope);
|
|
const taskArgs = await dapVariables(client, argsScope.variablesReference);
|
|
assert(
|
|
taskArgs.some(
|
|
(variable) =>
|
|
variable.name === "arg_0" && String(variable.value).includes("0")
|
|
)
|
|
);
|
|
const runtime = await dapVariables(client, runtimeScope.variablesReference);
|
|
assert(
|
|
runtime.some(
|
|
(variable) =>
|
|
variable.name === "runtime_backend" && variable.value === "LiveServices"
|
|
)
|
|
);
|
|
assert(
|
|
runtime.some(
|
|
(variable) => variable.name === "state" && variable.value === "Frozen"
|
|
)
|
|
);
|
|
const dapProcess = runtime.find(
|
|
(variable) => variable.name === "virtual_process_id"
|
|
)?.value;
|
|
assert.match(dapProcess || "", /^vp-[0-9a-f]{12}$/);
|
|
|
|
const taskContinue = client.send("continue", { threadId: taskStop.body.threadId });
|
|
await client.response(taskContinue, "continue");
|
|
const failedStop = await client.waitFor(
|
|
(message) =>
|
|
message.seq > taskStop.seq &&
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "exception",
|
|
5 * 60 * 1000
|
|
);
|
|
assert.strictEqual(failedStop.body.allThreadsStopped, false);
|
|
|
|
const beforeEdit = runJson(
|
|
clusterflux,
|
|
["task", "list", ...scope, "--process", dapProcess],
|
|
{ cwd: projectDir }
|
|
);
|
|
const originalTaskEvent = rawTaskEvents(beforeEdit).find(
|
|
(event) =>
|
|
event.task_definition === "task_trap" &&
|
|
event.terminal_state === "failed"
|
|
);
|
|
assert(originalTaskEvent, "DAP restart entry omitted its original child event");
|
|
assert(originalTaskEvent.attempt_id, "failed attempt omitted its identity");
|
|
|
|
const originalTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 {
|
|
#[cfg(target_arch = "wasm32")]
|
|
core::arch::wasm32::unreachable();
|
|
#[cfg(not(target_arch = "wasm32"))]
|
|
panic!("intentional task trap")
|
|
}`;
|
|
const replacementTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 {
|
|
_input + 42 // strict hosted compatible restart probe
|
|
}`;
|
|
const editedSource = originalSource.replace(originalTaskBody, replacementTaskBody);
|
|
assert.notStrictEqual(editedSource, originalSource);
|
|
fs.writeFileSync(sourcePath, editedSource);
|
|
|
|
const restartFrame = client.send("restartFrame", {
|
|
frameId: taskFrames[0].id,
|
|
});
|
|
const restartResponse = await client.response(restartFrame, "restartFrame");
|
|
const restartedStop = await client.waitFor(
|
|
(message) =>
|
|
message.seq > restartResponse.seq &&
|
|
message.type === "event" &&
|
|
message.event === "stopped" &&
|
|
message.body.reason === "breakpoint"
|
|
);
|
|
assert.strictEqual(restartedStop.body.allThreadsStopped, true);
|
|
const restartedContinue = client.send("continue", {
|
|
threadId: restartedStop.body.threadId,
|
|
});
|
|
await client.response(restartedContinue, "continue");
|
|
const restartedNodeReport = await worker.waitFor(
|
|
(value) =>
|
|
value.node_status === "completed" &&
|
|
value.virtual_thread === originalTaskEvent.task &&
|
|
value.task_assignment_response?.task_spec?.task_definition ===
|
|
"task_trap",
|
|
"edited task to execute on the live restarted node",
|
|
5 * 60 * 1000
|
|
);
|
|
assert.strictEqual(
|
|
restartedNodeReport.node_status,
|
|
"completed",
|
|
`edited task restart failed on the live node: ${JSON.stringify(restartedNodeReport)}`
|
|
);
|
|
assert.match(restartedNodeReport.stdout_tail, /"SmallJson":42/);
|
|
const restartedTask = restartedNodeReport.virtual_thread;
|
|
|
|
const afterEdit = await waitForCli(
|
|
"edited live task event",
|
|
() =>
|
|
runJson(clusterflux, ["task", "list", ...scope, "--process", dapProcess], {
|
|
cwd: projectDir,
|
|
}),
|
|
(tasks) =>
|
|
rawTaskEvents(tasks).some(
|
|
(event) =>
|
|
event.task === restartedTask &&
|
|
event.terminal_state === "completed" &&
|
|
JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 })
|
|
),
|
|
5 * 60 * 1000
|
|
);
|
|
const restartedEvent = rawTaskEvents(afterEdit).find(
|
|
(event) =>
|
|
event.task === restartedTask &&
|
|
event.terminal_state === "completed" &&
|
|
JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 })
|
|
);
|
|
assert(restartedEvent, "replacement attempt omitted its completed event");
|
|
assert(restartedEvent.attempt_id, "replacement attempt omitted its identity");
|
|
assert.notStrictEqual(restartedEvent.attempt_id, originalTaskEvent.attempt_id);
|
|
await client.waitFor(
|
|
(message) => message.type === "event" && message.event === "terminated",
|
|
5 * 60 * 1000
|
|
);
|
|
const disconnectStartedAt = Date.now();
|
|
await client.close();
|
|
disconnected = true;
|
|
const disconnectResponseMs = Date.now() - disconnectStartedAt;
|
|
assert(
|
|
disconnectResponseMs < 2_000,
|
|
`disconnect blocked for ${disconnectResponseMs} ms`
|
|
);
|
|
fs.writeFileSync(sourcePath, originalSource);
|
|
sourceRestored = true;
|
|
|
|
return {
|
|
duration_ms: Date.now() - scenarioStartedAt,
|
|
process: dapProcess,
|
|
main_source_line: mainLine,
|
|
task_breakpoint_line: taskLine,
|
|
launch_started_without_breakpoint: true,
|
|
configuration_response_ms: configurationResponseMs,
|
|
pause_response_ms: pauseResponseMs,
|
|
inspection_response_ms: inspectionResponseMs,
|
|
breakpoint_response_ms: breakpointResponseMs,
|
|
continue_response_ms: continueResponseMs,
|
|
disconnect_response_ms: disconnectResponseMs,
|
|
pause_all_threads_stopped: mainStop.body.allThreadsStopped,
|
|
task_all_threads_stopped: taskStop.body.allThreadsStopped,
|
|
original_task_instance: originalTaskEvent.task,
|
|
original_attempt: originalTaskEvent.attempt_id,
|
|
restarted_task_instance: restartedTask,
|
|
restarted_attempt: restartedEvent.attempt_id,
|
|
restarted_result: restartedEvent.result,
|
|
compatible_source_edit: "task_trap: trap -> input + 42",
|
|
original_arguments_preserved: true,
|
|
clean_vfs_boundary_used: true,
|
|
};
|
|
} finally {
|
|
if (!sourceRestored) fs.writeFileSync(sourcePath, originalSource);
|
|
if (!disconnected) {
|
|
await client.close().catch(() => {
|
|
if (client.child.exitCode === null) client.child.kill("SIGKILL");
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
requireEnabled();
|
|
const manifest = readJson(manifestPath);
|
|
assert.strictEqual(manifest.kind, "clusterflux-public-release");
|
|
assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint);
|
|
assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443");
|
|
|
|
const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-happy-"));
|
|
const installDir = path.join(workRoot, "install");
|
|
const checkout = path.join(workRoot, "public-repo");
|
|
const downloadPath = path.join(workRoot, "release.tar");
|
|
const extractedArtifact = path.join(workRoot, "artifact");
|
|
ensureDir(installDir);
|
|
run("tar", ["-xzf", binaryArchive(manifest), "-C", installDir]);
|
|
const publicSource = stagePublicCheckout(manifest, checkout);
|
|
|
|
const clusterflux = executable(installDir, "clusterflux");
|
|
const clusterfluxNode = executable(installDir, "clusterflux-node");
|
|
const clusterfluxDap = executable(installDir, "clusterflux-debug-dap");
|
|
const projectDir = path.join(checkout, "examples/launch-build-demo");
|
|
const suffix = String(Date.now());
|
|
const workerNode = `worker-${suffix}`;
|
|
|
|
let loginSession;
|
|
let receivedDefaultProject;
|
|
if (reuseSessionFile) {
|
|
const sourceSessionPath = path.resolve(reuseSessionFile);
|
|
const sourceProjectPath = path.join(path.dirname(sourceSessionPath), "project.json");
|
|
const sourceSession = readJson(sourceSessionPath);
|
|
const sourceProject = readJson(sourceProjectPath);
|
|
assert.strictEqual(sourceSession.kind, "human");
|
|
assert.strictEqual(sourceSession.coordinator, serviceEndpoint);
|
|
assert.strictEqual(sourceProject.coordinator, serviceEndpoint);
|
|
assert.strictEqual(sourceProject.tenant, sourceSession.tenant);
|
|
assert.strictEqual(sourceProject.project, sourceSession.project);
|
|
assert.strictEqual(sourceProject.user, sourceSession.user);
|
|
assert(
|
|
Number(sourceSession.expires_at) > Math.floor(Date.now() / 1000) + 300,
|
|
"reused CLI session expires too soon for the live journey"
|
|
);
|
|
const destinationConfig = path.join(projectDir, ".clusterflux");
|
|
ensureDir(destinationConfig);
|
|
fs.copyFileSync(sourceSessionPath, path.join(destinationConfig, "session.json"));
|
|
fs.copyFileSync(sourceProjectPath, path.join(destinationConfig, "project.json"));
|
|
fs.chmodSync(path.join(destinationConfig, "session.json"), 0o600);
|
|
fs.chmodSync(path.join(destinationConfig, "project.json"), 0o644);
|
|
loginSession = sourceSession;
|
|
receivedDefaultProject = true;
|
|
} else {
|
|
const login = runJson(
|
|
clusterflux,
|
|
["login", "--browser", "--json"],
|
|
{
|
|
cwd: projectDir,
|
|
env: { ...process.env, CLUSTERFLUX_BROWSER_OPEN_COMMAND: browserOpenCommand },
|
|
}
|
|
);
|
|
assert.strictEqual(login.plan.coordinator, serviceEndpoint);
|
|
assert.strictEqual(login.boundary.scoped_cli_session_received, true);
|
|
assert.strictEqual(login.boundary.provider_tokens_exposed_to_cli, false);
|
|
assert.strictEqual(login.boundary.provider_tokens_sent_to_nodes, false);
|
|
loginSession = login.coordinator_response.session;
|
|
assert(loginSession, "hosted browser login omitted its scoped session");
|
|
receivedDefaultProject =
|
|
typeof loginSession.project === "string" && loginSession.project.length > 0;
|
|
assert(receivedDefaultProject, "login did not return its server-owned project");
|
|
}
|
|
const sessionSecret =
|
|
loginSession.cli_session_secret || loginSession.session_secret;
|
|
assert(sessionSecret, "hosted browser login omitted the CLI session credential");
|
|
const tenant = loginSession.tenant;
|
|
const project = loginSession.project;
|
|
const user = loginSession.user;
|
|
for (const [name, value] of Object.entries({ tenant, project, user })) {
|
|
assert.strictEqual(typeof value, "string", `hosted session omitted ${name}`);
|
|
assert.notStrictEqual(value, "", `hosted session returned an empty ${name}`);
|
|
}
|
|
const scope = [
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--user",
|
|
user,
|
|
"--json",
|
|
];
|
|
|
|
if (!reuseSessionFile) {
|
|
const projectInit = runJson(
|
|
clusterflux,
|
|
[
|
|
"project",
|
|
"init",
|
|
...scope,
|
|
"--new-project",
|
|
project,
|
|
"--name",
|
|
"CLI Happy Path",
|
|
"--yes",
|
|
],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(projectInit.command, "project init");
|
|
assert.strictEqual(projectInit.project_config_written, true);
|
|
assert.strictEqual(projectInit.coordinator_response.type, "project_created");
|
|
}
|
|
|
|
const projectList = runJson(clusterflux, ["project", "list", ...scope], {
|
|
cwd: projectDir,
|
|
});
|
|
assert(projectList.project_count >= 1);
|
|
const projectSelect = runJson(
|
|
clusterflux,
|
|
["project", "select", ...scope, project],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(projectSelect.command, "project select");
|
|
|
|
const inspection = runJson(clusterflux, ["inspect", "--project", ".", "--json"], {
|
|
cwd: projectDir,
|
|
});
|
|
assert(
|
|
inspection.metadata.environments.some((environment) => environment.name === "linux")
|
|
);
|
|
assert(Array.isArray(inspection.source_provider_statuses));
|
|
assert(inspection.source_provider_manifest);
|
|
|
|
const runReport = runJson(
|
|
clusterflux,
|
|
["run", "build", "--project", ".", "--json"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(runReport.command, "run");
|
|
assert.strictEqual(runReport.status, "main_launched");
|
|
assert.strictEqual(runReport.task_launch.type, "main_launched");
|
|
const processId = runReport.process;
|
|
|
|
const parkedStatus = await waitForCli(
|
|
"hosted coordinator main to park before node enrollment",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) =>
|
|
status.live_process?.main_state === "running" &&
|
|
status.live_process?.main_wait_state === "waiting_for_node" &&
|
|
status.live_process?.connected_nodes?.length === 0,
|
|
120000
|
|
);
|
|
|
|
const grant = runJson(clusterflux, ["node", "enroll", ...scope], {
|
|
cwd: projectDir,
|
|
});
|
|
assert.strictEqual(grant.command, "node enroll");
|
|
const attach = runJson(
|
|
clusterflux,
|
|
[
|
|
"node",
|
|
"attach",
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--node",
|
|
workerNode,
|
|
"--enrollment-grant",
|
|
grant.enrollment_grant.grant,
|
|
"--json",
|
|
],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(attach.command, "node attach");
|
|
assert.strictEqual(attach.boundary.used_enrollment_exchange, true);
|
|
|
|
const credentialDir = path.join(projectDir, ".clusterflux", "nodes");
|
|
const credentialFiles = fs
|
|
.readdirSync(credentialDir)
|
|
.filter((file) => file.endsWith(".json"));
|
|
assert.strictEqual(credentialFiles.length, 1, "expected one persisted node credential");
|
|
const credentialPath = path.join(credentialDir, credentialFiles[0]);
|
|
const credentialBefore = fs.readFileSync(credentialPath);
|
|
const credentialDigest = sha256(credentialBefore);
|
|
if (process.platform !== "win32") {
|
|
assert.strictEqual(fs.statSync(credentialPath).mode & 0o777, 0o600);
|
|
}
|
|
|
|
const workerArgs = [
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--node",
|
|
workerNode,
|
|
"--worker",
|
|
"--project-root",
|
|
projectDir,
|
|
"--assignment-poll-ms",
|
|
"500",
|
|
"--emit-ready",
|
|
];
|
|
const spawnWorker = () =>
|
|
spawnJsonLines(clusterfluxNode, workerArgs, {
|
|
cwd: projectDir,
|
|
env: { ...process.env },
|
|
});
|
|
|
|
let worker = spawnWorker();
|
|
try {
|
|
const firstReady = await worker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"first persisted worker ready"
|
|
);
|
|
assert.strictEqual(firstReady.node, workerNode);
|
|
const initiallyOnline = runJson(
|
|
clusterflux,
|
|
["node", "status", ...scope, "--node", workerNode],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(nodeDescriptor(initiallyOnline, workerNode)?.online, true);
|
|
|
|
const completedTasks = await waitForCli(
|
|
"real hosted flagship completion",
|
|
() =>
|
|
runJson(clusterflux, ["task", "list", ...scope, "--process", processId], {
|
|
cwd: projectDir,
|
|
}),
|
|
(tasks) => completedFlagship(rawTaskEvents(tasks))
|
|
);
|
|
const firstEvents = rawTaskEvents(completedTasks);
|
|
assert(
|
|
firstEvents
|
|
.filter((event) => event.executor === "node")
|
|
.every((event) => event.node === workerNode)
|
|
);
|
|
assert(
|
|
firstEvents.some(
|
|
(event) =>
|
|
event.task_definition === "package_release" &&
|
|
event.terminal_state === "completed" &&
|
|
contentAddressedArtifactId(event, "release.tar")
|
|
)
|
|
);
|
|
|
|
const releasedFlagshipStatus = await waitForCli(
|
|
"completed flagship process to release its active slot",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
120000
|
|
);
|
|
|
|
const offlineStartedAt = Date.now();
|
|
await stopChild(worker.child);
|
|
const observedOffline = await waitForCli(
|
|
"server-derived worker stale/offline transition",
|
|
() =>
|
|
runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], {
|
|
cwd: projectDir,
|
|
}),
|
|
(status) => nodeDescriptor(status, workerNode)?.online === false,
|
|
120000
|
|
);
|
|
worker = spawnWorker();
|
|
const secondReady = await worker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"restarted persisted worker ready"
|
|
);
|
|
assert.strictEqual(secondReady.node, workerNode);
|
|
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
|
|
|
|
const nodeStatus = await waitForCli(
|
|
"server-derived worker online recovery",
|
|
() =>
|
|
runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], {
|
|
cwd: projectDir,
|
|
}),
|
|
(status) => nodeDescriptor(status, workerNode)?.online === true,
|
|
30000
|
|
);
|
|
assert(JSON.stringify(nodeStatus.response).includes(workerNode));
|
|
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
|
|
const nodeLivenessTransition = {
|
|
initial_online: nodeDescriptor(initiallyOnline, workerNode).online,
|
|
stale_offline: nodeDescriptor(observedOffline, workerNode).online,
|
|
recovered_online: nodeDescriptor(nodeStatus, workerNode).online,
|
|
offline_detection_ms: Date.now() - offlineStartedAt,
|
|
persisted_identity_reused: true,
|
|
};
|
|
|
|
const processStatus = runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(processStatus.command, "process status");
|
|
assert(processStatus.current_task_count >= firstEvents.length);
|
|
|
|
const logs = runJson(
|
|
clusterflux,
|
|
["logs", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert(logs.log_entries.length >= 4);
|
|
|
|
const artifacts = runJson(
|
|
clusterflux,
|
|
["artifact", "list", ...scope, "--process", processId],
|
|
{ cwd: projectDir }
|
|
);
|
|
const releaseArtifact = artifacts.artifacts.find((artifact) => {
|
|
const digest = artifact.digest;
|
|
return (
|
|
typeof digest === "string" &&
|
|
/^sha256:[0-9a-f]{64}$/.test(digest) &&
|
|
artifact.artifact === `release.tar-${digest.slice("sha256:".length)}`
|
|
);
|
|
});
|
|
assert(releaseArtifact, "hosted flagship did not publish release.tar");
|
|
assert.strictEqual(releaseArtifact.state, "metadata_flushed");
|
|
|
|
const download = runJson(
|
|
clusterflux,
|
|
[
|
|
"artifact",
|
|
"download",
|
|
...scope,
|
|
releaseArtifact.artifact,
|
|
"--to",
|
|
downloadPath,
|
|
],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(download.download_session.link_issued, true);
|
|
assert.strictEqual(download.local_download.local_bytes_written_by_cli, true);
|
|
assert.strictEqual(download.local_download.verified_digest, releaseArtifact.digest);
|
|
assert.strictEqual(sha256(fs.readFileSync(downloadPath)), releaseArtifact.digest);
|
|
|
|
ensureDir(extractedArtifact);
|
|
const archiveEntries = run("tar", ["-tf", downloadPath]);
|
|
assert.match(archiveEntries, /(?:^|\n)hello-clusterflux(?:\n|$)/);
|
|
run("tar", ["-xf", downloadPath, "-C", extractedArtifact]);
|
|
const builtExecutable = path.join(extractedArtifact, "hello-clusterflux");
|
|
assert(fs.existsSync(builtExecutable));
|
|
const builtOutput = run(builtExecutable, []).trim();
|
|
assert.strictEqual(builtOutput, "hello from a real Clusterflux build");
|
|
|
|
const dapEditRestart = await runLiveDapEditRestart({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
worker,
|
|
});
|
|
|
|
const restart = runJson(
|
|
clusterflux,
|
|
["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(restart.restart_request.accepted, true);
|
|
const cancel = runJson(
|
|
clusterflux,
|
|
["process", "cancel", ...scope, "--process", dapEditRestart.process, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
assert.strictEqual(cancel.cancel_request.accepted, true);
|
|
const releasedDap = await waitForCli(
|
|
"cancelled DAP process slot release",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["process", "status", ...scope, "--process", dapEditRestart.process],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => status.state === "not_active",
|
|
120000
|
|
);
|
|
assert.strictEqual(releasedDap.state, "not_active");
|
|
|
|
const identityWorkerNode = `identity-worker-${suffix}`;
|
|
const identityWorkerGrant = runJson(
|
|
clusterflux,
|
|
["node", "enroll", ...scope],
|
|
{ cwd: projectDir }
|
|
);
|
|
runJson(
|
|
clusterflux,
|
|
[
|
|
"node",
|
|
"attach",
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--node",
|
|
identityWorkerNode,
|
|
"--enrollment-grant",
|
|
identityWorkerGrant.enrollment_grant.grant,
|
|
"--json",
|
|
],
|
|
{ cwd: projectDir }
|
|
);
|
|
const identityWorker = spawnJsonLines(
|
|
clusterfluxNode,
|
|
[
|
|
"--coordinator",
|
|
serviceEndpoint,
|
|
"--tenant",
|
|
tenant,
|
|
"--project-id",
|
|
project,
|
|
"--node",
|
|
identityWorkerNode,
|
|
"--worker",
|
|
"--project-root",
|
|
projectDir,
|
|
"--assignment-poll-ms",
|
|
"500",
|
|
"--emit-ready",
|
|
],
|
|
{ cwd: projectDir, env: { ...process.env } }
|
|
);
|
|
let sameDefinitionIdentity;
|
|
try {
|
|
const identityWorkerReady = await identityWorker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"same-definition identity worker ready"
|
|
);
|
|
assert.strictEqual(identityWorkerReady.node, identityWorkerNode);
|
|
await waitForCli(
|
|
"same-definition identity worker online",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["node", "status", ...scope, "--node", identityWorkerNode],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => nodeDescriptor(status, identityWorkerNode)?.online === true,
|
|
30_000
|
|
);
|
|
await stopChild(worker.child);
|
|
worker = spawnWorker();
|
|
const refreshedPrimaryReady = await worker.waitFor(
|
|
(value) => value.node_status === "ready",
|
|
"same-definition primary worker capability refresh"
|
|
);
|
|
assert.strictEqual(refreshedPrimaryReady.node, workerNode);
|
|
const primaryStatus = await waitForCli(
|
|
"same-definition primary worker online after refresh",
|
|
() =>
|
|
runJson(
|
|
clusterflux,
|
|
["node", "status", ...scope, "--node", workerNode],
|
|
{ cwd: projectDir }
|
|
),
|
|
(status) => nodeDescriptor(status, workerNode)?.online === true,
|
|
30_000
|
|
);
|
|
const identityStatus = runJson(
|
|
clusterflux,
|
|
["node", "status", ...scope, "--node", identityWorkerNode],
|
|
{ cwd: projectDir }
|
|
);
|
|
const primarySnapshots =
|
|
nodeDescriptor(primaryStatus, workerNode)?.source_snapshots || [];
|
|
const identitySnapshots =
|
|
nodeDescriptor(identityStatus, identityWorkerNode)?.source_snapshots || [];
|
|
assert(
|
|
primarySnapshots.some((snapshot) => identitySnapshots.includes(snapshot)),
|
|
`same-definition workers do not share a current source snapshot: ${JSON.stringify({
|
|
primarySnapshots,
|
|
identitySnapshots,
|
|
})}`
|
|
);
|
|
sameDefinitionIdentity = await runSameDefinitionDapIdentity({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
});
|
|
} finally {
|
|
await stopChild(identityWorker.child);
|
|
runJson(
|
|
clusterflux,
|
|
["node", "revoke", ...scope, "--node", identityWorkerNode, "--yes"],
|
|
{ cwd: projectDir }
|
|
);
|
|
}
|
|
const repeatedParkWake = await runRepeatedParkWakeProof({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
});
|
|
const longJoin = await runLongJoinProof({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
});
|
|
|
|
const tenantIsolation = await runSecondTenantIsolation({
|
|
firstTenant: tenant,
|
|
firstProject: project,
|
|
firstProcess: dapEditRestart.process,
|
|
firstNode: workerNode,
|
|
firstArtifact: releaseArtifact.artifact,
|
|
manifest,
|
|
});
|
|
|
|
const securityNode = await prepareLiveNodeCredentialSecurity({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
tenant,
|
|
project,
|
|
suffix,
|
|
bundle: {
|
|
digest: runReport.bundle_digest,
|
|
moduleBase64: fs
|
|
.readFileSync(
|
|
path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module)
|
|
)
|
|
.toString("base64"),
|
|
entryExport: runReport.entry_export,
|
|
entryStableId: runReport.entry_stable_id,
|
|
},
|
|
});
|
|
const droppedConnectionRollback = await runDroppedConnectionRollback({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
});
|
|
const bandwidthPreallocation = await runLiveBandwidthPreallocation({
|
|
sessionSecret,
|
|
artifact: releaseArtifact.artifact,
|
|
maxBytes: download.local_download.bytes_written,
|
|
});
|
|
await stopChild(worker.child);
|
|
const relayEmergencyDisableResult = await runRelayEmergencyDisable({
|
|
clusterflux,
|
|
clusterfluxNode,
|
|
projectDir,
|
|
scope,
|
|
workerArgs,
|
|
sessionSecret,
|
|
maxBytes: download.local_download.bytes_written,
|
|
});
|
|
worker = relayEmergencyDisableResult.worker;
|
|
const relayEmergencyDisable = relayEmergencyDisableResult.evidence;
|
|
const agentCredentialSecurity = await runLiveAgentCredentialSecurity({
|
|
clusterfluxDap,
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
sessionSecret,
|
|
tenant,
|
|
project,
|
|
suffix,
|
|
bundle: {
|
|
digest: runReport.bundle_digest,
|
|
moduleBase64: fs
|
|
.readFileSync(
|
|
path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module)
|
|
)
|
|
.toString("base64"),
|
|
entryExport: runReport.entry_export,
|
|
entryStableId: runReport.entry_stable_id,
|
|
},
|
|
securityNode,
|
|
workerRuntime: {
|
|
node: workerNode,
|
|
child: worker.child,
|
|
worker,
|
|
identity: nodeIdentityFromPrivateKey(
|
|
readNodeCredential(projectDir, workerNode).credential.private_key
|
|
),
|
|
},
|
|
});
|
|
|
|
await stopChild(worker.child);
|
|
const liveSoak = await runLiveSoak({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
securityNode,
|
|
});
|
|
const revokedNodeCredential = await revokeSecurityNode({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
securityNode,
|
|
});
|
|
const quotaPreallocation = await runLiveQuotaPreallocation({
|
|
sessionSecret,
|
|
suffix,
|
|
});
|
|
const hostedLoginIsolation = await runHostedLoginIsolationBeforeRestart();
|
|
const serviceRestart = await restartHostedServiceAndResume({
|
|
clusterflux,
|
|
clusterfluxNode,
|
|
projectDir,
|
|
scope,
|
|
workerArgs,
|
|
workerNode,
|
|
credentialPath,
|
|
credentialDigest,
|
|
});
|
|
if (serviceRestart.worker) worker = serviceRestart.worker;
|
|
const relayAfterRestart = relayDurableState();
|
|
assert(
|
|
relayAfterRestart.ingress_used >=
|
|
bandwidthPreallocation.durable_after_release.ingress_used
|
|
);
|
|
assert(
|
|
relayAfterRestart.egress_used >=
|
|
bandwidthPreallocation.durable_after_release.egress_used
|
|
);
|
|
assert(
|
|
relayAfterRestart.abandoned_or_failed_used >=
|
|
bandwidthPreallocation.durable_after_release.abandoned_or_failed_used
|
|
);
|
|
bandwidthPreallocation.durable_after_restart = relayAfterRestart;
|
|
bandwidthPreallocation.restart_persistence = true;
|
|
bandwidthPreallocation.duration_ms =
|
|
Date.now() - bandwidthPreallocation.scenario_started_at_ms;
|
|
delete bandwidthPreallocation.scenario_started_at_ms;
|
|
await finishHostedLoginIsolationAfterRestart(hostedLoginIsolation);
|
|
const userCredentialSecurity = await finishUserCredentialSecurity({
|
|
clusterflux,
|
|
projectDir,
|
|
scope,
|
|
sessionSecret,
|
|
});
|
|
const configProvenance = configurationProvenance();
|
|
const concurrentCompileTasks = [
|
|
...new Set(
|
|
firstEvents
|
|
.filter(
|
|
(event) =>
|
|
event.task_definition === "compile_linux" &&
|
|
event.terminal_state === "completed"
|
|
)
|
|
.map((event) => event.task)
|
|
),
|
|
];
|
|
assert(concurrentCompileTasks.length >= 2);
|
|
const strictRequirementLedger = [
|
|
{ id: "01_authenticated_project", passed: Boolean(sessionSecret && tenant && project) },
|
|
{ id: "02_project_commands", passed: projectList.project_count >= 1 && projectSelect.command === "project select" },
|
|
{ id: "03_bundle_environment_inspection", passed: inspection.metadata.environments.some((environment) => environment.name === "linux") },
|
|
{ id: "04_main_parks_before_node", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" },
|
|
{ id: "05_enrollment_and_persisted_credential", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode },
|
|
{ id: "06_server_derived_node_liveness", passed: nodeLivenessTransition.initial_online === true && nodeLivenessTransition.stale_offline === false && nodeLivenessTransition.recovered_online === true },
|
|
{ id: "07_real_flagship_and_artifact", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" },
|
|
{ id: "08_same_definition_concurrency", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.completion_order.length === 2 && sameDefinitionIdentity.podman_paused_container_ids.length >= 2 && sameDefinitionIdentity.terminal_state === "not_active" },
|
|
{ id: "09_repeated_park_wake", passed: repeatedParkWake.completed_cycles >= 16 && repeatedParkWake.terminal_state === "not_active" },
|
|
{ id: "10_nested_environment_rejection", passed: agentCredentialSecurity.nested_environment_mismatch.denied === true },
|
|
{ id: "11_partial_freeze_warning_resume", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === false && agentCredentialSecurity.partial_freeze.podman_paused_container_ids.length > 0 && agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 },
|
|
{ id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true },
|
|
{ id: "13_long_join_and_process_lifecycle", passed: longJoin.duration_seconds > 120 && longJoin.terminal_state === "completed" && releasedFlagshipStatus.state === "not_active" && restart.restart_request.accepted === true && cancel.cancel_request.accepted === true },
|
|
{ id: "14_tenant_isolation", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true },
|
|
{ id: "15_user_credential_hostility", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) },
|
|
{ id: "16_node_credential_hostility", passed: securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true },
|
|
{ id: "17_agent_workflow_and_credentials", passed: agentCredentialSecurity.workflow.flagship_completed === true && agentCredentialSecurity.workflow.authenticated_without_browser === true && agentCredentialSecurity.workflow.external_direct_task_v1.denied === true && agentCredentialSecurity.workflow.child_launch === "task_launched" && agentCredentialSecurity.revoked.denied === true },
|
|
{ id: "18_dropped_response_rollback", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process" && droppedConnectionRollback.terminal_state === "not_active" },
|
|
{ id: "19_quota_and_bandwidth_preallocation", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.unrelated_tenant.unaffected_by_first_tenant_quota === true && bandwidthPreallocation.bytes_served_before_denial > 0 && bandwidthPreallocation.partial_abandon.ingress_delta > 0 && bandwidthPreallocation.partial_abandon.egress_delta > 0 && bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && bandwidthPreallocation.partial_abandon.reservation_released === true && bandwidthPreallocation.restart_persistence === true && relayEmergencyDisable.disabled.denied === true && relayEmergencyDisable.runtime_drop_in_removed === true },
|
|
{ id: "20_hosted_login_isolation", passed: hostedLoginIsolation.local_limited_status === 429 && hostedLoginIsolation.forwarding_spoof_status === 429 && hostedLoginIsolation.independent_vps_client_status === 200 && hostedLoginIsolation.control_route_after === "pong" && hostedLoginIsolation.after_service_restart.control_route === "pong" },
|
|
{ id: "21_soak_restart_and_provenance", passed: liveSoak.executed === true && serviceRestart.executed === true && manifest.source_tree_clean === true && configProvenance.clean === true },
|
|
];
|
|
const fullReleasePassed =
|
|
strictFullRelease &&
|
|
strictRequirementLedger.every((requirement) => requirement.passed === true);
|
|
const namedScenarios = [
|
|
{
|
|
id: "A",
|
|
name: "real asynchronous DAP lifecycle",
|
|
status: "passed",
|
|
duration_ms: dapEditRestart.duration_ms,
|
|
},
|
|
{
|
|
id: "B",
|
|
name: "same-definition task identity",
|
|
status: "passed",
|
|
duration_ms: sameDefinitionIdentity.duration_ms,
|
|
},
|
|
{
|
|
id: "C",
|
|
name: "real partial Podman freeze",
|
|
status: "passed",
|
|
duration_ms: agentCredentialSecurity.partial_freeze.elapsed_ms,
|
|
},
|
|
{
|
|
id: "D",
|
|
name: "actual CLI launch rollback",
|
|
status: "passed",
|
|
duration_ms: droppedConnectionRollback.duration_ms,
|
|
},
|
|
{
|
|
id: "E",
|
|
name: "hosted login isolation",
|
|
status: "passed",
|
|
duration_ms: hostedLoginIsolation.duration_ms,
|
|
},
|
|
{
|
|
id: "F",
|
|
name: "relay accounting",
|
|
status: "passed",
|
|
duration_ms: bandwidthPreallocation.duration_ms,
|
|
},
|
|
];
|
|
|
|
const report = {
|
|
kind: "clusterflux-cli-happy-path-live",
|
|
release_name: manifest.release_name,
|
|
source_commit: manifest.source_commit,
|
|
public_tree_identity: manifest.public_tree_identity,
|
|
service_addr: serviceAddr,
|
|
default_hosted_coordinator_endpoint: serviceEndpoint,
|
|
public_repository_url: publicSource.published ? publicSource.url : null,
|
|
public_source: publicSource,
|
|
tenant,
|
|
project,
|
|
worker_node: workerNode,
|
|
process: processId,
|
|
signed_in: true,
|
|
fresh_hosted_login: !reuseSessionFile,
|
|
reused_scoped_cli_session: Boolean(reuseSessionFile),
|
|
received_default_project: receivedDefaultProject,
|
|
process_started_before_node: true,
|
|
parked_main_observed: {
|
|
main_state: parkedStatus.live_process.main_state,
|
|
main_wait_state: parkedStatus.live_process.main_wait_state,
|
|
connected_nodes: parkedStatus.live_process.connected_nodes,
|
|
},
|
|
node_enrollment_grants_used: 2,
|
|
persisted_node_credential_mode:
|
|
process.platform === "win32" ? null : fs.statSync(credentialPath).mode & 0o777,
|
|
persisted_node_credential_digest: credentialDigest,
|
|
node_restarted_without_grant: true,
|
|
restarted_node_ready: secondReady.node,
|
|
node_liveness_transition: nodeLivenessTransition,
|
|
flagship_terminal_state: releasedFlagshipStatus.state,
|
|
flagship_task_definitions: [
|
|
"prepare_source",
|
|
"compile_linux",
|
|
"package_release",
|
|
],
|
|
rootless_podman_flagship_completed: true,
|
|
downstream_release_artifact: releaseArtifact,
|
|
artifact_download: {
|
|
bytes_written: download.local_download.bytes_written,
|
|
verified_digest: download.local_download.verified_digest,
|
|
executable_output: builtOutput,
|
|
},
|
|
flagship_process_released_automatically: true,
|
|
concurrent_same_definition_tasks: sameDefinitionIdentity,
|
|
flagship_same_definition_task_instances: concurrentCompileTasks,
|
|
repeated_park_wake: repeatedParkWake,
|
|
long_join: longJoin,
|
|
live_dap_edit_restart: dapEditRestart,
|
|
process_restart_requested: true,
|
|
process_cancel_requested: true,
|
|
dap_process_aborted: true,
|
|
tenant_isolation: tenantIsolation,
|
|
credential_security: {
|
|
user: userCredentialSecurity,
|
|
node: {
|
|
node: securityNode.node,
|
|
credential_digest: securityNode.credential_digest,
|
|
...securityNode.evidence,
|
|
revoked: revokedNodeCredential.denied,
|
|
},
|
|
agent: agentCredentialSecurity,
|
|
},
|
|
quota_preallocation: quotaPreallocation,
|
|
bandwidth_preallocation: bandwidthPreallocation,
|
|
relay_emergency_disable: relayEmergencyDisable,
|
|
hosted_login_isolation: hostedLoginIsolation,
|
|
dropped_connection_rollback: droppedConnectionRollback,
|
|
live_soak: liveSoak,
|
|
hosted_service_restart: {
|
|
...serviceRestart,
|
|
worker: undefined,
|
|
},
|
|
release_binding: {
|
|
manifest: manifestPath,
|
|
source_commit: manifest.source_commit,
|
|
source_tree_clean: manifest.source_tree_clean,
|
|
public_tree_identity: manifest.public_tree_identity,
|
|
binary_digests: manifest.binary_digests,
|
|
deployment: serviceRestart.after || deploymentProvenance(),
|
|
configuration: configProvenance,
|
|
},
|
|
commands,
|
|
named_scenarios: namedScenarios,
|
|
scenario_skips: [],
|
|
strict_requirement_ledger: strictRequirementLedger,
|
|
acceptance_result: fullReleasePassed ? "passed" : "partial",
|
|
};
|
|
ensureDir(path.dirname(reportPath));
|
|
fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
if (strictFullRelease && !fullReleasePassed) {
|
|
const failed = strictRequirementLedger
|
|
.filter((requirement) => requirement.passed !== true)
|
|
.map((requirement) => requirement.id);
|
|
throw new Error(`strict full release failed: ${failed.join(", ")}`);
|
|
}
|
|
console.log(`CLI happy-path live smoke passed: ${reportPath}`);
|
|
} finally {
|
|
await stopChild(worker?.child);
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error.stack || error.message);
|
|
process.exit(1);
|
|
});
|