clusterflux-public/scripts/self-hosted-coordinator-smoke.js
Clusterflux release b23e4e5bff Public release release-f70e47af061d
Source commit: f70e47af061d8f7ba27cd769c8cd2e2f8707dc72

Public tree identity: sha256:03a43db60d295811688bedaa5ad6460df0dbde27fbf37033997f4d8100f83ab2
2026-07-17 00:11:51 +02:00

662 lines
20 KiB
JavaScript

#!/usr/bin/env node
const assert = require("assert");
const cp = require("child_process");
const crypto = require("crypto");
const fs = require("fs");
const net = require("net");
const path = require("path");
const { coordinatorWireRequest } = require("./coordinator-wire");
const repo = path.resolve(__dirname, "..");
const teamArtifactDigest = `sha256:${"a".repeat(64)}`;
const teamEnvironmentDigest = `sha256:${crypto
.createHash("sha256")
.update("env-team-linux")
.digest("hex")}`;
const teamDependencyCacheDigest = `sha256:${crypto
.createHash("sha256")
.update("cargo-cache")
.digest("hex")}`;
const teamSourceDigest = `sha256:${crypto
.createHash("sha256")
.update("source-team")
.digest("hex")}`;
const coordinatorTaskModule = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]);
const coordinatorTaskBundleDigest = `sha256:${crypto
.createHash("sha256")
.update(coordinatorTaskModule)
.digest("hex")}`;
const adminToken = "self-hosted-smoke-admin-token";
const clientSessionSecret = "self-hosted-smoke-client-session-secret";
const releaseManifestPath =
process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST ||
path.join(repo, "target/public-release/public-release-manifest.json");
function sha256(value) {
return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`;
}
function digestFromParts(parts) {
const hash = crypto.createHash("sha256");
for (const value of parts) {
const bytes = Buffer.from(String(value));
const length = Buffer.alloc(8);
length.writeBigUInt64BE(BigInt(bytes.length));
hash.update(length);
hash.update(bytes);
}
return `sha256:${hash.digest("hex")}`;
}
function adminRequest(token, operation, tenant, actorUser, targetTenant, nonce) {
const issuedAtEpochSeconds = Math.floor(Date.now() / 1000);
return {
type: operation,
tenant,
actor_user: actorUser,
...(operation === "suspend_tenant" ? { target_tenant: targetTenant } : {}),
admin_proof: digestFromParts([
"clusterflux-admin-request-proof:v1",
sha256(token),
operation,
tenant,
actorUser,
targetTenant,
nonce,
issuedAtEpochSeconds,
]),
admin_nonce: nonce,
issued_at_epoch_seconds: issuedAtEpochSeconds,
};
}
function waitForJsonLine(child) {
return new Promise((resolve, reject) => {
let buffer = "";
child.stdout.on("data", (chunk) => {
buffer += chunk.toString();
const newline = buffer.indexOf("\n");
if (newline < 0) return;
try {
resolve(JSON.parse(buffer.slice(0, newline).trim()));
} catch (error) {
reject(error);
}
});
child.once("exit", (code) => {
reject(new Error(`process exited before JSON line with code ${code}`));
});
});
}
function send(addr, message) {
return new Promise((resolve, reject) => {
const socket = net.connect(addr.port, addr.host, () => {
socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`);
});
let buffer = "";
socket.on("data", (chunk) => {
buffer += chunk.toString();
const newline = buffer.indexOf("\n");
if (newline < 0) return;
socket.end();
try {
resolve(JSON.parse(buffer.slice(0, newline)));
} catch (error) {
reject(error);
}
});
socket.on("error", reject);
});
}
function authenticated(request, sessionSecret = clientSessionSecret) {
return {
type: "authenticated",
session_secret: sessionSecret,
request,
};
}
function linuxNodeCapabilities() {
return {
os: "Linux",
arch: "x86_64",
capabilities: ["Command", "RootlessPodman", "VfsArtifacts", "QuicDirect"],
environment_backends: ["Container"],
source_providers: ["filesystem", "git"]
};
}
function nodeIdentity(node) {
void node;
const { privateKey: privateKeyObject, publicKey } =
crypto.generateKeyPairSync("ed25519");
const publicDer = publicKey.export({
format: "der",
type: "spki",
});
return {
publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`,
privateKeyObject,
};
}
function signedRequestPayloadDigest(request) {
const canonicalize = (value, topLevel = true) => {
if (Array.isArray(value)) return value.map((entry) => canonicalize(entry, false));
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value)
.filter(
([key, entry]) =>
entry !== null &&
(!topLevel || !["agent_signature", "node_signature"].includes(key))
)
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
.map(([key, entry]) => [key, canonicalize(entry, false)])
);
}
return value;
};
return `sha256:${crypto
.createHash("sha256")
.update(JSON.stringify(canonicalize(request)))
.digest("hex")}`;
}
function nodeSignatureMessage(
node,
requestKind,
payloadDigest,
nonce,
issuedAtEpochSeconds
) {
const parts = [
"clusterflux-node-request-signature:v2",
node,
requestKind,
payloadDigest,
nonce,
String(issuedAtEpochSeconds),
];
return Buffer.concat(
parts.flatMap((part) => [
Buffer.from(`${Buffer.byteLength(part)}:`),
Buffer.from(part),
Buffer.from("\n"),
])
);
}
function signedNodeHeartbeat(node, identity) {
const request = { type: "node_heartbeat", node };
const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`;
const issuedAt = Math.floor(Date.now() / 1000);
const signature = crypto.sign(
null,
nodeSignatureMessage(
node,
"node_heartbeat",
signedRequestPayloadDigest(request),
nonce,
issuedAt
),
identity.privateKeyObject
);
return {
nonce,
issued_at_epoch_seconds: issuedAt,
signature: `ed25519:${signature.toString("base64")}`,
};
}
function signedNodeRequest(node, identity, requestKind, request) {
return {
type: "signed_node",
node,
node_signature: signedNodeHeartbeatForKind(node, identity, requestKind, request),
request,
};
}
function signedNodeHeartbeatForKind(node, identity, requestKind, request) {
const nonce = `${requestKind}-${process.pid}-${Date.now()}`;
const issuedAt = Math.floor(Date.now() / 1000);
const signature = crypto.sign(
null,
nodeSignatureMessage(
node,
requestKind,
signedRequestPayloadDigest(request),
nonce,
issuedAt
),
identity.privateKeyObject
);
return {
nonce,
issued_at_epoch_seconds: issuedAt,
signature: `ed25519:${signature.toString("base64")}`,
};
}
function commandOutput(command, args) {
try {
return cp
.execFileSync(command, args, {
cwd: repo,
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
})
.trim();
} catch (_) {
return null;
}
}
function runJson(command, args, options = {}) {
return JSON.parse(
cp.execFileSync(command, args, {
cwd: options.cwd || repo,
encoding: "utf8",
input: options.input,
})
);
}
function expectedSourceCommit() {
return (
process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT ||
commandOutput("git", ["rev-parse", "HEAD"])
);
}
function readReleaseManifest() {
if (!fs.existsSync(releaseManifestPath)) return null;
const manifest = JSON.parse(fs.readFileSync(releaseManifestPath, "utf8"));
if (manifest.kind !== "clusterflux-public-release") return null;
const expectedCommit = expectedSourceCommit();
if (expectedCommit && manifest.source_commit !== expectedCommit) return null;
return manifest;
}
function releaseIdentity() {
const manifest = readReleaseManifest();
return {
sourceCommit: manifest ? manifest.source_commit : expectedSourceCommit(),
releaseName:
(manifest && manifest.release_name) ||
process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME ||
null,
};
}
async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilities()) {
const identity = nodeIdentity(node);
const grant = await send(addr, authenticated({
type: "create_node_enrollment_grant",
ttl_seconds: 900,
}));
assert.strictEqual(grant.type, "node_enrollment_grant_created");
const attached = await send(addr, {
type: "exchange_node_enrollment_grant",
tenant: "team",
project: "self-hosted",
node,
public_key: identity.publicKey,
enrollment_grant: grant.grant,
});
assert.strictEqual(attached.type, "node_enrollment_exchanged");
const heartbeat = await send(addr, {
type: "node_heartbeat",
node,
node_signature: signedNodeHeartbeat(node, identity)
});
assert.strictEqual(heartbeat.type, "node_heartbeat");
const reported = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", {
type: "report_node_capabilities",
tenant: "team",
project: "self-hosted",
node,
capabilities,
cached_environment_digests: [teamEnvironmentDigest],
dependency_cache_digests: [teamDependencyCacheDigest],
source_snapshots: [teamSourceDigest],
artifact_locations: [],
direct_connectivity: true,
online: true
}));
assert.strictEqual(reported.type, "node_capabilities_recorded");
return identity;
}
(async () => {
const release = releaseIdentity();
const coordinator = cp.spawn(
"cargo",
[
"run",
"-q",
"-p",
"clusterflux-coordinator",
"--bin",
"clusterflux-coordinator",
"--",
"--listen",
"127.0.0.1:0"
],
{
cwd: repo,
env: {
...process.env,
CLUSTERFLUX_ADMIN_TOKEN: adminToken,
CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET: clientSessionSecret,
CLUSTERFLUX_SELF_HOSTED_TENANT: "team",
CLUSTERFLUX_SELF_HOSTED_PROJECT: "self-hosted",
CLUSTERFLUX_SELF_HOSTED_USER: "developer",
},
}
);
try {
const ready = await waitForJsonLine(coordinator);
const [host, portText] = ready.listen.split(":");
const addr = { host, port: Number(portText) };
assert.strictEqual(ready.client_authority, "strict");
assert.strictEqual(ready.self_hosted_session_bootstrapped, true);
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
const forgedBodyAuthority = await send(addr, {
type: "start_process",
tenant: "victim-tenant",
project: "victim-project",
actor_user: "forged-user",
process: "vp-forged",
});
assert.strictEqual(forgedBodyAuthority.type, "error");
assert.match(forgedBodyAuthority.message, /body.*identity.*not authority/i);
const wrongSession = await send(
addr,
authenticated({ type: "auth_status" }, "wrong-session-secret")
);
assert.strictEqual(wrongSession.type, "error");
assert.match(wrongSession.message, /session.*not recognized|not recognized.*session/i);
const authStatus = await send(addr, authenticated({ type: "auth_status" }));
assert.strictEqual(authStatus.type, "auth_status");
assert.strictEqual(authStatus.tenant, "team");
assert.strictEqual(authStatus.project, "self-hosted");
assert.strictEqual(authStatus.actor, "developer");
const missingAdminCredential = await send(
addr,
adminRequest("", "admin_status", "team", "forged-admin", "team", "admin-empty")
);
assert.strictEqual(missingAdminCredential.type, "error");
assert.match(missingAdminCredential.message, /admin.*proof.*invalid/i);
const wrongAdminCredential = await send(
addr,
adminRequest(
"wrong-token",
"admin_status",
"team",
"forged-admin",
"team",
"admin-wrong"
)
);
assert.strictEqual(wrongAdminCredential.type, "error");
assert.match(wrongAdminCredential.message, /admin.*proof.*invalid/i);
const replayableAdminRequest = adminRequest(
adminToken,
"admin_status",
"team",
"self-hosted-admin",
"team",
"admin-replay"
);
const directAdminStatus = await send(addr, replayableAdminRequest);
assert.strictEqual(directAdminStatus.type, "admin_status");
const replayedAdminStatus = await send(addr, replayableAdminRequest);
assert.strictEqual(replayedAdminStatus.type, "error");
assert.match(replayedAdminStatus.message, /nonce was already used/i);
const cargoTargetDir = path.resolve(
process.env.CARGO_TARGET_DIR || path.join(repo, "target")
);
const cliBin = path.join(
cargoTargetDir,
"debug",
process.platform === "win32" ? "clusterflux.exe" : "clusterflux"
);
const selfHostedCliProject = path.join(
repo,
"target/acceptance/self-hosted-cli-project"
);
fs.rmSync(selfHostedCliProject, { recursive: true, force: true });
fs.mkdirSync(selfHostedCliProject, { recursive: true });
const cliConnect = runJson(
cliBin,
[
"auth",
"connect-self-hosted",
"--coordinator",
ready.listen,
"--tenant",
"team",
"--project-id",
"self-hosted",
"--user",
"developer",
"--session-secret-stdin",
"--json",
],
{ cwd: selfHostedCliProject, input: `${clientSessionSecret}\n` }
);
assert.strictEqual(cliConnect.status, "connected");
assert.strictEqual(cliConnect.session_secret_read_from_stdin, true);
assert.strictEqual(cliConnect.session_secret_exposed_in_report, false);
assert.strictEqual(cliConnect.coordinator_response.type, "auth_status");
const sessionPath = path.join(
selfHostedCliProject,
".clusterflux/session.json"
);
const storedSession = JSON.parse(fs.readFileSync(sessionPath, "utf8"));
assert.strictEqual(storedSession.kind, "self_hosted");
assert.strictEqual(storedSession.session_secret, clientSessionSecret);
assert.strictEqual(storedSession.provider_tokens_exposed_to_cli, false);
assert.strictEqual(storedSession.provider_tokens_sent_to_nodes, false);
if (process.platform !== "win32") {
assert.strictEqual(fs.statSync(sessionPath).mode & 0o777, 0o600);
}
const cliAuthStatus = runJson(
cliBin,
["auth", "status", "--json"],
{ cwd: selfHostedCliProject }
);
assert.strictEqual(cliAuthStatus.active_coordinator, ready.listen);
assert.strictEqual(
cliAuthStatus.coordinator_account_status.used_cli_session_credential,
true
);
assert.strictEqual(
cliAuthStatus.coordinator_account_status.coordinator_response_type,
"auth_status"
);
const cliAdminStatus = runJson(cliBin, [
"admin",
"status",
"--coordinator",
ready.listen,
"--tenant",
"team",
"--user",
"self-hosted-admin",
"--admin-token",
adminToken,
"--json",
]);
assert.strictEqual(cliAdminStatus.response.type, "admin_status");
assert.strictEqual(cliAdminStatus.suspended, false);
const cliAdminSuspend = runJson(cliBin, [
"admin",
"suspend-tenant",
"--coordinator",
ready.listen,
"--tenant",
"team",
"--user",
"self-hosted-admin",
"--target-tenant",
"admin-probe-tenant",
"--admin-token",
adminToken,
"--yes",
"--json",
]);
assert.strictEqual(cliAdminSuspend.response.type, "tenant_suspended");
assert.strictEqual(cliAdminSuspend.suspended, true);
const cliAdminProbeStatus = runJson(cliBin, [
"admin",
"status",
"--coordinator",
ready.listen,
"--tenant",
"admin-probe-tenant",
"--user",
"self-hosted-admin",
"--admin-token",
adminToken,
"--json",
]);
assert.strictEqual(cliAdminProbeStatus.suspended, true);
const teamLinuxA = await attachTrustedNode(addr, "team-linux-a");
const teamLinuxBCapabilities = linuxNodeCapabilities();
teamLinuxBCapabilities.capabilities = teamLinuxBCapabilities.capabilities.filter(
(capability) => capability !== "VfsArtifacts"
);
await attachTrustedNode(addr, "team-linux-b", teamLinuxBCapabilities);
const placement = await send(addr, authenticated({
type: "schedule_task",
environment: {
os: "Linux",
arch: "x86_64",
capabilities: ["Command", "QuicDirect"]
},
environment_digest: teamEnvironmentDigest,
required_capabilities: ["VfsArtifacts"],
source_snapshot: teamSourceDigest,
required_artifacts: [],
prefer_node: "team-linux-a"
}));
assert.strictEqual(placement.type, "task_placement");
assert.strictEqual(placement.placement.node, "team-linux-a");
assert(placement.placement.reasons.includes("preferred node"));
assert(placement.placement.reasons.includes("warm environment cache"));
assert(placement.placement.reasons.includes("source snapshot already local"));
const started = await send(addr, authenticated({
type: "start_process",
process: "vp-team-build",
restart: false,
}));
assert.strictEqual(started.type, "process_started");
const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", {
type: "reconnect_node",
node: "team-linux-a",
process: "vp-team-build",
epoch: started.epoch
}));
assert.strictEqual(reconnected.type, "node_reconnected");
const launched = await send(addr, authenticated({
type: "launch_task",
task_spec: {
tenant: "team",
project: "self-hosted",
process: "vp-team-build",
task_definition: "compile-linux",
task_instance: "compile-linux",
dispatch: {
kind: "coordinator_node_wasm",
export: "compile_linux",
abi: "task_v1",
},
environment_id: null,
environment: null,
environment_digest: null,
required_capabilities: ["VfsArtifacts"],
dependency_cache: null,
source_snapshot: null,
required_artifacts: [],
args: [],
vfs_epoch: started.epoch,
bundle_digest: coordinatorTaskBundleDigest,
},
wait_for_node: false,
artifact_path: "/vfs/artifacts/team-output.txt",
wasm_module_base64: coordinatorTaskModule.toString("base64"),
}));
assert.strictEqual(launched.type, "error", JSON.stringify(launched));
assert.match(launched.message, /external callers may launch only EntrypointV1/);
const reportPath = path.join(repo, "target/acceptance/core-coordinator-compat.json");
fs.mkdirSync(path.dirname(reportPath), { recursive: true });
fs.writeFileSync(
reportPath,
`${JSON.stringify(
{
kind: "clusterflux-core-coordinator-compatibility",
source_commit: release.sourceCommit,
release_name: release.releaseName,
coordinator_implementation: "standalone-core-coordinator",
coordinator_addr: ready.listen,
client_authority: ready.client_authority,
authenticated_session: authStatus.authenticated,
forged_body_authority_denied: forgedBodyAuthority.type,
wrong_session_denied: wrongSession.type,
ping: "pong",
nodes: ["team-linux-a", "team-linux-b"],
task_placement: placement.type,
process_started: started.type,
external_task_v1_denied: launched.type,
self_hosted_admin: {
missing_credential_denied: missingAdminCredential.type,
wrong_credential_denied: wrongAdminCredential.type,
nonce_bound_proof_succeeded: directAdminStatus.type,
replay_denied: replayedAdminStatus.type,
cli_status: cliAdminStatus.response.type,
cli_suspend: cliAdminSuspend.response.type,
suspended_state_observed: cliAdminProbeStatus.suspended,
},
self_hosted_cli: {
connected: cliConnect.status,
secret_read_from_stdin: cliConnect.session_secret_read_from_stdin,
secret_exposed_in_report: cliConnect.session_secret_exposed_in_report,
session_file_mode:
process.platform === "win32"
? "windows-best-effort"
: (fs.statSync(sessionPath).mode & 0o777).toString(8),
authenticated_status:
cliAuthStatus.coordinator_account_status.coordinator_response_type,
},
},
null,
2
)}\n`
);
} finally {
coordinator.kill("SIGTERM");
}
console.log("Self-hosted coordinator smoke passed");
})().catch((error) => {
console.error(error.stack || error.message);
process.exit(1);
});