#!/usr/bin/env node const assert = require("assert"); const cp = require("child_process"); const crypto = require("crypto"); const fs = require("fs"); const net = require("net"); const path = require("path"); const { coordinatorWireRequest } = require("./coordinator-wire"); const repo = path.resolve(__dirname, ".."); const teamArtifactDigest = `sha256:${"a".repeat(64)}`; const teamEnvironmentDigest = `sha256:${crypto .createHash("sha256") .update("env-team-linux") .digest("hex")}`; const teamDependencyCacheDigest = `sha256:${crypto .createHash("sha256") .update("cargo-cache") .digest("hex")}`; const teamSourceDigest = `sha256:${crypto .createHash("sha256") .update("source-team") .digest("hex")}`; const coordinatorTaskModule = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); const coordinatorTaskBundleDigest = `sha256:${crypto .createHash("sha256") .update(coordinatorTaskModule) .digest("hex")}`; const adminToken = "self-hosted-smoke-admin-token"; const clientSessionSecret = "self-hosted-smoke-client-session-secret"; const releaseManifestPath = process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || path.join(repo, "target/public-release/public-release-manifest.json"); function sha256(value) { return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; } function digestFromParts(parts) { const hash = crypto.createHash("sha256"); for (const value of parts) { const bytes = Buffer.from(String(value)); const length = Buffer.alloc(8); length.writeBigUInt64BE(BigInt(bytes.length)); hash.update(length); hash.update(bytes); } return `sha256:${hash.digest("hex")}`; } function adminRequest(token, operation, tenant, actorUser, targetTenant, nonce) { const issuedAtEpochSeconds = Math.floor(Date.now() / 1000); return { type: operation, tenant, actor_user: actorUser, ...(operation === "suspend_tenant" ? { target_tenant: targetTenant } : {}), admin_proof: digestFromParts([ "clusterflux-admin-request-proof:v1", sha256(token), operation, tenant, actorUser, targetTenant, nonce, issuedAtEpochSeconds, ]), admin_nonce: nonce, issued_at_epoch_seconds: issuedAtEpochSeconds, }; } function waitForJsonLine(child) { return new Promise((resolve, reject) => { let buffer = ""; child.stdout.on("data", (chunk) => { buffer += chunk.toString(); const newline = buffer.indexOf("\n"); if (newline < 0) return; try { resolve(JSON.parse(buffer.slice(0, newline).trim())); } catch (error) { reject(error); } }); child.once("exit", (code) => { reject(new Error(`process exited before JSON line with code ${code}`)); }); }); } function send(addr, message) { return new Promise((resolve, reject) => { const socket = net.connect(addr.port, addr.host, () => { socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); }); let buffer = ""; socket.on("data", (chunk) => { buffer += chunk.toString(); const newline = buffer.indexOf("\n"); if (newline < 0) return; socket.end(); try { resolve(JSON.parse(buffer.slice(0, newline))); } catch (error) { reject(error); } }); socket.on("error", reject); }); } function authenticated(request, sessionSecret = clientSessionSecret) { return { type: "authenticated", session_secret: sessionSecret, request, }; } function linuxNodeCapabilities() { return { os: "Linux", arch: "x86_64", capabilities: ["Command", "RootlessPodman", "VfsArtifacts", "QuicDirect"], environment_backends: ["Container"], source_providers: ["filesystem", "git"] }; } function nodeIdentity(node) { void node; const { privateKey: privateKeyObject, publicKey } = crypto.generateKeyPairSync("ed25519"); const publicDer = publicKey.export({ format: "der", type: "spki", }); return { publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, privateKeyObject, }; } function signedRequestPayloadDigest(request) { const canonicalize = (value, topLevel = true) => { if (Array.isArray(value)) return value.map((entry) => canonicalize(entry, false)); if (value && typeof value === "object") { return Object.fromEntries( Object.entries(value) .filter( ([key, entry]) => entry !== null && (!topLevel || !["agent_signature", "node_signature"].includes(key)) ) .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) .map(([key, entry]) => [key, canonicalize(entry, false)]) ); } return value; }; return `sha256:${crypto .createHash("sha256") .update(JSON.stringify(canonicalize(request))) .digest("hex")}`; } function nodeSignatureMessage( node, requestKind, payloadDigest, nonce, issuedAtEpochSeconds ) { const parts = [ "clusterflux-node-request-signature:v2", node, requestKind, payloadDigest, nonce, String(issuedAtEpochSeconds), ]; return Buffer.concat( parts.flatMap((part) => [ Buffer.from(`${Buffer.byteLength(part)}:`), Buffer.from(part), Buffer.from("\n"), ]) ); } function signedNodeHeartbeat(tenant, project, node, identity) { const request = { type: "node_heartbeat", tenant, project, node }; const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`; const issuedAt = Math.floor(Date.now() / 1000); const signature = crypto.sign( null, nodeSignatureMessage( node, "node_heartbeat", signedRequestPayloadDigest(request), nonce, issuedAt ), identity.privateKeyObject ); return { nonce, issued_at_epoch_seconds: issuedAt, signature: `ed25519:${signature.toString("base64")}`, }; } function signedNodeRequest(node, identity, requestKind, request) { return { type: "signed_node", node, node_signature: signedNodeHeartbeatForKind(node, identity, requestKind, request), request, }; } function signedNodeHeartbeatForKind(node, identity, requestKind, request) { const nonce = `${requestKind}-${process.pid}-${Date.now()}`; const issuedAt = Math.floor(Date.now() / 1000); const signature = crypto.sign( null, nodeSignatureMessage( node, requestKind, signedRequestPayloadDigest(request), nonce, issuedAt ), identity.privateKeyObject ); return { nonce, issued_at_epoch_seconds: issuedAt, signature: `ed25519:${signature.toString("base64")}`, }; } function commandOutput(command, args) { try { return cp .execFileSync(command, args, { cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], }) .trim(); } catch (_) { return null; } } function runJson(command, args, options = {}) { return JSON.parse( cp.execFileSync(command, args, { cwd: options.cwd || repo, encoding: "utf8", input: options.input, }) ); } function expectedSourceCommit() { return ( process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || commandOutput("git", ["rev-parse", "HEAD"]) ); } function readReleaseManifest() { if (!fs.existsSync(releaseManifestPath)) return null; const manifest = JSON.parse(fs.readFileSync(releaseManifestPath, "utf8")); if (manifest.kind !== "clusterflux-public-release") return null; const expectedCommit = expectedSourceCommit(); if (expectedCommit && manifest.source_commit !== expectedCommit) return null; return manifest; } function releaseIdentity() { const manifest = readReleaseManifest(); return { sourceCommit: manifest ? manifest.source_commit : expectedSourceCommit(), releaseName: (manifest && manifest.release_name) || process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || null, }; } async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilities()) { const identity = nodeIdentity(node); const grant = await send(addr, authenticated({ type: "create_node_enrollment_grant", ttl_seconds: 900, })); assert.strictEqual(grant.type, "node_enrollment_grant_created"); const attached = await send(addr, { type: "exchange_node_enrollment_grant", tenant: "team", project: "self-hosted", node, public_key: identity.publicKey, enrollment_grant: grant.grant, }); assert.strictEqual(attached.type, "node_enrollment_exchanged"); const heartbeat = await send(addr, { type: "node_heartbeat", tenant: "team", project: "self-hosted", node, node_signature: signedNodeHeartbeat("team", "self-hosted", node, identity) }); assert.strictEqual(heartbeat.type, "node_heartbeat"); const reported = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { type: "report_node_capabilities", tenant: "team", project: "self-hosted", node, capabilities, cached_environment_digests: [teamEnvironmentDigest], dependency_cache_digests: [teamDependencyCacheDigest], source_snapshots: [teamSourceDigest], artifact_locations: [], direct_connectivity: true, online: true })); assert.strictEqual(reported.type, "node_capabilities_recorded"); return identity; } (async () => { const release = releaseIdentity(); const coordinator = cp.spawn( "cargo", [ "run", "-q", "-p", "clusterflux-coordinator", "--bin", "clusterflux-coordinator", "--", "--listen", "127.0.0.1:0" ], { cwd: repo, env: { ...process.env, CLUSTERFLUX_ADMIN_TOKEN: adminToken, CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET: clientSessionSecret, CLUSTERFLUX_SELF_HOSTED_TENANT: "team", CLUSTERFLUX_SELF_HOSTED_PROJECT: "self-hosted", CLUSTERFLUX_SELF_HOSTED_USER: "developer", }, } ); try { const ready = await waitForJsonLine(coordinator); const [host, portText] = ready.listen.split(":"); const addr = { host, port: Number(portText) }; assert.strictEqual(ready.client_authority, "strict"); assert.strictEqual(ready.self_hosted_session_bootstrapped, true); assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); const forgedBodyAuthority = await send(addr, { type: "start_process", tenant: "victim-tenant", project: "victim-project", actor_user: "forged-user", process: "vp-forged", }); assert.strictEqual(forgedBodyAuthority.type, "error"); assert.match(forgedBodyAuthority.message, /body.*identity.*not authority/i); const wrongSession = await send( addr, authenticated({ type: "auth_status" }, "wrong-session-secret") ); assert.strictEqual(wrongSession.type, "error"); assert.match(wrongSession.message, /session.*not recognized|not recognized.*session/i); const authStatus = await send(addr, authenticated({ type: "auth_status" })); assert.strictEqual(authStatus.type, "auth_status"); assert.strictEqual(authStatus.tenant, "team"); assert.strictEqual(authStatus.project, "self-hosted"); assert.strictEqual(authStatus.actor, "developer"); const missingAdminCredential = await send( addr, adminRequest("", "admin_status", "team", "forged-admin", "team", "admin-empty") ); assert.strictEqual(missingAdminCredential.type, "error"); assert.match(missingAdminCredential.message, /admin.*proof.*invalid/i); const wrongAdminCredential = await send( addr, adminRequest( "wrong-token", "admin_status", "team", "forged-admin", "team", "admin-wrong" ) ); assert.strictEqual(wrongAdminCredential.type, "error"); assert.match(wrongAdminCredential.message, /admin.*proof.*invalid/i); const replayableAdminRequest = adminRequest( adminToken, "admin_status", "team", "self-hosted-admin", "team", "admin-replay" ); const directAdminStatus = await send(addr, replayableAdminRequest); assert.strictEqual(directAdminStatus.type, "admin_status"); const replayedAdminStatus = await send(addr, replayableAdminRequest); assert.strictEqual(replayedAdminStatus.type, "error"); assert.match(replayedAdminStatus.message, /nonce was already used/i); const cargoTargetDir = path.resolve( process.env.CARGO_TARGET_DIR || path.join(repo, "target") ); const cliBin = path.join( cargoTargetDir, "debug", process.platform === "win32" ? "clusterflux.exe" : "clusterflux" ); const selfHostedCliProject = path.join( repo, "target/acceptance/self-hosted-cli-project" ); fs.rmSync(selfHostedCliProject, { recursive: true, force: true }); fs.mkdirSync(selfHostedCliProject, { recursive: true }); const cliConnect = runJson( cliBin, [ "auth", "connect-self-hosted", "--coordinator", ready.listen, "--tenant", "team", "--project-id", "self-hosted", "--user", "developer", "--session-secret-stdin", "--json", ], { cwd: selfHostedCliProject, input: `${clientSessionSecret}\n` } ); assert.strictEqual(cliConnect.status, "connected"); assert.strictEqual(cliConnect.session_secret_read_from_stdin, true); assert.strictEqual(cliConnect.session_secret_exposed_in_report, false); assert.strictEqual(cliConnect.coordinator_response.type, "auth_status"); const sessionPath = path.join( selfHostedCliProject, ".clusterflux/session.json" ); const storedSession = JSON.parse(fs.readFileSync(sessionPath, "utf8")); assert.strictEqual(storedSession.kind, "self_hosted"); assert.strictEqual(storedSession.session_secret, clientSessionSecret); assert.strictEqual(storedSession.provider_tokens_exposed_to_cli, false); assert.strictEqual(storedSession.provider_tokens_sent_to_nodes, false); if (process.platform !== "win32") { assert.strictEqual(fs.statSync(sessionPath).mode & 0o777, 0o600); } const cliAuthStatus = runJson( cliBin, ["auth", "status", "--json"], { cwd: selfHostedCliProject } ); assert.strictEqual(cliAuthStatus.active_coordinator, ready.listen); assert.strictEqual( cliAuthStatus.coordinator_account_status.used_cli_session_credential, true ); assert.strictEqual( cliAuthStatus.coordinator_account_status.coordinator_response_type, "auth_status" ); const cliAdminStatus = runJson(cliBin, [ "admin", "status", "--coordinator", ready.listen, "--tenant", "team", "--user", "self-hosted-admin", "--admin-token", adminToken, "--json", ]); assert.strictEqual(cliAdminStatus.response.type, "admin_status"); assert.strictEqual(cliAdminStatus.suspended, false); const cliAdminSuspend = runJson(cliBin, [ "admin", "suspend-tenant", "--coordinator", ready.listen, "--tenant", "team", "--user", "self-hosted-admin", "--target-tenant", "admin-probe-tenant", "--admin-token", adminToken, "--yes", "--json", ]); assert.strictEqual(cliAdminSuspend.response.type, "tenant_suspended"); assert.strictEqual(cliAdminSuspend.suspended, true); const cliAdminProbeStatus = runJson(cliBin, [ "admin", "status", "--coordinator", ready.listen, "--tenant", "admin-probe-tenant", "--user", "self-hosted-admin", "--admin-token", adminToken, "--json", ]); assert.strictEqual(cliAdminProbeStatus.suspended, true); const teamLinuxA = await attachTrustedNode(addr, "team-linux-a"); const teamLinuxBCapabilities = linuxNodeCapabilities(); teamLinuxBCapabilities.capabilities = teamLinuxBCapabilities.capabilities.filter( (capability) => capability !== "VfsArtifacts" ); await attachTrustedNode(addr, "team-linux-b", teamLinuxBCapabilities); const placement = await send(addr, authenticated({ type: "schedule_task", environment: { os: "Linux", arch: "x86_64", capabilities: ["Command", "QuicDirect"] }, environment_digest: teamEnvironmentDigest, required_capabilities: ["VfsArtifacts"], source_snapshot: teamSourceDigest, required_artifacts: [], prefer_node: "team-linux-a" })); assert.strictEqual(placement.type, "task_placement"); assert.strictEqual(placement.placement.node, "team-linux-a"); assert(placement.placement.reasons.includes("preferred node")); assert(placement.placement.reasons.includes("warm environment cache")); assert(placement.placement.reasons.includes("source snapshot already local")); const started = await send(addr, authenticated({ type: "start_process", process: "vp-team-build", restart: false, })); assert.strictEqual(started.type, "process_started"); const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", { type: "reconnect_node", tenant: "team", project: "self-hosted", node: "team-linux-a", process: "vp-team-build", epoch: started.epoch })); assert.strictEqual(reconnected.type, "node_reconnected"); const launched = await send(addr, authenticated({ type: "launch_task", task_spec: { tenant: "team", project: "self-hosted", process: "vp-team-build", task_definition: "compile-linux", task_instance: "compile-linux", dispatch: { kind: "coordinator_node_wasm", export: "compile_linux", abi: "task_v1", }, environment_id: null, environment: null, environment_digest: null, required_capabilities: ["VfsArtifacts"], dependency_cache: null, source_snapshot: null, required_artifacts: [], args: [], vfs_epoch: started.epoch, bundle_digest: coordinatorTaskBundleDigest, }, wait_for_node: false, artifact_path: "/vfs/artifacts/team-output.txt", wasm_module_base64: coordinatorTaskModule.toString("base64"), })); assert.strictEqual(launched.type, "error", JSON.stringify(launched)); assert.match(launched.message, /external callers may launch only EntrypointV1/); const reportPath = path.join(repo, "target/acceptance/core-coordinator-compat.json"); fs.mkdirSync(path.dirname(reportPath), { recursive: true }); fs.writeFileSync( reportPath, `${JSON.stringify( { kind: "clusterflux-core-coordinator-compatibility", source_commit: release.sourceCommit, release_name: release.releaseName, coordinator_implementation: "standalone-core-coordinator", coordinator_addr: ready.listen, client_authority: ready.client_authority, authenticated_session: authStatus.authenticated, forged_body_authority_denied: forgedBodyAuthority.type, wrong_session_denied: wrongSession.type, ping: "pong", nodes: ["team-linux-a", "team-linux-b"], task_placement: placement.type, process_started: started.type, external_task_v1_denied: launched.type, self_hosted_admin: { missing_credential_denied: missingAdminCredential.type, wrong_credential_denied: wrongAdminCredential.type, nonce_bound_proof_succeeded: directAdminStatus.type, replay_denied: replayedAdminStatus.type, cli_status: cliAdminStatus.response.type, cli_suspend: cliAdminSuspend.response.type, suspended_state_observed: cliAdminProbeStatus.suspended, }, self_hosted_cli: { connected: cliConnect.status, secret_read_from_stdin: cliConnect.session_secret_read_from_stdin, secret_exposed_in_report: cliConnect.session_secret_exposed_in_report, session_file_mode: process.platform === "win32" ? "windows-best-effort" : (fs.statSync(sessionPath).mode & 0o777).toString(8), authenticated_status: cliAuthStatus.coordinator_account_status.coordinator_response_type, }, }, null, 2 )}\n` ); } finally { coordinator.kill("SIGTERM"); } console.log("Self-hosted coordinator smoke passed"); })().catch((error) => { console.error(error.stack || error.message); process.exit(1); });