#!/usr/bin/env node const assert = require("assert"); const cp = require("child_process"); const crypto = require("crypto"); const fs = require("fs"); const os = require("os"); const path = require("path"); const { nodeIdentity, signedNodeRequest } = require("./node-signing"); const { ensureRootlessPodman, flagshipNodeCapabilities, launchFlagship, repo, runFlagshipWorker, send, waitForJsonLine, } = require("./real-flagship-harness"); const downloadNode = "node-download"; const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode); const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); async function downloadRetainedBytes(addr, link, artifact, expectedSize) { const chunks = []; let offset = 0; for (let attempt = 0; attempt < 500; attempt += 1) { const response = await send(addr, { type: "open_artifact_download_stream", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, token_digest: link.link.scoped_token_digest, chunk_bytes: 256 * 1024, }); assert.strictEqual(response.type, "artifact_download_stream"); if (!response.content_bytes_available) { assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending"); await delay(10); continue; } assert.strictEqual(response.content_source, "retaining_node_reverse_stream"); assert.strictEqual(response.content_offset, offset); const bytes = Buffer.from(response.content_base64, "base64"); assert.strictEqual(response.streamed_bytes, bytes.length); chunks.push(bytes); offset += bytes.length; if (response.content_eof) { const content = Buffer.concat(chunks); assert.strictEqual(content.length, expectedSize); return { content, response }; } } throw new Error("timed out waiting for retained artifact reverse stream"); } function downloadNodeCapabilities() { return flagshipNodeCapabilities(); } (async () => { ensureRootlessPodman(); const coordinator = cp.spawn( "cargo", [ "run", "-q", "-p", "clusterflux-coordinator", "--bin", "clusterflux-coordinator", "--", "--listen", "127.0.0.1:0", "--allow-local-trusted-loopback", ], { cwd: repo } ); let worker; try { const ready = await waitForJsonLine(coordinator); const [host, portText] = ready.listen.split(":"); const addr = { host, port: Number(portText) }; assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity); const workerReady = await worker.ready; assert.strictEqual(workerReady.node_status, "ready"); assert.strictEqual(workerReady.mode, "worker"); const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr); assert.strictEqual(compileEvent.status_code, 0); assert.strictEqual(packageEvent.status_code, 0); assert.deepStrictEqual(compileEvent.result, { Artifact: { id: compileEvent.artifact_path.slice("/vfs/artifacts/".length), digest: compileEvent.artifact_digest, size_bytes: compileEvent.artifact_size_bytes, }, }); assert.deepStrictEqual(packageEvent.result, { Artifact: { id: packageEvent.artifact_path.slice("/vfs/artifacts/".length), digest: packageEvent.artifact_digest, size_bytes: packageEvent.artifact_size_bytes, }, }); assert.ok(compileEvent.artifact_size_bytes > 0); assert.strictEqual(packageEvent.artifact_size_bytes, compileEvent.artifact_size_bytes); assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); const artifactPath = packageEvent.artifact_path; assert.match( artifactPath, /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ ); const artifact = artifactPath.slice("/vfs/artifacts/".length); const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { type: "report_node_capabilities", tenant: "tenant", project: "project", node: downloadNode, capabilities: downloadNodeCapabilities(), cached_environment_digests: [], dependency_cache_digests: [], source_snapshots: [], artifact_locations: [artifact], direct_connectivity: false, online: true, })); assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded"); const disconnectedLink = await send(addr, { type: "create_artifact_download_link", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, ttl_seconds: 60, }); assert.strictEqual(disconnectedLink.type, "artifact_download_link"); const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { type: "report_node_capabilities", tenant: "tenant", project: "project", node: downloadNode, capabilities: downloadNodeCapabilities(), cached_environment_digests: [], dependency_cache_digests: [], source_snapshots: [], artifact_locations: [artifact], direct_connectivity: true, online: true, })); assert.strictEqual(connectedReport.type, "node_capabilities_recorded"); const link = await send(addr, { type: "create_artifact_download_link", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, ttl_seconds: 60, }); assert.strictEqual(link.type, "artifact_download_link"); assert.strictEqual(link.link.tenant, "tenant"); assert.strictEqual(link.link.project, "project"); assert.strictEqual(link.link.process, virtualProcess); assert.deepStrictEqual(link.link.actor, { User: "user" }); assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/); assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60); assert.ok( link.link.url_path.endsWith( `/artifacts/tenant/project/${virtualProcess}/${artifact}` ) ); assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" }); const crossTenant = await send(addr, { type: "create_artifact_download_link", tenant: "other", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, ttl_seconds: 60, }); assert.strictEqual(crossTenant.type, "error"); assert.match(crossTenant.message, /tenant mismatch/); const crossProject = await send(addr, { type: "create_artifact_download_link", tenant: "tenant", project: "other-project", actor_user: "user", artifact, max_bytes: 1024 * 1024, ttl_seconds: 60, }); assert.strictEqual(crossProject.type, "error"); assert.match(crossProject.message, /project mismatch/); const crossTenantOpen = await send(addr, { type: "open_artifact_download_stream", tenant: "other", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, token_digest: link.link.scoped_token_digest, chunk_bytes: 1, }); assert.strictEqual(crossTenantOpen.type, "error"); assert.match(crossTenantOpen.message, /tenant mismatch/); const crossProjectOpen = await send(addr, { type: "open_artifact_download_stream", tenant: "tenant", project: "other-project", actor_user: "user", artifact, max_bytes: 1024 * 1024, token_digest: link.link.scoped_token_digest, chunk_bytes: 1, }); assert.strictEqual(crossProjectOpen.type, "error"); assert.match(crossProjectOpen.message, /project mismatch/); const guessed = await send(addr, { type: "open_artifact_download_stream", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, token_digest: "sha256:guessed", chunk_bytes: 1, }); assert.strictEqual(guessed.type, "error"); assert.match(guessed.message, /token is invalid/); const crossActorOpen = await send(addr, { type: "open_artifact_download_stream", tenant: "tenant", project: "project", actor_user: "other-user", artifact, max_bytes: 1024 * 1024, token_digest: link.link.scoped_token_digest, chunk_bytes: 1, }); assert.strictEqual(crossActorOpen.type, "error"); assert.match(crossActorOpen.message, /token is invalid/); const downloaded = await downloadRetainedBytes( addr, link, artifact, packageEvent.artifact_size_bytes, ); const downloadedDigest = `sha256:${crypto .createHash("sha256") .update(downloaded.content) .digest("hex")}`; assert.strictEqual(downloadedDigest, packageEvent.artifact_digest); const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-")); try { const executable = path.join(inspect, "hello-clusterflux"); fs.writeFileSync(executable, downloaded.content); fs.chmodSync(executable, 0o755); assert.strictEqual( cp.execFileSync(executable, { encoding: "utf8" }), "hello from a real Clusterflux build\n" ); } finally { fs.rmSync(inspect, { recursive: true, force: true }); } const cliDownloadDirectory = fs.mkdtempSync( path.join(os.tmpdir(), "clusterflux-cli-download-") ); try { const cliDownloadPath = path.join(cliDownloadDirectory, "hello-clusterflux"); const cliDownload = JSON.parse( cp.execFileSync( "cargo", [ "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", "artifact", "download", artifact, "--to", cliDownloadPath, "--max-bytes", "1048576", "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, "--tenant", "tenant", "--project-id", "project", "--user", "user", "--json", ], { cwd: repo, env: process.env, encoding: "utf8" } ) ); assert.strictEqual(cliDownload.command, "artifact download"); assert.strictEqual(cliDownload.local_download.status, "local_bytes_written"); assert.strictEqual( cliDownload.local_download.verified_digest, packageEvent.artifact_digest ); assert.strictEqual( `sha256:${crypto .createHash("sha256") .update(fs.readFileSync(cliDownloadPath)) .digest("hex")}`, packageEvent.artifact_digest ); } finally { fs.rmSync(cliDownloadDirectory, { recursive: true, force: true }); } assert.strictEqual(downloaded.response.content_eof, true); assert.strictEqual( downloaded.response.charged_download_bytes, packageEvent.artifact_size_bytes ); assert.strictEqual(downloaded.response.link.artifact, artifact); const crossActorRevoke = await send(addr, { type: "revoke_artifact_download_link", tenant: "tenant", project: "project", actor_user: "other-user", artifact, token_digest: link.link.scoped_token_digest, }); assert.strictEqual(crossActorRevoke.type, "error"); assert.match(crossActorRevoke.message, /token is invalid/); const revoked = await send(addr, { type: "revoke_artifact_download_link", tenant: "tenant", project: "project", actor_user: "user", artifact, token_digest: link.link.scoped_token_digest, }); assert.strictEqual(revoked.type, "artifact_download_link_revoked"); assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest); const revokedOpen = await send(addr, { type: "open_artifact_download_stream", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, token_digest: link.link.scoped_token_digest, chunk_bytes: 1, }); assert.strictEqual(revokedOpen.type, "error"); assert.match(revokedOpen.message, /revoked/); const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { type: "report_node_capabilities", tenant: "tenant", project: "project", node: downloadNode, capabilities: downloadNodeCapabilities(), cached_environment_digests: [], dependency_cache_digests: [], source_snapshots: [], artifact_locations: [], direct_connectivity: false, online: true, })); assert.strictEqual(gcReport.type, "node_capabilities_recorded"); const collectedLink = await send(addr, { type: "create_artifact_download_link", tenant: "tenant", project: "project", actor_user: "user", artifact, max_bytes: 1024 * 1024, ttl_seconds: 60, }); assert.strictEqual(collectedLink.type, "error"); assert.match(collectedLink.message, /unavailable from current retention/); } finally { worker?.child.kill("SIGTERM"); coordinator.kill("SIGTERM"); } console.log("Artifact download smoke passed"); })().catch((error) => { console.error(error.stack || error.message); process.exit(1); });