a6ca9e26dd
Source commit: be1f654ae8de7d5eeb2005b4e1a05bf1ebf3124f Public tree identity: sha256:ab53679496be4be7d1e02bc00723072774d1a3f87e10cc56558a752f28f6abb1
401 lines
14 KiB
JavaScript
Executable file
401 lines
14 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
|
|
|
const assert = require("assert");
|
|
const cp = require("child_process");
|
|
const crypto = require("crypto");
|
|
const fs = require("fs");
|
|
const os = require("os");
|
|
const path = require("path");
|
|
const { nodeIdentity, signedNodeRequest } = require("./node-signing");
|
|
const {
|
|
ensureRootlessPodman,
|
|
flagshipNodeCapabilities,
|
|
launchFlagship,
|
|
repo,
|
|
runFlagshipWorker,
|
|
send,
|
|
waitForJsonLine,
|
|
} = require("./real-flagship-harness");
|
|
|
|
const downloadNode = "node-download";
|
|
const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode);
|
|
|
|
const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
|
|
async function downloadRetainedBytes(addr, link, artifact, expectedSize) {
|
|
const chunks = [];
|
|
let offset = 0;
|
|
for (let attempt = 0; attempt < 500; attempt += 1) {
|
|
const response = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: link.link.scoped_token_digest,
|
|
chunk_bytes: 256 * 1024,
|
|
});
|
|
assert.strictEqual(response.type, "artifact_download_stream");
|
|
if (!response.content_bytes_available) {
|
|
assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending");
|
|
await delay(10);
|
|
continue;
|
|
}
|
|
assert.strictEqual(response.content_source, "retaining_node_reverse_stream");
|
|
assert.strictEqual(response.content_offset, offset);
|
|
const bytes = Buffer.from(response.content_base64, "base64");
|
|
assert.strictEqual(response.streamed_bytes, bytes.length);
|
|
chunks.push(bytes);
|
|
offset += bytes.length;
|
|
if (response.content_eof) {
|
|
const content = Buffer.concat(chunks);
|
|
assert.strictEqual(content.length, expectedSize);
|
|
return { content, response };
|
|
}
|
|
}
|
|
throw new Error("timed out waiting for retained artifact reverse stream");
|
|
}
|
|
|
|
function downloadNodeCapabilities() {
|
|
return flagshipNodeCapabilities();
|
|
}
|
|
|
|
(async () => {
|
|
ensureRootlessPodman();
|
|
const coordinator = cp.spawn(
|
|
"cargo",
|
|
[
|
|
"run",
|
|
"-q",
|
|
"-p",
|
|
"clusterflux-coordinator",
|
|
"--bin",
|
|
"clusterflux-coordinator",
|
|
"--",
|
|
"--listen",
|
|
"127.0.0.1:0",
|
|
"--allow-local-trusted-loopback",
|
|
],
|
|
{ cwd: repo }
|
|
);
|
|
|
|
let worker;
|
|
try {
|
|
const ready = await waitForJsonLine(coordinator);
|
|
const [host, portText] = ready.listen.split(":");
|
|
const addr = { host, port: Number(portText) };
|
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
|
|
|
worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity);
|
|
const workerReady = await worker.ready;
|
|
assert.strictEqual(workerReady.node_status, "ready");
|
|
assert.strictEqual(workerReady.mode, "worker");
|
|
const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr);
|
|
assert.strictEqual(compileEvent.status_code, 0);
|
|
assert.strictEqual(packageEvent.status_code, 0);
|
|
assert.deepStrictEqual(compileEvent.result, {
|
|
Artifact: {
|
|
id: compileEvent.artifact_path.slice("/vfs/artifacts/".length),
|
|
digest: compileEvent.artifact_digest,
|
|
size_bytes: compileEvent.artifact_size_bytes,
|
|
},
|
|
});
|
|
assert.deepStrictEqual(packageEvent.result, {
|
|
Artifact: {
|
|
id: packageEvent.artifact_path.slice("/vfs/artifacts/".length),
|
|
digest: packageEvent.artifact_digest,
|
|
size_bytes: packageEvent.artifact_size_bytes,
|
|
},
|
|
});
|
|
assert.ok(compileEvent.artifact_size_bytes > 0);
|
|
assert.ok(packageEvent.artifact_size_bytes > compileEvent.artifact_size_bytes);
|
|
assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
|
|
assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
|
|
const artifactPath = packageEvent.artifact_path;
|
|
assert.match(artifactPath, /^\/vfs\/artifacts\/release\.tar-[0-9a-f]{64}$/);
|
|
const artifact = artifactPath.slice("/vfs/artifacts/".length);
|
|
|
|
const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
|
type: "report_node_capabilities",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
node: downloadNode,
|
|
capabilities: downloadNodeCapabilities(),
|
|
cached_environment_digests: [],
|
|
dependency_cache_digests: [],
|
|
source_snapshots: [],
|
|
artifact_locations: [artifact],
|
|
direct_connectivity: false,
|
|
online: true,
|
|
}));
|
|
assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded");
|
|
|
|
const disconnectedLink = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60,
|
|
});
|
|
assert.strictEqual(disconnectedLink.type, "artifact_download_link");
|
|
|
|
const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
|
type: "report_node_capabilities",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
node: downloadNode,
|
|
capabilities: downloadNodeCapabilities(),
|
|
cached_environment_digests: [],
|
|
dependency_cache_digests: [],
|
|
source_snapshots: [],
|
|
artifact_locations: [artifact],
|
|
direct_connectivity: true,
|
|
online: true,
|
|
}));
|
|
assert.strictEqual(connectedReport.type, "node_capabilities_recorded");
|
|
|
|
const link = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60,
|
|
});
|
|
assert.strictEqual(link.type, "artifact_download_link");
|
|
assert.strictEqual(link.link.tenant, "tenant");
|
|
assert.strictEqual(link.link.project, "project");
|
|
assert.strictEqual(link.link.process, virtualProcess);
|
|
assert.deepStrictEqual(link.link.actor, { User: "user" });
|
|
assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
|
|
assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000));
|
|
assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60);
|
|
assert.ok(
|
|
link.link.url_path.endsWith(
|
|
`/artifacts/tenant/project/${virtualProcess}/${artifact}`
|
|
)
|
|
);
|
|
assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" });
|
|
|
|
const crossTenant = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "other",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60,
|
|
});
|
|
assert.strictEqual(crossTenant.type, "error");
|
|
assert.match(crossTenant.message, /tenant mismatch/);
|
|
|
|
const crossProject = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "other-project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60,
|
|
});
|
|
assert.strictEqual(crossProject.type, "error");
|
|
assert.match(crossProject.message, /project mismatch/);
|
|
|
|
const crossTenantOpen = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "other",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: link.link.scoped_token_digest,
|
|
chunk_bytes: 1,
|
|
});
|
|
assert.strictEqual(crossTenantOpen.type, "error");
|
|
assert.match(crossTenantOpen.message, /tenant mismatch/);
|
|
|
|
const crossProjectOpen = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "tenant",
|
|
project: "other-project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: link.link.scoped_token_digest,
|
|
chunk_bytes: 1,
|
|
});
|
|
assert.strictEqual(crossProjectOpen.type, "error");
|
|
assert.match(crossProjectOpen.message, /project mismatch/);
|
|
|
|
const guessed = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: "sha256:guessed",
|
|
chunk_bytes: 1,
|
|
});
|
|
assert.strictEqual(guessed.type, "error");
|
|
assert.match(guessed.message, /token is invalid/);
|
|
|
|
const crossActorOpen = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "other-user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: link.link.scoped_token_digest,
|
|
chunk_bytes: 1,
|
|
});
|
|
assert.strictEqual(crossActorOpen.type, "error");
|
|
assert.match(crossActorOpen.message, /token is invalid/);
|
|
|
|
const downloaded = await downloadRetainedBytes(
|
|
addr,
|
|
link,
|
|
artifact,
|
|
packageEvent.artifact_size_bytes,
|
|
);
|
|
const downloadedDigest = `sha256:${crypto
|
|
.createHash("sha256")
|
|
.update(downloaded.content)
|
|
.digest("hex")}`;
|
|
assert.strictEqual(downloadedDigest, packageEvent.artifact_digest);
|
|
const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-"));
|
|
try {
|
|
const archive = path.join(inspect, "release.tar");
|
|
fs.writeFileSync(archive, downloaded.content);
|
|
const listing = cp.execFileSync("tar", ["-tf", archive], { encoding: "utf8" });
|
|
assert.strictEqual(listing.trim(), "hello-clusterflux");
|
|
cp.execFileSync("tar", ["-xf", archive, "-C", inspect]);
|
|
fs.chmodSync(path.join(inspect, "hello-clusterflux"), 0o755);
|
|
assert.strictEqual(
|
|
cp.execFileSync(path.join(inspect, "hello-clusterflux"), { encoding: "utf8" }),
|
|
"hello from a real Clusterflux build\n"
|
|
);
|
|
} finally {
|
|
fs.rmSync(inspect, { recursive: true, force: true });
|
|
}
|
|
const cliDownloadDirectory = fs.mkdtempSync(
|
|
path.join(os.tmpdir(), "clusterflux-cli-download-")
|
|
);
|
|
try {
|
|
const cliDownloadPath = path.join(cliDownloadDirectory, "release.tar");
|
|
const cliDownload = JSON.parse(
|
|
cp.execFileSync(
|
|
"cargo",
|
|
[
|
|
"run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--",
|
|
"artifact", "download", artifact,
|
|
"--to", cliDownloadPath,
|
|
"--max-bytes", "1048576",
|
|
"--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`,
|
|
"--tenant", "tenant",
|
|
"--project-id", "project",
|
|
"--user", "user",
|
|
"--json",
|
|
],
|
|
{ cwd: repo, env: process.env, encoding: "utf8" }
|
|
)
|
|
);
|
|
assert.strictEqual(cliDownload.command, "artifact download");
|
|
assert.strictEqual(cliDownload.local_download.status, "local_bytes_written");
|
|
assert.strictEqual(
|
|
cliDownload.local_download.verified_digest,
|
|
packageEvent.artifact_digest
|
|
);
|
|
assert.strictEqual(
|
|
`sha256:${crypto
|
|
.createHash("sha256")
|
|
.update(fs.readFileSync(cliDownloadPath))
|
|
.digest("hex")}`,
|
|
packageEvent.artifact_digest
|
|
);
|
|
} finally {
|
|
fs.rmSync(cliDownloadDirectory, { recursive: true, force: true });
|
|
}
|
|
assert.strictEqual(downloaded.response.content_eof, true);
|
|
assert.strictEqual(
|
|
downloaded.response.charged_download_bytes,
|
|
packageEvent.artifact_size_bytes
|
|
);
|
|
assert.strictEqual(downloaded.response.link.artifact, artifact);
|
|
|
|
const crossActorRevoke = await send(addr, {
|
|
type: "revoke_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "other-user",
|
|
artifact,
|
|
token_digest: link.link.scoped_token_digest,
|
|
});
|
|
assert.strictEqual(crossActorRevoke.type, "error");
|
|
assert.match(crossActorRevoke.message, /token is invalid/);
|
|
|
|
const revoked = await send(addr, {
|
|
type: "revoke_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
token_digest: link.link.scoped_token_digest,
|
|
});
|
|
assert.strictEqual(revoked.type, "artifact_download_link_revoked");
|
|
assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest);
|
|
|
|
const revokedOpen = await send(addr, {
|
|
type: "open_artifact_download_stream",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
token_digest: link.link.scoped_token_digest,
|
|
chunk_bytes: 1,
|
|
});
|
|
assert.strictEqual(revokedOpen.type, "error");
|
|
assert.match(revokedOpen.message, /revoked/);
|
|
|
|
const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
|
type: "report_node_capabilities",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
node: downloadNode,
|
|
capabilities: downloadNodeCapabilities(),
|
|
cached_environment_digests: [],
|
|
dependency_cache_digests: [],
|
|
source_snapshots: [],
|
|
artifact_locations: [],
|
|
direct_connectivity: false,
|
|
online: true,
|
|
}));
|
|
assert.strictEqual(gcReport.type, "node_capabilities_recorded");
|
|
|
|
const collectedLink = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60,
|
|
});
|
|
assert.strictEqual(collectedLink.type, "error");
|
|
assert.match(collectedLink.message, /unavailable from current retention/);
|
|
} finally {
|
|
worker?.child.kill("SIGTERM");
|
|
coordinator.kill("SIGTERM");
|
|
}
|
|
|
|
console.log("Artifact download smoke passed");
|
|
})().catch((error) => {
|
|
console.error(error.stack || error.message);
|
|
process.exit(1);
|
|
});
|