michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Export artifact bytes explicitly

Browse source
This commit is contained in:
Michel Paulissen 2026-07-03 23:43:45 +02:00
parent 2a86900e57
commit 22fa4fc675
9 changed files with 346 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -545,6 +545,7 @@ name = "disasmer-cli"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"clap",
"disasmer-core",
"serde",
@ -556,6 +557,7 @@ dependencies = [
name = "disasmer-coordinator"
version = "0.1.0"
dependencies = [
"base64",
"disasmer-core",
"postgres",
"serde",

1
Cargo.toml
View file

@ -18,6 +18,7 @@ repository = "https://example.invalid/disasmer"
[workspace.dependencies]
anyhow = "1.0"
base64 = "0.22"
clap = { version = "4.5", features = ["derive"] }
futures-executor = "0.3"
hex = "0.4"

4
DISASMER_PUBLIC_TREE.json
View file

@ -1,7 +1,7 @@
{
"kind": "disasmer-filtered-public-tree",
"source_commit": "96ad7a88ee59d224eb78789fab7620527ef37a6d",
"release_name": "dryrun-96ad7a88ee59",
"source_commit": "0dc80fb7a350886260d9504cd7daf6d122d3a6a5",
"release_name": "dryrun-0dc80fb7a350",
"filtered_out": [
"private/**",
"experiments/**",

1
crates/disasmer-cli/Cargo.toml
View file

@ -11,6 +11,7 @@ path = "src/main.rs"
[dependencies]
anyhow.workspace = true
base64.workspace = true
clap.workspace = true
disasmer-core = { path = "../disasmer-core" }
serde.workspace = true

224
crates/disasmer-cli/src/main.rs
View file

@ -6,6 +6,7 @@ use std::process::{Command, Stdio};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clap::{Args, Parser, Subcommand};
use disasmer_core::{
diagnose_environment_references, BrowserLoginFlow, BundleIdentityInputs, BundleMetadata,
@ -22,6 +23,7 @@ const BROWSER_CALLBACK_ADDR: &str = "127.0.0.1:45173";
const BROWSER_CALLBACK_PATH: &str = "/callback";
const DEFAULT_BROWSER_LOGIN_CALLBACK_TIMEOUT_SECONDS: u64 = 300;
const DOCTOR_COORDINATOR_TIMEOUT: Duration = Duration::from_millis(500);
const DEFAULT_ARTIFACT_EXPORT_MAX_BYTES: u64 = 64 * 1024 * 1024;
#[derive(Clone, Debug, Parser)]
#[command(name = "disasmer", version, arg_required_else_help = true)]
@ -1524,7 +1526,19 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
"direct_connectivity": true,
"failure_reason": "",
}))?;
let export_plan = artifact_export_plan_summary(&response, &args.to);
let mut export_plan = artifact_export_plan_summary(&response, &args.to);
let local_export =
if response.get("type").and_then(Value::as_str) == Some("artifact_export_plan") {
artifact_export_local_write_followup(&mut session, &args, &response)?
} else {
json!({
"status": "skipped",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
})
};
apply_local_export_summary(&mut export_plan, &local_export);
return Ok(json!({
"command": "artifact export",
"coordinator": coordinator,
@ -1532,6 +1546,7 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
"to": args.to,
"receiver_node": args.receiver_node,
"export_plan": export_plan,
"local_export": local_export,
"response": response,
"coordinator_session_requests": session.requests(),
}));
@ -1551,6 +1566,145 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
}))
}
fn artifact_export_local_write_followup(
session: &mut JsonLineSession,
args: &ArtifactExportArgs,
export_response: &Value,
) -> Result<Value> {
let artifact_size = export_response
.get("artifact_size_bytes")
.and_then(Value::as_u64)
.unwrap_or(DEFAULT_ARTIFACT_EXPORT_MAX_BYTES);
let nonce = command_nonce("artifact-export");
let link_response = session.request(json!({
"type": "create_artifact_download_link",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"artifact": args.artifact,
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
"token_nonce": nonce,
"now_epoch_seconds": 0,
}))?;
if link_response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
return Ok(json!({
"status": "download_link_failed",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"link_response": link_response,
}));
}
let token_digest = link_response
.pointer("/link/scoped_token_digest")
.cloned()
.context("artifact download link did not include a scoped token digest")?;
let stream_response = session.request(json!({
"type": "open_artifact_download_stream",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"artifact": args.artifact,
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
"token_nonce": nonce,
"token_digest": token_digest,
"now_epoch_seconds": 1,
"chunk_bytes": artifact_size,
}))?;
if stream_response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
return Ok(json!({
"status": "download_stream_failed",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"stream": artifact_stream_summary(&stream_response),
}));
}
let Some(content_base64) = stream_response
.get("content_base64")
.and_then(Value::as_str)
else {
return Ok(json!({
"status": "download_stream_opened_without_content",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"stream": artifact_stream_summary(&stream_response),
}));
};
let bytes = BASE64_STANDARD
.decode(content_base64)
.context("artifact download stream returned invalid base64 content")?;
if let Some(parent) = args.to.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
std::fs::write(&args.to, &bytes)
.with_context(|| format!("failed to write {}", args.to.display()))?;
Ok(json!({
"status": "local_bytes_written",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": true,
"bytes_written": bytes.len(),
"content_bytes_available": true,
"content_source": stream_response.get("content_source").cloned().unwrap_or(Value::Null),
"stream": artifact_stream_summary(&stream_response),
}))
}
fn artifact_stream_summary(response: &Value) -> Value {
json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"streamed_bytes": response.get("streamed_bytes").cloned().unwrap_or_else(|| json!(0)),
"charged_download_bytes": response.get("charged_download_bytes").cloned().unwrap_or_else(|| json!(0)),
"content_bytes_available": response.get("content_bytes_available").cloned().unwrap_or_else(|| json!(false)),
"content_source": response.get("content_source").cloned().unwrap_or(Value::Null),
"content_material_returned_in_report": false,
"error": response.get("message").cloned().unwrap_or(Value::Null),
})
}
fn apply_local_export_summary(export_plan: &mut Value, local_export: &Value) {
let Some(object) = export_plan.as_object_mut() else {
return;
};
object.insert(
"local_bytes_written_by_cli".to_owned(),
local_export
.get("local_bytes_written_by_cli")
.cloned()
.unwrap_or_else(|| json!(false)),
);
object.insert(
"local_export_status".to_owned(),
local_export
.get("status")
.cloned()
.unwrap_or_else(|| json!("unknown")),
);
object.insert(
"content_bytes_available".to_owned(),
local_export
.get("content_bytes_available")
.cloned()
.unwrap_or_else(|| json!(false)),
);
if let Some(bytes_written) = local_export.get("bytes_written") {
object.insert("bytes_written".to_owned(), bytes_written.clone());
object.insert(
"writes_require_data_plane_followup".to_owned(),
json!(false),
);
}
}
fn dap_plan(args: DapArgs) -> Result<Value> {
Ok(json!({
"command": "dap",
@ -3273,6 +3427,7 @@ fn artifact_export_plan_summary(response: &Value, to: &Path) -> Value {
"local_bytes_written_by_cli": false,
"writes_require_data_plane_followup": true,
"default_durable_store_assumed": false,
"artifact_size_bytes": response.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
"source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
"receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
"transport": plan.get("transport").cloned().unwrap_or(Value::Null),
@ -6364,11 +6519,33 @@ mod tests {
r#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#
);
let export_response = concat!(
r#"{"type":"artifact_export_plan","source_node":"node-a","receiver_node":"node-b","plan":{"transport":"NativeQuic","scope":{"tenant":"tenant","project":"project","process":"vp","object":{"Artifact":"app.txt"},"authorization_subject":"artifact-export:node-a-to-node-b"},"source":{"node":"node-a","advertised_addr":"quic://node-a","public_key_fingerprint":"sha256:source"},"destination":{"node":"node-b","advertised_addr":"quic://node-b","public_key_fingerprint":"sha256:destination"},"authorization_digest":"sha256:auth","coordinator_assisted_rendezvous":true,"coordinator_bulk_relay_allowed":false}}"#
r#"{"type":"artifact_export_plan","source_node":"node-a","receiver_node":"node-b","artifact_size_bytes":9,"plan":{"transport":"NativeQuic","scope":{"tenant":"tenant","project":"project","process":"vp","object":{"Artifact":"app.txt"},"authorization_subject":"artifact-export:node-a-to-node-b"},"source":{"node":"node-a","advertised_addr":"quic://node-a","public_key_fingerprint":"sha256:source"},"destination":{"node":"node-b","advertised_addr":"quic://node-b","public_key_fingerprint":"sha256:destination"},"authorization_digest":"sha256:auth","coordinator_assisted_rendezvous":true,"coordinator_bulk_relay_allowed":false}}"#
);
let export_download_response = concat!(
r#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:export-token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":67108864,"policy_context_digest":"sha256:policy"}}"#
);
let export_stream_response = concat!(
r#"{"type":"artifact_download_stream","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:export-token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":67108864,"policy_context_digest":"sha256:policy"},"streamed_bytes":9,"charged_download_bytes":9,"content_bytes_available":true,"content_base64":"YXBwLWJ5dGVz","content_source":"coordinator_complete_stdout_tail"}"#
);
for expected in ["create_artifact_download_link", "export_artifact_to_node"] {
let (mut stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut line = String::new();
reader.read_line(&mut line).unwrap();
assert!(line.contains(r#""type":"create_artifact_download_link""#));
assert!(line.contains(r#""tenant":"tenant""#));
assert!(line.contains(r#""project":"project""#));
assert!(line.contains(r#""artifact":"app.txt""#));
assert!(line.contains(r#""max_bytes":2048"#));
stream.write_all(download_response.as_bytes()).unwrap();
stream.write_all(b"\n").unwrap();
let (mut stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
for (expected, phase) in [
("export_artifact_to_node", "export-plan"),
("create_artifact_download_link", "export-download"),
("open_artifact_download_stream", "export-stream"),
] {
let mut line = String::new();
reader.read_line(&mut line).unwrap();
assert!(line.contains(&format!(r#""type":"{expected}""#)));
@ -6376,11 +6553,21 @@ mod tests {
assert!(line.contains(r#""project":"project""#));
assert!(line.contains(r#""artifact":"app.txt""#));
if expected == "create_artifact_download_link" {
assert!(line.contains(r#""max_bytes":2048"#));
stream.write_all(download_response.as_bytes()).unwrap();
} else {
assert_eq!(phase, "export-download");
assert!(line.contains(&format!(
r#""max_bytes":{}"#,
DEFAULT_ARTIFACT_EXPORT_MAX_BYTES
)));
stream
.write_all(export_download_response.as_bytes())
.unwrap();
} else if expected == "export_artifact_to_node" {
assert!(line.contains(r#""receiver_node":"node-b""#));
stream.write_all(export_response.as_bytes()).unwrap();
} else {
assert!(line.contains(r#""chunk_bytes":9"#));
assert!(line.contains(r#""token_digest":"sha256:export-token""#));
stream.write_all(export_stream_response.as_bytes()).unwrap();
}
stream.write_all(b"\n").unwrap();
}
@ -6393,6 +6580,8 @@ mod tests {
json: false,
};
let temp = tempfile::tempdir().unwrap();
let export_path = temp.path().join("exports/app.txt");
let download = artifact_download_report(ArtifactDownloadArgs {
scope: scope.clone(),
artifact: "app.txt".to_owned(),
@ -6402,7 +6591,7 @@ mod tests {
let export = artifact_export_report(ArtifactExportArgs {
scope,
artifact: "app.txt".to_owned(),
to: PathBuf::from("/tmp/app.txt"),
to: export_path.clone(),
receiver_node: "node-b".to_owned(),
})
.unwrap();
@ -6438,8 +6627,25 @@ mod tests {
assert_eq!(export["export_plan"]["source_node"], "node-a");
assert_eq!(export["export_plan"]["receiver_node"], "node-b");
assert_eq!(export["export_plan"]["artifact"], "app.txt");
assert_eq!(export["export_plan"]["local_path"], "/tmp/app.txt");
assert_eq!(export["export_plan"]["local_bytes_written_by_cli"], false);
assert_eq!(
export["export_plan"]["local_path"].as_str().unwrap(),
export_path.to_string_lossy().as_ref()
);
assert_eq!(export["export_plan"]["artifact_size_bytes"], 9);
assert_eq!(export["export_plan"]["local_bytes_written_by_cli"], true);
assert_eq!(
export["export_plan"]["local_export_status"],
"local_bytes_written"
);
assert_eq!(export["export_plan"]["bytes_written"], 9);
assert_eq!(
export["local_export"]["stream"]["content_material_returned_in_report"],
false
);
assert_eq!(
std::fs::read(&export_path).unwrap().as_slice(),
b"app-bytes"
);
assert_eq!(
export["export_plan"]["default_durable_store_assumed"],
false

1
crates/disasmer-coordinator/Cargo.toml
View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
disasmer-core = { path = "../disasmer-core" }
postgres.workspace = true
serde.workspace = true

53
crates/disasmer-coordinator/src/service.rs
View file

@ -3,6 +3,7 @@ use std::io::{BufRead, BufReader, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::sync::{Arc, Mutex};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{
Actor, AgentId, ArtifactId, ArtifactRegistry, Authorization, Capability, CapabilityReportError,
CredentialKind, DataPlaneObject, DataPlaneScope, DefaultScheduler, Digest,
@ -673,11 +674,17 @@ pub enum CoordinatorResponse {
link: DownloadLink,
streamed_bytes: u64,
charged_download_bytes: u64,
content_bytes_available: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_base64: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_source: Option<String>,
},
ArtifactExportPlan {
plan: DirectBulkTransferPlan,
source_node: NodeId,
receiver_node: NodeId,
artifact_size_bytes: u64,
},
Error {
message: String,
@ -2163,10 +2170,16 @@ impl CoordinatorService {
} => {
let _ = token_nonce;
let context = user_context(tenant, project, actor_user);
let artifact = ArtifactId::new(artifact);
let policy = DownloadPolicy { max_bytes };
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
let chunk_bytes = chunk_bytes.min(downloadable_size);
let mut stream = self.artifact_registry.open_download_stream(
&context,
&ArtifactId::new(artifact),
&DownloadPolicy { max_bytes },
&artifact,
&policy,
&token_digest,
now_epoch_seconds,
&self.download_limits,
@ -2181,10 +2194,18 @@ impl CoordinatorService {
)?;
let charged_download_bytes =
self.download_meter.used(&LimitKind::ArtifactDownloadBytes);
let content_base64 =
self.download_stream_content_base64(&context, &artifact, stream.streamed_bytes);
let content_bytes_available = content_base64.is_some();
let content_source =
content_bytes_available.then(|| "coordinator_complete_stdout_tail".to_owned());
Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link,
streamed_bytes: stream.streamed_bytes,
charged_download_bytes,
content_bytes_available,
content_base64,
content_source,
})
}
CoordinatorRequest::RevokeArtifactDownloadLink {
@ -2254,11 +2275,39 @@ impl CoordinatorService {
plan,
source_node,
receiver_node,
artifact_size_bytes: metadata.size,
})
}
}
}
fn download_stream_content_base64(
&self,
context: &disasmer_core::AuthContext,
artifact: &ArtifactId,
streamed_bytes: u64,
) -> Option<String> {
self.task_events
.iter()
.rev()
.find(|event| {
event.tenant == context.tenant
&& event.project == context.project
&& event.status_code == Some(0)
&& !event.stdout_truncated
&& event.stdout_bytes == event.stdout_tail.as_bytes().len() as u64
&& event.artifact_size_bytes == Some(event.stdout_bytes)
&& event.artifact_size_bytes == Some(streamed_bytes)
&& event
.artifact_path
.as_ref()
.map(artifact_id_from_path)
.as_ref()
== Some(artifact)
})
.map(|event| BASE64_STANDARD.encode(event.stdout_tail.as_bytes()))
}
fn record_debug_audit_event(
&mut self,
tenant: TenantId,

51
scripts/artifact-export-smoke.js
View file

@ -2,7 +2,9 @@
const assert = require("assert");
const cp = require("child_process");
const fs = require("fs");
const net = require("net");
const os = require("os");
const path = require("path");
const repo = path.resolve(__dirname, "..");
@ -111,6 +113,20 @@ function nodeCapabilities() {
};
}
function runJson(command, args, options = {}) {
const result = cp.spawnSync(command, args, {
cwd: repo,
encoding: "utf8",
...options,
});
if (result.status !== 0) {
throw new Error(
`${command} ${args.join(" ")} failed with code ${result.status}\n${result.stderr}\n${result.stdout}`
);
}
return JSON.parse(result.stdout);
}
async function reportNode(addr, node, { directConnectivity = true, online = true } = {}) {
const response = await send(addr, {
type: "report_node_capabilities",
@ -192,6 +208,41 @@ async function reportNode(addr, node, { directConnectivity = true, online = true
assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true);
assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false);
assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/);
assert.strictEqual(exportPlan.artifact_size_bytes, produced.stdout_bytes);
const temp = fs.mkdtempSync(path.join(os.tmpdir(), "disasmer-artifact-export-"));
const exportPath = path.join(temp, "export-output.txt");
const cliExport = runJson("cargo", [
"run",
"-q",
"-p",
"disasmer-cli",
"--bin",
"disasmer",
"--",
"artifact",
"export",
"--coordinator",
`${addr.host}:${addr.port}`,
"--tenant",
"tenant",
"--project-id",
"project",
"--user",
"user",
"--json",
"export-output.txt",
"--receiver-node",
"node-export-receiver",
"--to",
exportPath,
]);
assert.strictEqual(cliExport.command, "artifact export");
assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true);
assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written");
assert.strictEqual(cliExport.export_plan.bytes_written, produced.stdout_bytes);
assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false);
assert.strictEqual(fs.readFileSync(exportPath, "utf8"), produced.stdout_tail);
const crossTenant = await send(addr, {
type: "export_artifact_to_node",

20
scripts/cli-first-contract-smoke.js
View file

@ -81,6 +81,11 @@ expect(
"CLI-first non-e2e gate wording",
/scripts\/acceptance-cli-first\.sh[\s\S]*final public-release e2e remains intentionally excluded/
);
expect(
criteria,
"artifact export explicit local byte write",
/artifact export <id> --to <path>` writes bytes[\s\S]*explicit bounded download stream[\s\S]*complete staged content is available/
);
expect(
cliFirstAcceptance,
"CLI-first acceptance report",
@ -330,6 +335,21 @@ expect(
"CLI blocks build before scheduling unsafe inputs",
/blocked_before_schedule[\s\S]*scheduled_work[\s\S]*false/
);
expect(
cli,
"CLI artifact export writes explicit local bytes from stream content",
/fn artifact_export_local_write_followup[\s\S]*open_artifact_download_stream[\s\S]*content_base64[\s\S]*std::fs::write/
);
expect(
cli,
"CLI artifact export keeps content out of reports",
/fn artifact_stream_summary[\s\S]*content_material_returned_in_report[\s\S]*false/
);
expect(
coordinator,
"coordinator exposes conservative artifact stream content",
/fn download_stream_content_base64[\s\S]*stdout_truncated[\s\S]*artifact_size_bytes[\s\S]*BASE64_STANDARD\.encode/
);
for (const [name, pattern] of [
["agent --json flag", /struct AgentEnrollArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],