Export artifact bytes explicitly
This commit is contained in:
parent
2a86900e57
commit
22fa4fc675
9 changed files with 346 additions and 15 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -545,6 +545,7 @@ name = "disasmer-cli"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64",
|
||||||
"clap",
|
"clap",
|
||||||
"disasmer-core",
|
"disasmer-core",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -556,6 +557,7 @@ dependencies = [
|
||||||
name = "disasmer-coordinator"
|
name = "disasmer-coordinator"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"disasmer-core",
|
"disasmer-core",
|
||||||
"postgres",
|
"postgres",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ repository = "https://example.invalid/disasmer"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
|
base64 = "0.22"
|
||||||
clap = { version = "4.5", features = ["derive"] }
|
clap = { version = "4.5", features = ["derive"] }
|
||||||
futures-executor = "0.3"
|
futures-executor = "0.3"
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"kind": "disasmer-filtered-public-tree",
|
"kind": "disasmer-filtered-public-tree",
|
||||||
"source_commit": "96ad7a88ee59d224eb78789fab7620527ef37a6d",
|
"source_commit": "0dc80fb7a350886260d9504cd7daf6d122d3a6a5",
|
||||||
"release_name": "dryrun-96ad7a88ee59",
|
"release_name": "dryrun-0dc80fb7a350",
|
||||||
"filtered_out": [
|
"filtered_out": [
|
||||||
"private/**",
|
"private/**",
|
||||||
"experiments/**",
|
"experiments/**",
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
|
base64.workspace = true
|
||||||
clap.workspace = true
|
clap.workspace = true
|
||||||
disasmer-core = { path = "../disasmer-core" }
|
disasmer-core = { path = "../disasmer-core" }
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ use std::process::{Command, Stdio};
|
||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
|
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
||||||
use clap::{Args, Parser, Subcommand};
|
use clap::{Args, Parser, Subcommand};
|
||||||
use disasmer_core::{
|
use disasmer_core::{
|
||||||
diagnose_environment_references, BrowserLoginFlow, BundleIdentityInputs, BundleMetadata,
|
diagnose_environment_references, BrowserLoginFlow, BundleIdentityInputs, BundleMetadata,
|
||||||
|
|
@ -22,6 +23,7 @@ const BROWSER_CALLBACK_ADDR: &str = "127.0.0.1:45173";
|
||||||
const BROWSER_CALLBACK_PATH: &str = "/callback";
|
const BROWSER_CALLBACK_PATH: &str = "/callback";
|
||||||
const DEFAULT_BROWSER_LOGIN_CALLBACK_TIMEOUT_SECONDS: u64 = 300;
|
const DEFAULT_BROWSER_LOGIN_CALLBACK_TIMEOUT_SECONDS: u64 = 300;
|
||||||
const DOCTOR_COORDINATOR_TIMEOUT: Duration = Duration::from_millis(500);
|
const DOCTOR_COORDINATOR_TIMEOUT: Duration = Duration::from_millis(500);
|
||||||
|
const DEFAULT_ARTIFACT_EXPORT_MAX_BYTES: u64 = 64 * 1024 * 1024;
|
||||||
|
|
||||||
#[derive(Clone, Debug, Parser)]
|
#[derive(Clone, Debug, Parser)]
|
||||||
#[command(name = "disasmer", version, arg_required_else_help = true)]
|
#[command(name = "disasmer", version, arg_required_else_help = true)]
|
||||||
|
|
@ -1524,7 +1526,19 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
|
||||||
"direct_connectivity": true,
|
"direct_connectivity": true,
|
||||||
"failure_reason": "",
|
"failure_reason": "",
|
||||||
}))?;
|
}))?;
|
||||||
let export_plan = artifact_export_plan_summary(&response, &args.to);
|
let mut export_plan = artifact_export_plan_summary(&response, &args.to);
|
||||||
|
let local_export =
|
||||||
|
if response.get("type").and_then(Value::as_str) == Some("artifact_export_plan") {
|
||||||
|
artifact_export_local_write_followup(&mut session, &args, &response)?
|
||||||
|
} else {
|
||||||
|
json!({
|
||||||
|
"status": "skipped",
|
||||||
|
"explicit_user_action": true,
|
||||||
|
"local_path": &args.to,
|
||||||
|
"local_bytes_written_by_cli": false,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
apply_local_export_summary(&mut export_plan, &local_export);
|
||||||
return Ok(json!({
|
return Ok(json!({
|
||||||
"command": "artifact export",
|
"command": "artifact export",
|
||||||
"coordinator": coordinator,
|
"coordinator": coordinator,
|
||||||
|
|
@ -1532,6 +1546,7 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
|
||||||
"to": args.to,
|
"to": args.to,
|
||||||
"receiver_node": args.receiver_node,
|
"receiver_node": args.receiver_node,
|
||||||
"export_plan": export_plan,
|
"export_plan": export_plan,
|
||||||
|
"local_export": local_export,
|
||||||
"response": response,
|
"response": response,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
}));
|
}));
|
||||||
|
|
@ -1551,6 +1566,145 @@ fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn artifact_export_local_write_followup(
|
||||||
|
session: &mut JsonLineSession,
|
||||||
|
args: &ArtifactExportArgs,
|
||||||
|
export_response: &Value,
|
||||||
|
) -> Result<Value> {
|
||||||
|
let artifact_size = export_response
|
||||||
|
.get("artifact_size_bytes")
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(DEFAULT_ARTIFACT_EXPORT_MAX_BYTES);
|
||||||
|
let nonce = command_nonce("artifact-export");
|
||||||
|
let link_response = session.request(json!({
|
||||||
|
"type": "create_artifact_download_link",
|
||||||
|
"tenant": args.scope.tenant,
|
||||||
|
"project": args.scope.project,
|
||||||
|
"actor_user": args.scope.user,
|
||||||
|
"artifact": args.artifact,
|
||||||
|
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
|
||||||
|
"token_nonce": nonce,
|
||||||
|
"now_epoch_seconds": 0,
|
||||||
|
}))?;
|
||||||
|
if link_response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
|
||||||
|
return Ok(json!({
|
||||||
|
"status": "download_link_failed",
|
||||||
|
"explicit_user_action": true,
|
||||||
|
"local_path": &args.to,
|
||||||
|
"local_bytes_written_by_cli": false,
|
||||||
|
"content_bytes_available": false,
|
||||||
|
"link_response": link_response,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let token_digest = link_response
|
||||||
|
.pointer("/link/scoped_token_digest")
|
||||||
|
.cloned()
|
||||||
|
.context("artifact download link did not include a scoped token digest")?;
|
||||||
|
let stream_response = session.request(json!({
|
||||||
|
"type": "open_artifact_download_stream",
|
||||||
|
"tenant": args.scope.tenant,
|
||||||
|
"project": args.scope.project,
|
||||||
|
"actor_user": args.scope.user,
|
||||||
|
"artifact": args.artifact,
|
||||||
|
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
|
||||||
|
"token_nonce": nonce,
|
||||||
|
"token_digest": token_digest,
|
||||||
|
"now_epoch_seconds": 1,
|
||||||
|
"chunk_bytes": artifact_size,
|
||||||
|
}))?;
|
||||||
|
if stream_response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
|
||||||
|
return Ok(json!({
|
||||||
|
"status": "download_stream_failed",
|
||||||
|
"explicit_user_action": true,
|
||||||
|
"local_path": &args.to,
|
||||||
|
"local_bytes_written_by_cli": false,
|
||||||
|
"content_bytes_available": false,
|
||||||
|
"stream": artifact_stream_summary(&stream_response),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(content_base64) = stream_response
|
||||||
|
.get("content_base64")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
else {
|
||||||
|
return Ok(json!({
|
||||||
|
"status": "download_stream_opened_without_content",
|
||||||
|
"explicit_user_action": true,
|
||||||
|
"local_path": &args.to,
|
||||||
|
"local_bytes_written_by_cli": false,
|
||||||
|
"content_bytes_available": false,
|
||||||
|
"stream": artifact_stream_summary(&stream_response),
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
let bytes = BASE64_STANDARD
|
||||||
|
.decode(content_base64)
|
||||||
|
.context("artifact download stream returned invalid base64 content")?;
|
||||||
|
if let Some(parent) = args.to.parent() {
|
||||||
|
std::fs::create_dir_all(parent)
|
||||||
|
.with_context(|| format!("failed to create {}", parent.display()))?;
|
||||||
|
}
|
||||||
|
std::fs::write(&args.to, &bytes)
|
||||||
|
.with_context(|| format!("failed to write {}", args.to.display()))?;
|
||||||
|
|
||||||
|
Ok(json!({
|
||||||
|
"status": "local_bytes_written",
|
||||||
|
"explicit_user_action": true,
|
||||||
|
"local_path": &args.to,
|
||||||
|
"local_bytes_written_by_cli": true,
|
||||||
|
"bytes_written": bytes.len(),
|
||||||
|
"content_bytes_available": true,
|
||||||
|
"content_source": stream_response.get("content_source").cloned().unwrap_or(Value::Null),
|
||||||
|
"stream": artifact_stream_summary(&stream_response),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn artifact_stream_summary(response: &Value) -> Value {
|
||||||
|
json!({
|
||||||
|
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
|
||||||
|
"streamed_bytes": response.get("streamed_bytes").cloned().unwrap_or_else(|| json!(0)),
|
||||||
|
"charged_download_bytes": response.get("charged_download_bytes").cloned().unwrap_or_else(|| json!(0)),
|
||||||
|
"content_bytes_available": response.get("content_bytes_available").cloned().unwrap_or_else(|| json!(false)),
|
||||||
|
"content_source": response.get("content_source").cloned().unwrap_or(Value::Null),
|
||||||
|
"content_material_returned_in_report": false,
|
||||||
|
"error": response.get("message").cloned().unwrap_or(Value::Null),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_local_export_summary(export_plan: &mut Value, local_export: &Value) {
|
||||||
|
let Some(object) = export_plan.as_object_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
object.insert(
|
||||||
|
"local_bytes_written_by_cli".to_owned(),
|
||||||
|
local_export
|
||||||
|
.get("local_bytes_written_by_cli")
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| json!(false)),
|
||||||
|
);
|
||||||
|
object.insert(
|
||||||
|
"local_export_status".to_owned(),
|
||||||
|
local_export
|
||||||
|
.get("status")
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| json!("unknown")),
|
||||||
|
);
|
||||||
|
object.insert(
|
||||||
|
"content_bytes_available".to_owned(),
|
||||||
|
local_export
|
||||||
|
.get("content_bytes_available")
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| json!(false)),
|
||||||
|
);
|
||||||
|
if let Some(bytes_written) = local_export.get("bytes_written") {
|
||||||
|
object.insert("bytes_written".to_owned(), bytes_written.clone());
|
||||||
|
object.insert(
|
||||||
|
"writes_require_data_plane_followup".to_owned(),
|
||||||
|
json!(false),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn dap_plan(args: DapArgs) -> Result<Value> {
|
fn dap_plan(args: DapArgs) -> Result<Value> {
|
||||||
Ok(json!({
|
Ok(json!({
|
||||||
"command": "dap",
|
"command": "dap",
|
||||||
|
|
@ -3273,6 +3427,7 @@ fn artifact_export_plan_summary(response: &Value, to: &Path) -> Value {
|
||||||
"local_bytes_written_by_cli": false,
|
"local_bytes_written_by_cli": false,
|
||||||
"writes_require_data_plane_followup": true,
|
"writes_require_data_plane_followup": true,
|
||||||
"default_durable_store_assumed": false,
|
"default_durable_store_assumed": false,
|
||||||
|
"artifact_size_bytes": response.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
|
||||||
"source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
|
"source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
|
||||||
"receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
|
"receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
|
||||||
"transport": plan.get("transport").cloned().unwrap_or(Value::Null),
|
"transport": plan.get("transport").cloned().unwrap_or(Value::Null),
|
||||||
|
|
@ -6364,11 +6519,33 @@ mod tests {
|
||||||
r#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#
|
r#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#
|
||||||
);
|
);
|
||||||
let export_response = concat!(
|
let export_response = concat!(
|
||||||
r#"{"type":"artifact_export_plan","source_node":"node-a","receiver_node":"node-b","plan":{"transport":"NativeQuic","scope":{"tenant":"tenant","project":"project","process":"vp","object":{"Artifact":"app.txt"},"authorization_subject":"artifact-export:node-a-to-node-b"},"source":{"node":"node-a","advertised_addr":"quic://node-a","public_key_fingerprint":"sha256:source"},"destination":{"node":"node-b","advertised_addr":"quic://node-b","public_key_fingerprint":"sha256:destination"},"authorization_digest":"sha256:auth","coordinator_assisted_rendezvous":true,"coordinator_bulk_relay_allowed":false}}"#
|
r#"{"type":"artifact_export_plan","source_node":"node-a","receiver_node":"node-b","artifact_size_bytes":9,"plan":{"transport":"NativeQuic","scope":{"tenant":"tenant","project":"project","process":"vp","object":{"Artifact":"app.txt"},"authorization_subject":"artifact-export:node-a-to-node-b"},"source":{"node":"node-a","advertised_addr":"quic://node-a","public_key_fingerprint":"sha256:source"},"destination":{"node":"node-b","advertised_addr":"quic://node-b","public_key_fingerprint":"sha256:destination"},"authorization_digest":"sha256:auth","coordinator_assisted_rendezvous":true,"coordinator_bulk_relay_allowed":false}}"#
|
||||||
|
);
|
||||||
|
let export_download_response = concat!(
|
||||||
|
r#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:export-token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":67108864,"policy_context_digest":"sha256:policy"}}"#
|
||||||
|
);
|
||||||
|
let export_stream_response = concat!(
|
||||||
|
r#"{"type":"artifact_download_stream","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant/project/vp/app.txt","scoped_token_digest":"sha256:export-token","expires_at_epoch_seconds":60,"tenant":"tenant","project":"project","process":"vp","actor":{"User":"user"},"max_bytes":67108864,"policy_context_digest":"sha256:policy"},"streamed_bytes":9,"charged_download_bytes":9,"content_bytes_available":true,"content_base64":"YXBwLWJ5dGVz","content_source":"coordinator_complete_stdout_tail"}"#
|
||||||
);
|
);
|
||||||
for expected in ["create_artifact_download_link", "export_artifact_to_node"] {
|
|
||||||
let (mut stream, _) = listener.accept().unwrap();
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
assert!(line.contains(r#""type":"create_artifact_download_link""#));
|
||||||
|
assert!(line.contains(r#""tenant":"tenant""#));
|
||||||
|
assert!(line.contains(r#""project":"project""#));
|
||||||
|
assert!(line.contains(r#""artifact":"app.txt""#));
|
||||||
|
assert!(line.contains(r#""max_bytes":2048"#));
|
||||||
|
stream.write_all(download_response.as_bytes()).unwrap();
|
||||||
|
stream.write_all(b"\n").unwrap();
|
||||||
|
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||||
|
for (expected, phase) in [
|
||||||
|
("export_artifact_to_node", "export-plan"),
|
||||||
|
("create_artifact_download_link", "export-download"),
|
||||||
|
("open_artifact_download_stream", "export-stream"),
|
||||||
|
] {
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
reader.read_line(&mut line).unwrap();
|
reader.read_line(&mut line).unwrap();
|
||||||
assert!(line.contains(&format!(r#""type":"{expected}""#)));
|
assert!(line.contains(&format!(r#""type":"{expected}""#)));
|
||||||
|
|
@ -6376,11 +6553,21 @@ mod tests {
|
||||||
assert!(line.contains(r#""project":"project""#));
|
assert!(line.contains(r#""project":"project""#));
|
||||||
assert!(line.contains(r#""artifact":"app.txt""#));
|
assert!(line.contains(r#""artifact":"app.txt""#));
|
||||||
if expected == "create_artifact_download_link" {
|
if expected == "create_artifact_download_link" {
|
||||||
assert!(line.contains(r#""max_bytes":2048"#));
|
assert_eq!(phase, "export-download");
|
||||||
stream.write_all(download_response.as_bytes()).unwrap();
|
assert!(line.contains(&format!(
|
||||||
} else {
|
r#""max_bytes":{}"#,
|
||||||
|
DEFAULT_ARTIFACT_EXPORT_MAX_BYTES
|
||||||
|
)));
|
||||||
|
stream
|
||||||
|
.write_all(export_download_response.as_bytes())
|
||||||
|
.unwrap();
|
||||||
|
} else if expected == "export_artifact_to_node" {
|
||||||
assert!(line.contains(r#""receiver_node":"node-b""#));
|
assert!(line.contains(r#""receiver_node":"node-b""#));
|
||||||
stream.write_all(export_response.as_bytes()).unwrap();
|
stream.write_all(export_response.as_bytes()).unwrap();
|
||||||
|
} else {
|
||||||
|
assert!(line.contains(r#""chunk_bytes":9"#));
|
||||||
|
assert!(line.contains(r#""token_digest":"sha256:export-token""#));
|
||||||
|
stream.write_all(export_stream_response.as_bytes()).unwrap();
|
||||||
}
|
}
|
||||||
stream.write_all(b"\n").unwrap();
|
stream.write_all(b"\n").unwrap();
|
||||||
}
|
}
|
||||||
|
|
@ -6393,6 +6580,8 @@ mod tests {
|
||||||
json: false,
|
json: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let export_path = temp.path().join("exports/app.txt");
|
||||||
let download = artifact_download_report(ArtifactDownloadArgs {
|
let download = artifact_download_report(ArtifactDownloadArgs {
|
||||||
scope: scope.clone(),
|
scope: scope.clone(),
|
||||||
artifact: "app.txt".to_owned(),
|
artifact: "app.txt".to_owned(),
|
||||||
|
|
@ -6402,7 +6591,7 @@ mod tests {
|
||||||
let export = artifact_export_report(ArtifactExportArgs {
|
let export = artifact_export_report(ArtifactExportArgs {
|
||||||
scope,
|
scope,
|
||||||
artifact: "app.txt".to_owned(),
|
artifact: "app.txt".to_owned(),
|
||||||
to: PathBuf::from("/tmp/app.txt"),
|
to: export_path.clone(),
|
||||||
receiver_node: "node-b".to_owned(),
|
receiver_node: "node-b".to_owned(),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
@ -6438,8 +6627,25 @@ mod tests {
|
||||||
assert_eq!(export["export_plan"]["source_node"], "node-a");
|
assert_eq!(export["export_plan"]["source_node"], "node-a");
|
||||||
assert_eq!(export["export_plan"]["receiver_node"], "node-b");
|
assert_eq!(export["export_plan"]["receiver_node"], "node-b");
|
||||||
assert_eq!(export["export_plan"]["artifact"], "app.txt");
|
assert_eq!(export["export_plan"]["artifact"], "app.txt");
|
||||||
assert_eq!(export["export_plan"]["local_path"], "/tmp/app.txt");
|
assert_eq!(
|
||||||
assert_eq!(export["export_plan"]["local_bytes_written_by_cli"], false);
|
export["export_plan"]["local_path"].as_str().unwrap(),
|
||||||
|
export_path.to_string_lossy().as_ref()
|
||||||
|
);
|
||||||
|
assert_eq!(export["export_plan"]["artifact_size_bytes"], 9);
|
||||||
|
assert_eq!(export["export_plan"]["local_bytes_written_by_cli"], true);
|
||||||
|
assert_eq!(
|
||||||
|
export["export_plan"]["local_export_status"],
|
||||||
|
"local_bytes_written"
|
||||||
|
);
|
||||||
|
assert_eq!(export["export_plan"]["bytes_written"], 9);
|
||||||
|
assert_eq!(
|
||||||
|
export["local_export"]["stream"]["content_material_returned_in_report"],
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&export_path).unwrap().as_slice(),
|
||||||
|
b"app-bytes"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
export["export_plan"]["default_durable_store_assumed"],
|
export["export_plan"]["default_durable_store_assumed"],
|
||||||
false
|
false
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
base64.workspace = true
|
||||||
disasmer-core = { path = "../disasmer-core" }
|
disasmer-core = { path = "../disasmer-core" }
|
||||||
postgres.workspace = true
|
postgres.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ use std::io::{BufRead, BufReader, Write};
|
||||||
use std::net::{SocketAddr, TcpListener, TcpStream};
|
use std::net::{SocketAddr, TcpListener, TcpStream};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
||||||
use disasmer_core::{
|
use disasmer_core::{
|
||||||
Actor, AgentId, ArtifactId, ArtifactRegistry, Authorization, Capability, CapabilityReportError,
|
Actor, AgentId, ArtifactId, ArtifactRegistry, Authorization, Capability, CapabilityReportError,
|
||||||
CredentialKind, DataPlaneObject, DataPlaneScope, DefaultScheduler, Digest,
|
CredentialKind, DataPlaneObject, DataPlaneScope, DefaultScheduler, Digest,
|
||||||
|
|
@ -673,11 +674,17 @@ pub enum CoordinatorResponse {
|
||||||
link: DownloadLink,
|
link: DownloadLink,
|
||||||
streamed_bytes: u64,
|
streamed_bytes: u64,
|
||||||
charged_download_bytes: u64,
|
charged_download_bytes: u64,
|
||||||
|
content_bytes_available: bool,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
content_base64: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
content_source: Option<String>,
|
||||||
},
|
},
|
||||||
ArtifactExportPlan {
|
ArtifactExportPlan {
|
||||||
plan: DirectBulkTransferPlan,
|
plan: DirectBulkTransferPlan,
|
||||||
source_node: NodeId,
|
source_node: NodeId,
|
||||||
receiver_node: NodeId,
|
receiver_node: NodeId,
|
||||||
|
artifact_size_bytes: u64,
|
||||||
},
|
},
|
||||||
Error {
|
Error {
|
||||||
message: String,
|
message: String,
|
||||||
|
|
@ -2163,10 +2170,16 @@ impl CoordinatorService {
|
||||||
} => {
|
} => {
|
||||||
let _ = token_nonce;
|
let _ = token_nonce;
|
||||||
let context = user_context(tenant, project, actor_user);
|
let context = user_context(tenant, project, actor_user);
|
||||||
|
let artifact = ArtifactId::new(artifact);
|
||||||
|
let policy = DownloadPolicy { max_bytes };
|
||||||
|
let downloadable_size = self
|
||||||
|
.artifact_registry
|
||||||
|
.downloadable_size(&context, &artifact, &policy)?;
|
||||||
|
let chunk_bytes = chunk_bytes.min(downloadable_size);
|
||||||
let mut stream = self.artifact_registry.open_download_stream(
|
let mut stream = self.artifact_registry.open_download_stream(
|
||||||
&context,
|
&context,
|
||||||
&ArtifactId::new(artifact),
|
&artifact,
|
||||||
&DownloadPolicy { max_bytes },
|
&policy,
|
||||||
&token_digest,
|
&token_digest,
|
||||||
now_epoch_seconds,
|
now_epoch_seconds,
|
||||||
&self.download_limits,
|
&self.download_limits,
|
||||||
|
|
@ -2181,10 +2194,18 @@ impl CoordinatorService {
|
||||||
)?;
|
)?;
|
||||||
let charged_download_bytes =
|
let charged_download_bytes =
|
||||||
self.download_meter.used(&LimitKind::ArtifactDownloadBytes);
|
self.download_meter.used(&LimitKind::ArtifactDownloadBytes);
|
||||||
|
let content_base64 =
|
||||||
|
self.download_stream_content_base64(&context, &artifact, stream.streamed_bytes);
|
||||||
|
let content_bytes_available = content_base64.is_some();
|
||||||
|
let content_source =
|
||||||
|
content_bytes_available.then(|| "coordinator_complete_stdout_tail".to_owned());
|
||||||
Ok(CoordinatorResponse::ArtifactDownloadStream {
|
Ok(CoordinatorResponse::ArtifactDownloadStream {
|
||||||
link: stream.link,
|
link: stream.link,
|
||||||
streamed_bytes: stream.streamed_bytes,
|
streamed_bytes: stream.streamed_bytes,
|
||||||
charged_download_bytes,
|
charged_download_bytes,
|
||||||
|
content_bytes_available,
|
||||||
|
content_base64,
|
||||||
|
content_source,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
CoordinatorRequest::RevokeArtifactDownloadLink {
|
CoordinatorRequest::RevokeArtifactDownloadLink {
|
||||||
|
|
@ -2254,11 +2275,39 @@ impl CoordinatorService {
|
||||||
plan,
|
plan,
|
||||||
source_node,
|
source_node,
|
||||||
receiver_node,
|
receiver_node,
|
||||||
|
artifact_size_bytes: metadata.size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn download_stream_content_base64(
|
||||||
|
&self,
|
||||||
|
context: &disasmer_core::AuthContext,
|
||||||
|
artifact: &ArtifactId,
|
||||||
|
streamed_bytes: u64,
|
||||||
|
) -> Option<String> {
|
||||||
|
self.task_events
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.find(|event| {
|
||||||
|
event.tenant == context.tenant
|
||||||
|
&& event.project == context.project
|
||||||
|
&& event.status_code == Some(0)
|
||||||
|
&& !event.stdout_truncated
|
||||||
|
&& event.stdout_bytes == event.stdout_tail.as_bytes().len() as u64
|
||||||
|
&& event.artifact_size_bytes == Some(event.stdout_bytes)
|
||||||
|
&& event.artifact_size_bytes == Some(streamed_bytes)
|
||||||
|
&& event
|
||||||
|
.artifact_path
|
||||||
|
.as_ref()
|
||||||
|
.map(artifact_id_from_path)
|
||||||
|
.as_ref()
|
||||||
|
== Some(artifact)
|
||||||
|
})
|
||||||
|
.map(|event| BASE64_STANDARD.encode(event.stdout_tail.as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
fn record_debug_audit_event(
|
fn record_debug_audit_event(
|
||||||
&mut self,
|
&mut self,
|
||||||
tenant: TenantId,
|
tenant: TenantId,
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,9 @@
|
||||||
|
|
||||||
const assert = require("assert");
|
const assert = require("assert");
|
||||||
const cp = require("child_process");
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
const net = require("net");
|
const net = require("net");
|
||||||
|
const os = require("os");
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
|
|
||||||
const repo = path.resolve(__dirname, "..");
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
|
@ -111,6 +113,20 @@ function nodeCapabilities() {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function runJson(command, args, options = {}) {
|
||||||
|
const result = cp.spawnSync(command, args, {
|
||||||
|
cwd: repo,
|
||||||
|
encoding: "utf8",
|
||||||
|
...options,
|
||||||
|
});
|
||||||
|
if (result.status !== 0) {
|
||||||
|
throw new Error(
|
||||||
|
`${command} ${args.join(" ")} failed with code ${result.status}\n${result.stderr}\n${result.stdout}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return JSON.parse(result.stdout);
|
||||||
|
}
|
||||||
|
|
||||||
async function reportNode(addr, node, { directConnectivity = true, online = true } = {}) {
|
async function reportNode(addr, node, { directConnectivity = true, online = true } = {}) {
|
||||||
const response = await send(addr, {
|
const response = await send(addr, {
|
||||||
type: "report_node_capabilities",
|
type: "report_node_capabilities",
|
||||||
|
|
@ -192,6 +208,41 @@ async function reportNode(addr, node, { directConnectivity = true, online = true
|
||||||
assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true);
|
assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true);
|
||||||
assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false);
|
assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false);
|
||||||
assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/);
|
assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.strictEqual(exportPlan.artifact_size_bytes, produced.stdout_bytes);
|
||||||
|
|
||||||
|
const temp = fs.mkdtempSync(path.join(os.tmpdir(), "disasmer-artifact-export-"));
|
||||||
|
const exportPath = path.join(temp, "export-output.txt");
|
||||||
|
const cliExport = runJson("cargo", [
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"disasmer-cli",
|
||||||
|
"--bin",
|
||||||
|
"disasmer",
|
||||||
|
"--",
|
||||||
|
"artifact",
|
||||||
|
"export",
|
||||||
|
"--coordinator",
|
||||||
|
`${addr.host}:${addr.port}`,
|
||||||
|
"--tenant",
|
||||||
|
"tenant",
|
||||||
|
"--project-id",
|
||||||
|
"project",
|
||||||
|
"--user",
|
||||||
|
"user",
|
||||||
|
"--json",
|
||||||
|
"export-output.txt",
|
||||||
|
"--receiver-node",
|
||||||
|
"node-export-receiver",
|
||||||
|
"--to",
|
||||||
|
exportPath,
|
||||||
|
]);
|
||||||
|
assert.strictEqual(cliExport.command, "artifact export");
|
||||||
|
assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true);
|
||||||
|
assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written");
|
||||||
|
assert.strictEqual(cliExport.export_plan.bytes_written, produced.stdout_bytes);
|
||||||
|
assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false);
|
||||||
|
assert.strictEqual(fs.readFileSync(exportPath, "utf8"), produced.stdout_tail);
|
||||||
|
|
||||||
const crossTenant = await send(addr, {
|
const crossTenant = await send(addr, {
|
||||||
type: "export_artifact_to_node",
|
type: "export_artifact_to_node",
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,11 @@ expect(
|
||||||
"CLI-first non-e2e gate wording",
|
"CLI-first non-e2e gate wording",
|
||||||
/scripts\/acceptance-cli-first\.sh[\s\S]*final public-release e2e remains intentionally excluded/
|
/scripts\/acceptance-cli-first\.sh[\s\S]*final public-release e2e remains intentionally excluded/
|
||||||
);
|
);
|
||||||
|
expect(
|
||||||
|
criteria,
|
||||||
|
"artifact export explicit local byte write",
|
||||||
|
/artifact export <id> --to <path>` writes bytes[\s\S]*explicit bounded download stream[\s\S]*complete staged content is available/
|
||||||
|
);
|
||||||
expect(
|
expect(
|
||||||
cliFirstAcceptance,
|
cliFirstAcceptance,
|
||||||
"CLI-first acceptance report",
|
"CLI-first acceptance report",
|
||||||
|
|
@ -330,6 +335,21 @@ expect(
|
||||||
"CLI blocks build before scheduling unsafe inputs",
|
"CLI blocks build before scheduling unsafe inputs",
|
||||||
/blocked_before_schedule[\s\S]*scheduled_work[\s\S]*false/
|
/blocked_before_schedule[\s\S]*scheduled_work[\s\S]*false/
|
||||||
);
|
);
|
||||||
|
expect(
|
||||||
|
cli,
|
||||||
|
"CLI artifact export writes explicit local bytes from stream content",
|
||||||
|
/fn artifact_export_local_write_followup[\s\S]*open_artifact_download_stream[\s\S]*content_base64[\s\S]*std::fs::write/
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
cli,
|
||||||
|
"CLI artifact export keeps content out of reports",
|
||||||
|
/fn artifact_stream_summary[\s\S]*content_material_returned_in_report[\s\S]*false/
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
coordinator,
|
||||||
|
"coordinator exposes conservative artifact stream content",
|
||||||
|
/fn download_stream_content_base64[\s\S]*stdout_truncated[\s\S]*artifact_size_bytes[\s\S]*BASE64_STANDARD\.encode/
|
||||||
|
);
|
||||||
|
|
||||||
for (const [name, pattern] of [
|
for (const [name, pattern] of [
|
||||||
["agent --json flag", /struct AgentEnrollArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
["agent --json flag", /struct AgentEnrollArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue