.gitignore

@@ -1,8 +1,5 @@
 /target/
+/web/target/
 /.clusterflux/
 **/.clusterflux/
 /vscode-extension/node_modules/
-/private/*/Cargo.lock
-!/private/hosted-policy/Cargo.lock
-/private/*/target/
-/scripts/containers-home/

CLUSTERFLUX_PUBLIC_TREE.json

@@ -1,22 +0,0 @@
-{
-  "kind": "clusterflux-filtered-public-tree",
-  "source_commit": "98d969b26488b4cda8b2381fa870276a00ca98ea",
-  "release_name": "release-98d969b26488",
-  "filtered_out": [
-    "private/**",
-    "internal/**",
-    "experiments/**",
-    ".git",
-    "target",
-    "git-ignored source paths",
-    "root/*.md except README.md",
-    "**/.clusterflux/**",
-    ".forgejo/**"
-  ],
-  "public_export": {
-    "host_neutral": true,
-    "include_forgejo_workflows": false
-  },
-  "forgejo_host": "git.michelpaulissen.com",
-  "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
-}

Cargo.lock

@@ -290,6 +290,20 @@ dependencies = [
  "wasmparser 0.245.1",
 ]
 
+[[package]]
+name = "clusterflux-client"
+version = "0.1.0"
+dependencies = [
+ "base64",
+ "clusterflux-control",
+ "clusterflux-coordinator",
+ "clusterflux-core",
+ "serde",
+ "serde_json",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
 [[package]]
 name = "clusterflux-control"
 version = "0.1.0"
@@ -967,6 +981,13 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hello-build"
+version = "0.1.0"
+dependencies = [
+ "clusterflux-sdk",
+]
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -1144,16 +1165,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "launch-build-demo"
-version = "0.1.0"
-dependencies = [
- "clusterflux-sdk",
- "futures-executor",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -1670,6 +1681,14 @@ dependencies = [
  "yasna",
 ]
 
+[[package]]
+name = "recovery-build"
+version = "0.1.0"
+dependencies = [
+ "clusterflux-sdk",
+ "serde",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 resolver = "2"
+exclude = ["web"]
 members = [
   "crates/clusterflux-cli",
+  "crates/clusterflux-client",
   "crates/clusterflux-control",
   "crates/clusterflux-coordinator",
   "crates/clusterflux-core",
@@ -10,13 +12,14 @@ members = [
   "crates/clusterflux-node",
   "crates/clusterflux-sdk",
   "crates/clusterflux-wasm-runtime",
-  "examples/launch-build-demo",
+    "examples/hello-build",
+    "examples/recovery-build",
 ]
 
 [workspace.package]
 edition = "2021"
 license = "Apache-2.0 OR MIT"
-repository = "https://git.michelpaulissen.com/michel/clusterflux-public"
+repository = "https://clusterflux.lesstuff.com"
 
 [workspace.dependencies]
 anyhow = "1.0"

README.md

@@ -1,14 +1,82 @@
 # Clusterflux
 
-Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
+Clusterflux runs a Rust-defined workflow as one distributed virtual process. The
-coordinator hosts the async main, while attached nodes execute Wasm tasks,
+async main runs serverless, provisioning nodes to run tasks through rootless
-rootless containers, and native commands. Tasks exchange canonical values and
+containers. Tasks on nodes exchange data simply and efficiently.
-portable typed handles instead of sharing host memory.
+
+The user experience is built to be as much as possible like building a regular
+program. The processes are debuggable as normal through a debugger adapter.
+
+The primary use case is to consolidate build processes into a single streamlined
+developer experience. An example program can be seen below:
+
+~~~rust
+use clusterflux::prelude::*;
+
+#[clusterflux::task(capabilities = "command")]
+pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
+    let executable = fs::output("hello-clusterflux")?;
+    Command::new("cc")
+        .args([
+            "-Os",
+            "-static",
+            "-s",
+            "fixture/hello-clusterflux.c",
+            "-o",
+            executable.as_str(),
+        ])
+        .cwd(source.mount()?)
+        .env("SOURCE_DATE_EPOCH", "0")
+        .network_disabled()
+        .run()
+        .await?;
+    fs::publish(&executable).await
+}
+
+#[clusterflux::main]
+pub async fn build() -> Result<Artifact> {
+    let source = source::current_project().snapshot().await?;
+    let compile = clusterflux::spawn!(compile(source))
+        .on(clusterflux::env!("linux"))
+        .await?;
+    compile.join().await
+}
+~~~
+
+After setup, this build pipeline could be deployed as easily as launching it
+through your IDE. This repository includes a VS Code extension to make
+development as straightforward as possible. A full collection of CLI tools is
+included for advanced usage.
+
+Clusterflux is explicitly local-first. It is trivial to provision existing
+hardware as resources. Bulk data will typically not leave the local network,
+allowing maximum throughput. The same capability, however, also makes it
+possible to leverage cloud resources easily.
 
 Start with [Getting started](docs/getting-started.md). It takes you through
 authentication, project setup, node enrollment, a run, debugging, task restart,
 and artifact download.
 
+## Build a real executable
+
+The primary example is intentionally small: [hello-build](examples/hello-build)
+snapshots its source project, spawns one container-backed compile task, and
+publishes the resulting executable as a retained artifact.
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The final command prints `hello from a real Clusterflux build`. The source uses
+only the public SDK path: current-project snapshot, `spawn!`, `Command::run`, and
+artifact publication. See [recovery-build](examples/recovery-build) for two
+same-definition task instances, a real command failure, and operator restart.
+
 ## What you get
 
 - One virtual process with distinct task instances and restart attempts.
@@ -29,6 +97,9 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Rootless Podman is required on Linux nodes that build or run a declared
 Containerfile environment. Install VS Code when you want the graphical debug
 workflow.
@@ -54,8 +125,8 @@ explicit-key option.
 Choose an entrypoint and run it:
 
 ~~~bash
-clusterflux bundle inspect --project examples/launch-build-demo
+clusterflux bundle inspect --project examples/hello-build
-clusterflux run --project examples/launch-build-demo build
+clusterflux run --project examples/hello-build build
 ~~~
 
 Inspect the result:
@@ -65,7 +136,8 @@ clusterflux process status
 clusterflux task list
 clusterflux logs
 clusterflux artifact list
-clusterflux artifact download app.txt --to ./app.txt
+# Use the `artifact` value returned by the list command, for example:
+clusterflux artifact download hello-clusterflux-4f61c2... --to ./hello-clusterflux
 ~~~
 
 To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md).

SECURITY.md

@@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained.
 
 ## Report a vulnerability
 
-Email security@michelpaulissen.com with a concise description, affected version
+Email security@clusterflux.lesstuff.com with a concise description, affected version
 or source revision, reproduction steps, and impact. Do not open a public issue
 for an unpatched vulnerability or include credentials, session tokens, private
 keys, provider tokens, customer data, or operator secrets in a public report.

crates/clusterflux-cli/src/admin.rs

@@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
                 .unwrap_or(json!(false)),
             "response": response,
             "safe_default": "read_only",
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
         "command": "admin status",
         "mode": "self_hosted_local",
         "safe_default": "read_only",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
         "project": project.clone(),
         "user": user,
         "coordinator": scope.coordinator,
-        "private_website_required": false,
+        "external_website_required": false,
         "self_hosted_cli_only": true,
         "project_config_written": project_init
             .get("project_config_written")
@@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
             {
                 "step": "start_self_hosted_coordinator",
                 "command": "clusterflux-coordinator --listen 127.0.0.1:0",
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_or_link_project",
                 "command": "clusterflux project init --yes",
                 "completed": true,
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_node_enrollment_grant",
@@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "attach_worker_node",
@@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "run_process",
@@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "inspect_status_logs_artifacts",
@@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux artifact list",
                     "clusterflux quota status",
                 ],
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "revoke_access",
                 "command": "clusterflux admin revoke-node --node <node-id> --yes",
-                "private_website_required": false,
+                "external_website_required": false,
             }
         ],
     }))
@@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
             "actor_tenant": actor_tenant,
             "actor_user": actor_user,
             "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
         "status": "requires_coordinator",
         "requires_confirmation": !args.yes,
         "tenant": tenant,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 

crates/clusterflux-cli/src/artifact.rs

@@ -12,6 +12,7 @@ use crate::client::{
 };
 use crate::config::StoredCliSession;
 use crate::errors::cli_error_summary_for_category;
+use crate::process::hydrate_process_scope;
 use crate::process_events::{
     artifact_download_grant_disclosures, artifact_download_session_summary,
     artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
@@ -27,9 +28,10 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
 }
 
 pub(crate) fn artifact_list_report_with_session(
-    args: ArtifactListArgs,
+    mut args: ArtifactListArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,
@@ -53,9 +55,10 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Val
 }
 
 pub(crate) fn artifact_download_report_with_session(
-    args: ArtifactDownloadArgs,
+    mut args: ArtifactDownloadArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(
@@ -143,9 +146,10 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value>
 }
 
 pub(crate) fn artifact_export_report_with_session(
-    args: ArtifactExportArgs,
+    mut args: ArtifactExportArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(

crates/clusterflux-cli/src/auth.rs

@@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<V
                 "suspension_known": false,
                 "account_state_known": false,
                 "account_status": "unknown",
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
             })
         });
@@ -263,7 +263,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -296,7 +296,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -316,7 +316,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -338,7 +338,7 @@ fn coordinator_auth_status_summary(
             "account_status": "unknown",
             "suspension_known": false,
             "account_state_known": false,
-            "private_moderation_details_exposed": false,
+            "sensitive_moderation_details_exposed": false,
             "signup_failure_details_exposed": false,
             "machine_error": cli_error_summary(message),
             "coordinator_response_type": "error",
@@ -403,7 +403,7 @@ fn coordinator_auth_status_summary(
         "manual_review": manual_review,
         "sanitized_reason": sanitized_reason,
         "next_actions": next_actions,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
         "coordinator_session_requests": coordinator_session_requests,

crates/clusterflux-cli/src/auth_scope.rs

@@ -60,7 +60,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
         "account_status": "unknown",
         "suspension_known": false,
         "account_state_known": false,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "next_actions": ["clusterflux login --browser"],
     })

crates/clusterflux-cli/src/config.rs

@@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::CliScopeArgs;
 
-pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
+pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com";
-    "https://clusterflux.michelpaulissen.com";
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub(crate) struct ProjectConfig {

crates/clusterflux-cli/src/debug.rs

@@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
         "command": "dap",
         "adapter": dap_binary_path()?.display().to_string(),
         "args": args.args,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
                 .cloned()
                 .unwrap_or_else(|| json!(0)),
             "debug_reads_quota_limited": true,
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
         "dap": dap,
         "authorized": "unknown_without_coordinator",
         "debug_reads_quota_limited": "unknown_without_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }

crates/clusterflux-cli/src/errors.rs

@@ -38,7 +38,10 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s
             );
             object.insert("community_tier_language".to_owned(), json!(true));
             object.insert("community_tier_label".to_owned(), json!("community tier"));
-            object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
+            object.insert(
+                "sensitive_abuse_heuristics_exposed".to_owned(),
+                json!(false),
+            );
         }
     }
     summary

crates/clusterflux-cli/src/logs.rs

@@ -3,6 +3,7 @@ use serde_json::{json, Value};
 
 use crate::client::list_task_events_if_available_with_session;
 use crate::config::StoredCliSession;
+use crate::process::hydrate_process_scope;
 use crate::process_events::log_entries;
 use crate::LogsArgs;
 
@@ -12,9 +13,10 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
 }
 
 pub(crate) fn logs_report_with_session(
-    args: LogsArgs,
+    mut args: LogsArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,

crates/clusterflux-cli/src/node.rs

@@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
             "tenant": tenant,
             "project": project,
             "user": user,
-            "private_website_required": false,
+            "external_website_required": false,
             "enrollment_grant": enrollment_grant,
             "response": response,
             "coordinator_session_requests": session.requests(),
@@ -161,7 +161,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
     Ok(json!({
         "command": "node enroll",
         "status": "requires_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
         "requested_ttl_seconds": args.ttl_seconds,
         "enrollment_grant": null,
         "reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
@@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
     };
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &tenant,
+        "project": &project,
         "node": &plan.node,
     });
     let heartbeat_signature = sign_node_request(

crates/clusterflux-cli/src/output.rs

@@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         }
         push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
         if let Some(exposed) = account
-            .get("private_moderation_details_exposed")
+            .get("sensitive_moderation_details_exposed")
             .and_then(Value::as_bool)
         {
-            lines.push(format!("private moderation details exposed: {exposed}"));
+            lines.push(format!("sensitive moderation details exposed: {exposed}"));
         }
     }
     if let Some(coordinator_selection) = value.get("coordinator") {
@@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         ));
     }
     if let Some(flag) = value
-        .get("private_website_required")
+        .get("external_website_required")
         .and_then(Value::as_bool)
     {
-        lines.push(format!("private website required: {flag}"));
+        lines.push(format!("external website required: {flag}"));
     }
     if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
         let actions = next_actions

crates/clusterflux-cli/src/process.rs

@@ -15,7 +15,10 @@ use crate::{
     ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
 };
 
-fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
+pub(crate) fn hydrate_process_scope(
+    scope: &mut CliScopeArgs,
+    stored_session: Option<&StoredCliSession>,
+) {
     if scope.coordinator.is_none() {
         scope.coordinator = stored_session
             .filter(|session| session.session_secret.is_some())

crates/clusterflux-cli/src/process_events.rs

@@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
         "cross_tenant_reuse_allowed": false,
         "unauthorized_project_reuse_allowed": false,
         "default_durable_store_assumed": false,
-        "private_website_required": false,
+        "external_website_required": false,
     }])
 }
 
@@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<
         "current_usage": current_usage,
         "limits": quota_limits_value(),
         "next_blocked_action": next_blocked_action,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
     })
 }
 

crates/clusterflux-cli/src/project.rs

@@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
     Ok(json!({
         "command": "project init",
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config_written": true,
         "project_config_write_after_coordinator_acceptance": coordinator.is_some(),
         "coordinator_create_before_local_write": coordinator.is_some(),
@@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "config_format": "clusterflux_project_config_v1",
             "links_current_directory": true,
             "writes_current_directory_only": true,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "safe_defaults": {
             "tenant": config.tenant.clone(),
@@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "default_project_id_used": args.new_project == "project",
             "default_project_name_used": args.name == "Clusterflux Project",
             "browser_interaction_required": false,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "project_config": config,
         "config_file": project_config_file(&cwd),
@@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
             "user": user,
             "projects": projects,
             "project_count": project_count,
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
         "source": "local_project_config",
         "projects": projects,
         "project_count": project_count,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
         "selected_project": selected_project,
         "project_config_written": true,
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config": config,
         "coordinator_response": coordinator_response,
     }))

crates/clusterflux-cli/src/quota.rs

@@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "user": effective_scope.user,
         "coordinator": coordinator,
         "project_config": config,
-        "policy_surface": "generic public quota categories; hosted tuning remains private policy",
+        "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined",
         "limits": limits,
         "window_seconds": window_seconds,
         "current_usage": current_usage,
@@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "next_blocked_action": quota_next_blocked_action(&current_usage),
         "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
         "quota_tier": quota_tier,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
         "quota_response": quota_status,
     }))
 }

crates/clusterflux-cli/src/run.rs

@@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
         "safe_failure": true,
         "message": message,
         "next_actions": next_actions,
-        "private_website_required": false,
+        "external_website_required": false,
         "machine_error": machine_error,
     })
 }
@@ -195,12 +195,14 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         );
     }
     let process = "vp-current".to_owned();
+    let launch_attempt = new_launch_attempt_id();
     let mut session = JsonLineSession::connect(&coordinator)?;
     let mut request = json!({
         "type": "start_process",
         "tenant": tenant.clone(),
         "project": project.clone(),
         "process": process.clone(),
+        "launch_attempt": launch_attempt.clone(),
         "restart": false,
     });
     request = authenticated_human_or_local_trusted_workflow(
@@ -219,6 +221,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         &tenant,
         &project,
         &process,
+        &launch_attempt,
     )?;
     let run_start = run_start_summary(&response);
     let status = run_start
@@ -286,6 +289,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
                     tenant: &tenant,
                     project: &project,
                     process: &process,
+                    launch_attempt: &launch_attempt,
                 };
                 let cleanup = rollback_failed_process_launch_reconnecting(
                     &coordinator,
@@ -325,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         "task_launch": launch_task_response,
         "coordinator_response": response,
         "coordinator_session_requests": session.requests(),
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -340,6 +344,7 @@ fn request_process_start_with_rollback(
     tenant: &str,
     project: &str,
     process: &str,
+    launch_attempt: &str,
 ) -> Result<Value> {
     let rollback = ProcessLaunchRollback {
         cli_session,
@@ -348,9 +353,25 @@ fn request_process_start_with_rollback(
         tenant,
         project,
         process,
+        launch_attempt,
     };
     match session.request_allow_error(request) {
-        Ok(response) => Ok(response),
+        Ok(response) => {
+            if response.get("type").and_then(Value::as_str) == Some("process_started")
+                && response.get("launch_attempt").and_then(Value::as_str) != Some(launch_attempt)
+            {
+                let ownership_error = anyhow::anyhow!(
+                    "coordinator returned a process-start acknowledgement owned by a different launch attempt"
+                );
+                rollback_failed_process_launch_reconnecting(
+                    coordinator,
+                    &rollback,
+                    &json!({ "error": ownership_error.to_string(), "phase": "start_process_ownership" }),
+                )?;
+                return Err(ownership_error);
+            }
+            Ok(response)
+        }
         Err(start_error) => {
             let cleanup = rollback_failed_process_launch_reconnecting(
                 coordinator,
@@ -370,6 +391,17 @@ fn request_process_start_with_rollback(
     }
 }
 
+static NEXT_LAUNCH_ATTEMPT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+
+fn new_launch_attempt_id() -> String {
+    let sequence = NEXT_LAUNCH_ATTEMPT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_nanos();
+    format!("launch-{}-{nanos}-{sequence}", std::process::id())
+}
+
 struct ProcessLaunchRollback<'a> {
     cli_session: &'a CliSession,
     fallback_user: &'a str,
@@ -377,6 +409,7 @@ struct ProcessLaunchRollback<'a> {
     tenant: &'a str,
     project: &'a str,
     process: &'a str,
+    launch_attempt: &'a str,
 }
 
 fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> {
@@ -402,6 +435,7 @@ fn rollback_failed_process_launch(
             "tenant": context.tenant,
             "project": context.project,
             "process": context.process,
+            "launch_attempt": context.launch_attempt,
         }),
         context.cli_session,
         context.fallback_user,
@@ -983,6 +1017,10 @@ mod transactional_launch_tests {
                 payload.get("process").and_then(Value::as_str),
                 Some("vp-current")
             );
+            assert_eq!(
+                payload.get("launch_attempt").and_then(Value::as_str),
+                Some("launch-test")
+            );
             writeln!(
                 stream,
                 "{}",
@@ -1003,6 +1041,7 @@ mod transactional_launch_tests {
             tenant: "tenant-a",
             project: "project-a",
             process: "vp-current",
+            launch_attempt: "launch-test",
         };
         rollback_failed_process_launch(
             &mut session,
@@ -1024,6 +1063,7 @@ mod transactional_launch_tests {
                 .read_line(&mut start_line)
                 .unwrap();
             assert!(start_line.contains("\"type\":\"start_process\""));
+            assert!(start_line.contains("\"launch_attempt\":\"launch-test\""));
             drop(stream);
 
             let (mut cleanup_stream, _) = listener.accept().unwrap();
@@ -1032,6 +1072,7 @@ mod transactional_launch_tests {
                 .read_line(&mut cleanup_line)
                 .unwrap();
             assert!(cleanup_line.contains("\"type\":\"abort_process\""));
+            assert!(cleanup_line.contains("\"launch_attempt\":\"launch-test\""));
             writeln!(
                 cleanup_stream,
                 "{}",
@@ -1043,6 +1084,12 @@ mod transactional_launch_tests {
                 })
             )
             .unwrap();
+            listener.set_nonblocking(true).unwrap();
+            std::thread::sleep(std::time::Duration::from_millis(25));
+            assert!(matches!(
+                listener.accept().unwrap_err().kind(),
+                std::io::ErrorKind::WouldBlock
+            ));
         });
         let mut session = JsonLineSession::connect(&address).unwrap();
         let error = request_process_start_with_rollback(
@@ -1053,6 +1100,7 @@ mod transactional_launch_tests {
                 "project": "project-a",
                 "actor_user": "user-a",
                 "process": "vp-current",
+                "launch_attempt": "launch-test",
                 "restart": false
             }),
             &address,
@@ -1062,10 +1110,63 @@ mod transactional_launch_tests {
             "tenant-a",
             "project-a",
             "vp-current",
+            "launch-test",
         )
         .unwrap_err()
         .to_string();
         assert!(error.contains("closed") || error.contains("response"));
         server.join().unwrap();
     }
+
+    #[test]
+    fn definitive_start_rejection_does_not_abort_existing_process() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap().to_string();
+        let server = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().unwrap();
+            let mut line = String::new();
+            std::io::BufReader::new(stream.try_clone().unwrap())
+                .read_line(&mut line)
+                .unwrap();
+            writeln!(
+                stream,
+                "{}",
+                json!({
+                    "type": "error",
+                    "message": "project already has active virtual process vp-existing"
+                })
+            )
+            .unwrap();
+            listener.set_nonblocking(true).unwrap();
+            std::thread::sleep(std::time::Duration::from_millis(25));
+            assert_eq!(
+                listener.accept().unwrap_err().kind(),
+                std::io::ErrorKind::WouldBlock
+            );
+        });
+        let mut session = JsonLineSession::connect(&address).unwrap();
+        let response = request_process_start_with_rollback(
+            &mut session,
+            json!({
+                "type": "start_process",
+                "tenant": "tenant-a",
+                "project": "project-a",
+                "actor_user": "user-a",
+                "process": "vp-current",
+                "launch_attempt": "launch-rejected",
+                "restart": false
+            }),
+            &address,
+            &CliSession::Anonymous,
+            "user-a",
+            None,
+            "tenant-a",
+            "project-a",
+            "vp-current",
+            "launch-rejected",
+        )
+        .unwrap();
+        assert_eq!(response.get("type").and_then(Value::as_str), Some("error"));
+        server.join().unwrap();
+    }
 }

crates/clusterflux-cli/src/tests.rs

@@ -10,6 +10,15 @@ fn parse(args: &[&str]) -> Cli {
     Cli::parse_from(args)
 }
 
+fn launch_attempt_from_wire(line: &str) -> String {
+    let wire: Value = serde_json::from_str(line).unwrap();
+    wire.pointer("/payload/request/launch_attempt")
+        .or_else(|| wire.pointer("/payload/launch_attempt"))
+        .and_then(Value::as_str)
+        .expect("start request must carry a launch attempt")
+        .to_owned()
+}
+
 fn write_runnable_wasm_project(project: &Path) {
     let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
     let sdk = cli_manifest_dir.parent().unwrap().join("clusterflux-sdk");
@@ -115,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
                 assert_eq!(summary["resource_category"], "api_calls");
                 assert_eq!(summary["community_tier_language"], true);
                 assert_eq!(summary["community_tier_label"], "community tier");
-                assert_eq!(summary["private_abuse_heuristics_exposed"], false);
+                assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false);
                 let rendered = human_report(&json!({
                     "command": "run",
                     "machine_error": summary,
@@ -198,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
     assert_eq!(report["status"], "authentication_required");
     assert_eq!(report["non_interactive"], true);
     assert_eq!(report["browser_opened"], false);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["machine_error"]["category"], "authentication");
     assert_eq!(report["machine_error"]["stable_exit_code"], 20);
     assert_eq!(report["machine_error"]["browser_opened"], false);
@@ -438,10 +447,11 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
         assert!(line.contains(r#""nonce":"agent-signature-"#));
         assert!(line.contains(r#""signature":"ed25519:"#));
         assert!(!line.contains(r#""actor_user""#));
+        let launch_attempt = launch_attempt_from_wire(&line);
         stream
                 .write_all(
                     format!(
-                        r#"{{"type":"process_started","process":"vp-current","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
+                        r#"{{"type":"process_started","process":"vp-current","launch_attempt":"{launch_attempt}","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
                     )
                     .as_bytes(),
                 )
@@ -588,7 +598,32 @@ fn run_with_human_session_derives_workflow_scope_from_authenticated_envelope() {
             assert!(wire["payload"]["request"].get("tenant").is_none());
             assert!(wire["payload"]["request"].get("project").is_none());
             assert!(wire["payload"]["request"].get("actor_user").is_none());
+            if expected_operation == "start_process" {
+                let launch_attempt = launch_attempt_from_wire(&line);
+                stream
+                    .write_all(
+                        serde_json::to_string(&json!({
+                            "type": "process_started",
+                            "process": "vp-current",
+                            "launch_attempt": launch_attempt,
+                            "epoch": 7,
+                            "actor": {
+                                "kind": "user",
+                                "user": "user-session",
+                                "agent": null,
+                                "credential_kind": "cli_device_session",
+                                "public_key_fingerprint": null,
+                                "authenticated_without_browser": false,
+                                "scopes": ["project:read", "project:run"]
+                            }
+                        }))
+                        .unwrap()
+                        .as_bytes(),
+                    )
+                    .unwrap();
+            } else {
                 stream.write_all(response).unwrap();
+            }
             stream.write_all(b"\n").unwrap();
         }
     });
@@ -661,6 +696,18 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
                 assert!(line.contains(r#""project":"project-live""#));
                 assert!(line.contains(r#""process":"vp-current""#));
                 assert!(line.contains(r#""restart":false"#));
+                let response = if index == 0 {
+                    let launch_attempt = launch_attempt_from_wire(&line);
+                    serde_json::to_string(&json!({
+                        "type": "process_started",
+                        "process": "vp-current",
+                        "launch_attempt": launch_attempt,
+                        "epoch": 7,
+                    }))
+                    .unwrap()
+                } else {
+                    response.to_owned()
+                };
                 stream.write_all(response.as_bytes()).unwrap();
                 stream.write_all(b"\n").unwrap();
                 if index == 0 {
@@ -780,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() {
     assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
     assert_eq!(rejected["machine_error"]["community_tier_language"], true);
     assert_eq!(
-        rejected["machine_error"]["private_abuse_heuristics_exposed"],
+        rejected["machine_error"]["sensitive_abuse_heuristics_exposed"],
         false
     );
     assert!(rejected["machine_error"]["next_actions"]
@@ -1618,12 +1665,11 @@ fn node_attach_refuses_a_symlink_credential_target() {
 fn hosted_coordinator_remains_a_real_https_control_endpoint() {
     assert_eq!(
         control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
+        "https://clusterflux.lesstuff.com/api/v1/control"
     );
     assert_eq!(
-        control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
+        control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
-            .unwrap(),
+        "https://clusterflux.lesstuff.com/api/v1/control"
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
     );
     assert!(control_endpoint_identity("http://operator.example.test").is_err());
     assert_eq!(
@@ -1741,7 +1787,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         assert!(!line.contains(r#""actor_user":"user-session""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1804,7 +1850,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         "active"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
 }
@@ -1886,7 +1932,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
 }
 
 #[test]
-fn auth_status_queries_coordinator_account_state_without_private_moderation_details() {
+fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() {
     let listener = TcpListener::bind("127.0.0.1:0").unwrap();
     let addr = listener.local_addr().unwrap().to_string();
     let server = std::thread::spawn(move || {
@@ -1900,7 +1946,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         assert!(line.contains(r#""actor_user":"user-live""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1940,7 +1986,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         "account or tenant is suspended by hosted policy"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
     assert_eq!(
@@ -1958,13 +2004,13 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
     let serialized = serde_json::to_string(&report).unwrap();
     assert!(!serialized.contains("abuse_score"));
     assert!(!serialized.contains("moderation_notes"));
-    assert!(!serialized.contains("private moderation note"));
+    assert!(!serialized.contains("sensitive moderation note"));
 
     let rendered = human_report(&report);
     assert!(rendered.contains("account status: suspended"));
     assert!(rendered.contains("account suspended: true"));
-    assert!(rendered.contains("private moderation details exposed: false"));
+    assert!(rendered.contains("sensitive moderation details exposed: false"));
-    assert!(!rendered.contains("private moderation note"));
+    assert!(!rendered.contains("sensitive moderation note"));
 }
 
 #[test]
@@ -2015,11 +2061,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
                     "manual_review": manual_review,
                     "sanitized_reason": reason,
                     "next_actions": ["contact the hosted operator"],
-                    "private_moderation_details_exposed": false,
+                    "sensitive_moderation_details_exposed": false,
                     "signup_failure_details_exposed": false,
                     "abuse_score": 99,
-                    "moderation_notes": "private moderation note",
+                    "moderation_notes": "sensitive moderation note",
-                    "signup_policy_trace": "private signup trace",
+                    "signup_policy_trace": "sensitive signup trace",
                 })
             )
             .unwrap();
@@ -2058,7 +2104,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
             true
         );
         assert_eq!(
-            report["coordinator_account_status"]["private_moderation_details_exposed"],
+            report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
             false
         );
         assert_eq!(
@@ -2069,11 +2115,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
         assert!(!serialized.contains("abuse_score"));
         assert!(!serialized.contains("moderation_notes"));
         assert!(!serialized.contains("signup_policy_trace"));
-        assert!(!serialized.contains("private moderation note"));
+        assert!(!serialized.contains("sensitive moderation note"));
         let rendered = human_report(&report);
         assert!(rendered.contains(&format!("account status: {status}")));
         assert!(rendered.contains(rendered_marker));
-        assert!(!rendered.contains("private moderation note"));
+        assert!(!rendered.contains("sensitive moderation note"));
     }
     server.join().unwrap();
 }
@@ -2188,11 +2234,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
 
     assert_eq!(report["command"], "admin bootstrap");
     assert_eq!(report["mode"], "self_hosted_local");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["self_hosted_cli_only"], true);
     assert_eq!(report["project_config_written"], true);
     assert_eq!(report["project_init"]["command"], "project init");
-    assert_eq!(report["project_init"]["private_website_required"], false);
+    assert_eq!(report["project_init"]["external_website_required"], false);
     assert_eq!(
         report["project_init"]["project_config"]["project"],
         "self-hosted"
@@ -2218,7 +2264,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
     }
     assert!(steps.iter().all(|step| {
         !step
-            .get("private_website_required")
+            .get("external_website_required")
             .and_then(Value::as_bool)
             .unwrap_or(false)
     }));
@@ -2846,13 +2892,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
 
     assert_eq!(status["command"], "admin status");
     assert_eq!(status["safe_default"], "read_only");
-    assert_eq!(status["private_website_required"], false);
+    assert_eq!(status["external_website_required"], false);
     assert_eq!(status["suspended"], false);
     assert_eq!(suspended["command"], "admin suspend-tenant");
     assert_eq!(suspended["tenant"], "tenant");
     assert_eq!(suspended["actor_tenant"], "admin-tenant");
     assert_eq!(suspended["suspended"], true);
-    assert_eq!(suspended["private_website_required"], false);
+    assert_eq!(suspended["external_website_required"], false);
 }
 
 #[test]
@@ -2921,7 +2967,7 @@ fn debug_attach_reports_public_authorization() {
     assert_eq!(report["charged_debug_read_bytes"], 1024);
     assert_eq!(report["used_debug_read_bytes"], 1024);
     assert_eq!(report["debug_reads_quota_limited"], true);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
 }
 
 #[test]
@@ -2958,6 +3004,14 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
                 "restart_task",
                 br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
             ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(),
+            ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(),
+            ),
             (
                 "create_artifact_download_link",
                 br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
@@ -3062,9 +3116,26 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
-    artifact_download_report_with_session(
+    let logs = logs_report_with_session(
+        LogsArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+            task: Some("task-a".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let artifacts = artifact_list_report_with_session(
+        ArtifactListArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let download = artifact_download_report_with_session(
         ArtifactDownloadArgs {
-            scope: coordinator_scope.clone(),
+            scope,
             artifact: "app.txt".to_owned(),
             to: None,
             max_bytes: 2048,
@@ -3072,6 +3143,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
+    assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n");
+    assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt");
+    assert_eq!(download["coordinator"], session.coordinator);
     debug_attach_report_with_dap_and_session(
         DebugAttachArgs {
             scope: coordinator_scope,
@@ -3264,7 +3338,7 @@ fn project_init_uses_public_create_before_writing_local_config() {
         created["safe_defaults"]["browser_interaction_required"],
         false
     );
-    assert_eq!(created["private_website_required"], false);
+    assert_eq!(created["external_website_required"], false);
     assert_eq!(
         read_project_config(temp_success.path())
             .unwrap()
@@ -3364,7 +3438,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(list["source"], "public_coordinator_api");
     assert_eq!(list["project_count"], 1);
     assert_eq!(list["projects"][0]["id"], "project-a");
-    assert_eq!(list["private_website_required"], false);
+    assert_eq!(list["external_website_required"], false);
     assert_eq!(list["coordinator_session_requests"], 1);
 
     let selected = project_select_report(
@@ -3379,7 +3453,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(selected["source"], "public_coordinator_api");
     assert_eq!(selected["selected_project"]["id"], "project-a");
     assert_eq!(selected["project_config_written"], true);
-    assert_eq!(selected["private_website_required"], false);
+    assert_eq!(selected["external_website_required"], false);
     assert_eq!(
         read_project_config(temp.path()).unwrap().unwrap().project,
         "project-a"
@@ -4408,7 +4482,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     let temp = tempfile::tempdir().unwrap();
     let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .join("../..")
-        .join("examples/launch-build-demo");
+        .join("examples/hello-build");
     let output = temp.path().join("bundle");
 
     let report = build_report(
@@ -4426,8 +4500,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     assert_eq!(report["command"], "build");
     assert_eq!(report["content_addressed"], true);
     assert_eq!(report["contains_full_repository_upload"], false);
-    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 10);
+    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2);
-    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 5);
+    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1);
     assert!(output.join("module.wasm").is_file());
     assert!(output.join("manifest.json").is_file());
     assert!(output.join("task-descriptors.json").is_file());
@@ -4438,9 +4512,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
         .unwrap()
         .iter()
         .any(|descriptor| {
-            descriptor["name"] == "task_add_one"
+            descriptor["name"] == "compile"
-                && descriptor["argument_schema"] == "input : i32"
+                && descriptor["argument_schema"] == "source : SourceSnapshot"
-                && descriptor["result_schema"] == "i32"
+                && descriptor["result_schema"] == "Result < Artifact >"
                 && descriptor["restart_compatibility_hash"]
                     .as_str()
                     .unwrap()
@@ -4601,7 +4675,7 @@ fn node_enroll_reports_short_lived_public_api_grant() {
 
     assert_eq!(report["command"], "node enroll");
     assert_eq!(report["status"], "created");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["tenant"], "tenant");
     assert_eq!(report["project"], "project");
     assert_eq!(report["user"], "user");
@@ -4744,7 +4818,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() {
     )
     .unwrap();
     assert_eq!(enroll["status"], "requires_coordinator");
-    assert_eq!(enroll["private_website_required"], false);
+    assert_eq!(enroll["external_website_required"], false);
     assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
     assert_eq!(enroll["requested_ttl_seconds"], 60);
 

crates/clusterflux-client/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "clusterflux-client"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+clusterflux-control = { path = "../clusterflux-control" }
+clusterflux-core = { path = "../clusterflux-core" }
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["sync", "time"] }
+
+[dev-dependencies]
+clusterflux-coordinator = { path = "../clusterflux-coordinator" }

crates/clusterflux-client/README.md

@@ -0,0 +1,68 @@
+# clusterflux-client
+
+`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web
+backend. It talks to the same versioned control and login endpoints as the CLI.
+Callers do not construct JSON envelopes or place session secrets into request
+objects.
+
+The crate intentionally contains no coordinator implementation, persistence
+types, scheduler policy, or website-specific business rules. A web backend only
+needs this crate to authenticate and work with account status, projects, Agent
+keys, node enrollment and liveness, current/recent processes, task attempts,
+recent logs, artifacts, quota state, process control, task recovery, and Debug
+Epoch state. It does not expose workflow launch or whole-process replay.
+
+```rust,no_run
+use clusterflux_client::{ClusterfluxClient, SessionCredential};
+
+# async fn example() -> Result<(), clusterflux_client::ClientError> {
+let credential = SessionCredential::from_secret(
+    std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"),
+);
+let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")?
+    .with_session_credential(&credential);
+
+let account = client.account_status().await?;
+let nodes = client.list_nodes().await?;
+let processes = client.list_processes(None, 20).await?;
+# let _ = (account, nodes, processes);
+# Ok(())
+# }
+```
+
+## Login boundary
+
+`begin_browser_login` starts the hosted Authentik flow. The identity callback
+returns only to the fixed configured website URL with a short-lived handoff.
+The website backend calls `exchange_browser_login_handoff` once and stores the
+returned `SessionCredential` only in encrypted server-side session storage.
+The credential has redacted `Debug` output and deliberately does not implement
+Serde serialization. Logout calls the existing session-revocation operation.
+
+The browser must not receive or persist the Clusterflux session credential,
+provider tokens, authorization code, PKCE verifier, or CLI polling secret.
+
+## Errors, bounds, and transport
+
+API failures are returned as `ClientError::Api(ApiError)`. Decisions should use
+the stable `code`, `category`, `retryable`, and `request_id` fields, while
+`message` is for people. The client rejects an error whose request ID does not
+match the originating request.
+
+Paginated list methods require an explicit bounded page size. Process pages are
+limited to 100 entries; node, artifact, and recent-log pages to 200.
+`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes
+its cursor. Recent logs are ephemeral and can report truncation or sequence
+loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates
+offsets and decodes bounded chunks without materializing the complete artifact.
+
+`ControlTransport` has bounded connect and I/O timeouts and performs blocking
+network work outside the async executor. Dropping a request future cancels the
+caller’s wait; any already-started blocking I/O remains bounded by its timeout.
+`MockTransport` supplies deterministic responses and records exact envelopes
+for application tests.
+
+`CLIENT_API_VERSION` is the one supported control protocol version. The
+checked-in `web_operations.json` contract fixture records every website
+operation and its stable error shape, and the crate’s contract suite checks the
+fixture.

crates/clusterflux-client/src/lib.rs

@@ -0,0 +1,924 @@
+mod protocol;
+mod transport;
+mod types;
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION};
+use protocol::{AuthenticatedRequest, LoginRequest, WireResponse};
+use serde_json::json;
+use thiserror::Error;
+use transport::{ClientTransport, TransportRequest};
+
+pub use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind,
+    Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId,
+    ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId,
+    UserId, VfsPath,
+};
+pub use transport::{
+    ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse,
+};
+pub use types::*;
+
+pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION;
+
+#[derive(Debug, Error)]
+pub enum ClientError {
+    #[error("Clusterflux API error: {0}")]
+    Api(ApiError),
+    #[error(transparent)]
+    Transport(#[from] ClientTransportError),
+    #[error("Clusterflux client protocol error: {0}")]
+    Protocol(String),
+}
+
+#[derive(Clone)]
+pub struct ClusterfluxClient {
+    transport: Arc<dyn ClientTransport>,
+    session_secret: Arc<Mutex<Option<String>>>,
+    next_request: Arc<AtomicU64>,
+}
+
+impl ClusterfluxClient {
+    pub fn connect(endpoint: impl Into<String>) -> Result<Self, ClientError> {
+        Ok(Self::with_transport(ControlTransport::new(endpoint)?))
+    }
+
+    pub fn with_transport(transport: impl ClientTransport) -> Self {
+        Self {
+            transport: Arc::new(transport),
+            session_secret: Arc::new(Mutex::new(None)),
+            next_request: Arc::new(AtomicU64::new(1)),
+        }
+    }
+
+    pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self {
+        self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone())));
+        self
+    }
+
+    pub fn is_session_configured(&self) -> bool {
+        self.session_secret
+            .lock()
+            .map(|secret| secret.is_some())
+            .unwrap_or(false)
+    }
+
+    pub async fn begin_browser_login(&self) -> Result<BrowserLoginStart, ClientError> {
+        match self
+            .send_login(LoginRequest::BeginWebBrowserLogin {})
+            .await?
+        {
+            WireResponse::WebBrowserLoginStarted {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            } => Ok(BrowserLoginStart {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_login_started")),
+        }
+    }
+
+    pub async fn exchange_browser_login_handoff(
+        &self,
+        transaction_id: impl Into<String>,
+        handoff_code: impl Into<String>,
+    ) -> Result<BrowserSession, ClientError> {
+        match self
+            .send_login(LoginRequest::ExchangeWebLoginHandoff {
+                transaction_id: transaction_id.into(),
+                handoff_code: handoff_code.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserSession { session } => Ok(BrowserSession {
+                tenant: session.tenant,
+                project: session.project,
+                user: session.user,
+                credential: SessionCredential(session.session_secret),
+                expires_at_epoch_seconds: session.expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_session")),
+        }
+    }
+
+    pub async fn cancel_browser_login(
+        &self,
+        transaction_id: impl Into<String>,
+    ) -> Result<(), ClientError> {
+        match self
+            .send_login(LoginRequest::CancelWebBrowserLogin {
+                transaction_id: transaction_id.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserLoginCancelled {} => Ok(()),
+            _ => Err(unexpected_response("web_browser_login_cancelled")),
+        }
+    }
+
+    pub async fn account_status(&self) -> Result<AccountStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AuthStatus)
+            .await?
+        {
+            WireResponse::AuthStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            } => Ok(AccountStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            }),
+            _ => Err(unexpected_response("auth_status")),
+        }
+    }
+
+    pub async fn logout(&self) -> Result<(), ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeCliSession)
+            .await?
+        {
+            WireResponse::CliSessionRevoked {} => {
+                *self.session_secret.lock().map_err(|_| {
+                    ClientError::Protocol("session credential lock was poisoned".to_owned())
+                })? = None;
+                Ok(())
+            }
+            _ => Err(unexpected_response("cli_session_revoked")),
+        }
+    }
+
+    pub async fn list_projects(&self) -> Result<Vec<Project>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProjects)
+            .await?
+        {
+            WireResponse::Projects { projects } => Ok(projects),
+            _ => Err(unexpected_response("projects")),
+        }
+    }
+
+    pub async fn create_project(
+        &self,
+        project: ProjectId,
+        name: impl Into<String>,
+    ) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateProject {
+                project,
+                name: name.into(),
+            })
+            .await?
+        {
+            WireResponse::ProjectCreated { project } => Ok(project),
+            _ => Err(unexpected_response("project_created")),
+        }
+    }
+
+    pub async fn select_project(&self, project: ProjectId) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::SelectProject { project })
+            .await?
+        {
+            WireResponse::ProjectSelected { project } => Ok(project),
+            _ => Err(unexpected_response("project_selected")),
+        }
+    }
+
+    pub async fn register_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn list_agent_public_keys(&self) -> Result<Vec<AgentPublicKey>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys)
+            .await?
+        {
+            WireResponse::AgentPublicKeys { records } => Ok(records),
+            _ => Err(unexpected_response("agent_public_keys")),
+        }
+    }
+
+    pub async fn rotate_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn revoke_agent_public_key(
+        &self,
+        agent: AgentId,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent })
+            .await
+    }
+
+    async fn agent_key_mutation(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<AgentPublicKey, ClientError> {
+        match self.send_authenticated(request).await? {
+            WireResponse::AgentPublicKey { record } => Ok(record),
+            _ => Err(unexpected_response("agent_public_key")),
+        }
+    }
+
+    pub async fn create_node_enrollment_grant(
+        &self,
+        ttl_seconds: u64,
+    ) -> Result<NodeEnrollmentGrant, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds })
+            .await?
+        {
+            WireResponse::NodeEnrollmentGrantCreated {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            } => Ok(NodeEnrollmentGrant {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("node_enrollment_grant_created")),
+        }
+    }
+
+    pub async fn list_nodes(&self) -> Result<Vec<NodeSummary>, ClientError> {
+        Ok(self.list_nodes_page(None, 200).await?.nodes)
+    }
+
+    pub async fn list_nodes_page(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<NodePage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::NodeSummaries { nodes, next_cursor } => {
+                Ok(NodePage { nodes, next_cursor })
+            }
+            _ => Err(unexpected_response("node_summaries")),
+        }
+    }
+
+    pub async fn revoke_node(&self, node: NodeId) -> Result<NodeRevocation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node })
+            .await?
+        {
+            WireResponse::NodeCredentialRevoked {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            } => Ok(NodeRevocation {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            }),
+            _ => Err(unexpected_response("node_credential_revoked")),
+        }
+    }
+
+    pub async fn list_processes(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ProcessPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::ProcessSummaries {
+                processes,
+                next_cursor,
+            } => Ok(ProcessPage {
+                processes,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("process_summaries")),
+        }
+    }
+
+    pub async fn cancel_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CancelProcess { process })
+            .await?
+        {
+            WireResponse::ProcessCancellationRequested {
+                process,
+                cancelled_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: cancelled_tasks,
+                affected_nodes,
+                aborted: false,
+            }),
+            _ => Err(unexpected_response("process_cancellation_requested")),
+        }
+    }
+
+    pub async fn abort_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AbortProcess {
+                process,
+                launch_attempt: None,
+            })
+            .await?
+        {
+            WireResponse::ProcessAborted {
+                process,
+                aborted_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: aborted_tasks,
+                affected_nodes,
+                aborted: true,
+            }),
+            _ => Err(unexpected_response("process_aborted")),
+        }
+    }
+
+    pub async fn quota_status(&self) -> Result<QuotaStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::QuotaStatus)
+            .await?
+        {
+            WireResponse::QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            } => Ok(QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("quota_status")),
+        }
+    }
+
+    pub async fn list_task_events(
+        &self,
+        process: Option<ProcessId>,
+    ) -> Result<Vec<TaskCompletionEvent>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskEvents { process })
+            .await?
+        {
+            WireResponse::TaskEvents { events } => Ok(events),
+            _ => Err(unexpected_response("task_events")),
+        }
+    }
+
+    pub async fn list_task_snapshots(
+        &self,
+        process: ProcessId,
+    ) -> Result<Vec<TaskAttemptSnapshot>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process })
+            .await?
+        {
+            WireResponse::TaskSnapshots { snapshots } => Ok(snapshots),
+            _ => Err(unexpected_response("task_snapshots")),
+        }
+    }
+
+    pub async fn list_recent_logs(
+        &self,
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<RecentLogPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::RecentLogs {
+                entries,
+                next_sequence,
+                history_truncated,
+            } => Ok(RecentLogPage {
+                entries,
+                next_sequence,
+                history_truncated,
+            }),
+            _ => Err(unexpected_response("recent_logs")),
+        }
+    }
+
+    pub async fn restart_task(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    ) -> Result<TaskRestart, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RestartTask {
+                process,
+                task,
+                replacement_bundle,
+            })
+            .await?
+        {
+            WireResponse::TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            } => Ok(TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("task_restart")),
+        }
+    }
+
+    pub async fn resolve_task_failure(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    ) -> Result<TaskFailureResolutionResult, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResolveTaskFailure {
+                process,
+                task,
+                resolution,
+            })
+            .await?
+        {
+            WireResponse::TaskFailureResolved {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            } => Ok(TaskFailureResolutionResult {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            }),
+            _ => Err(unexpected_response("task_failure_resolved")),
+        }
+    }
+
+    pub async fn debug_attach(&self, process: ProcessId) -> Result<DebugAttach, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::DebugAttach { process })
+            .await?
+        {
+            WireResponse::DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            } => Ok(DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_attach")),
+        }
+    }
+
+    pub async fn create_debug_epoch(
+        &self,
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: impl Into<String>,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateDebugEpoch {
+                process,
+                stopped_task,
+                reason: reason.into(),
+            })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn resume_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn inspect_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            } => Ok(DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch_status")),
+        }
+    }
+
+    pub async fn list_artifacts(
+        &self,
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ArtifactPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::Artifacts {
+                artifacts,
+                next_cursor,
+            } => Ok(ArtifactPage {
+                artifacts,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("artifacts")),
+        }
+    }
+
+    pub async fn get_artifact(&self, artifact: ArtifactId) -> Result<ArtifactSummary, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::GetArtifact { artifact })
+            .await?
+        {
+            WireResponse::Artifact { artifact } => Ok(artifact),
+            _ => Err(unexpected_response("artifact")),
+        }
+    }
+
+    pub async fn begin_artifact_download(
+        &self,
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+        chunk_bytes: u64,
+    ) -> Result<ArtifactDownload, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink {
+                artifact: artifact.clone(),
+                max_bytes,
+                ttl_seconds,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload {
+                client: self.clone(),
+                artifact,
+                max_bytes,
+                chunk_bytes,
+                token_digest: link.scoped_token_digest.clone(),
+                link,
+                expected_offset: 0,
+                finished: false,
+            }),
+            _ => Err(unexpected_response("artifact_download_link")),
+        }
+    }
+
+    async fn send_authenticated(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<WireResponse, ClientError> {
+        let session_secret = self
+            .session_secret
+            .lock()
+            .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))?
+            .clone()
+            .ok_or_else(|| {
+                ClientError::Api(ApiError::from_message(
+                    "client",
+                    "no authenticated Clusterflux session is configured",
+                ))
+            })?;
+        self.send(
+            CONTROL_API_PATH,
+            json!({
+                "type": "authenticated",
+                "session_secret": session_secret,
+                "request": request,
+            }),
+        )
+        .await
+    }
+
+    async fn send_login(&self, request: LoginRequest) -> Result<WireResponse, ClientError> {
+        let payload = serde_json::to_value(request)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        self.send(LOGIN_API_PATH, payload).await
+    }
+
+    async fn send(
+        &self,
+        api_path: &str,
+        payload: serde_json::Value,
+    ) -> Result<WireResponse, ClientError> {
+        let request_number = self.next_request.fetch_add(1, Ordering::Relaxed);
+        let request_id = format!("client-{request_number}");
+        let envelope = coordinator_wire_request(&request_id, payload);
+        let body = serde_json::to_vec(&envelope)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        let response = self
+            .transport
+            .send(TransportRequest {
+                api_path: api_path.to_owned(),
+                body,
+            })
+            .await?;
+        let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| {
+            ClientError::Protocol(format!("decode typed API response: {error}"))
+        })?;
+        match response {
+            WireResponse::Error {
+                code,
+                category,
+                message,
+                retryable,
+                request_id: response_request_id,
+            } => {
+                if response_request_id != request_id {
+                    return Err(ClientError::Protocol(format!(
+                        "error response request_id {response_request_id} does not match {request_id}"
+                    )));
+                }
+                Err(ClientError::Api(ApiError::new(
+                    code,
+                    category,
+                    message,
+                    retryable,
+                    response_request_id,
+                )))
+            }
+            response => Ok(response),
+        }
+    }
+}
+
+pub struct ArtifactDownload {
+    client: ClusterfluxClient,
+    artifact: ArtifactId,
+    max_bytes: u64,
+    chunk_bytes: u64,
+    token_digest: Digest,
+    link: clusterflux_core::DownloadLink,
+    expected_offset: u64,
+    finished: bool,
+}
+
+impl ArtifactDownload {
+    pub fn link(&self) -> &clusterflux_core::DownloadLink {
+        &self.link
+    }
+
+    pub async fn next_chunk(&mut self) -> Result<ArtifactDownloadPoll, ClientError> {
+        if self.finished {
+            return Ok(ArtifactDownloadPoll::Chunk {
+                offset: self.expected_offset,
+                bytes: Vec::new(),
+                eof: true,
+            });
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream {
+                artifact: self.artifact.clone(),
+                max_bytes: self.max_bytes,
+                token_digest: self.token_digest.clone(),
+                chunk_bytes: self.chunk_bytes,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadStream {
+                content_bytes_available,
+                content_offset,
+                content_eof,
+                content_base64,
+                ..
+            } => {
+                if !content_bytes_available {
+                    return Ok(ArtifactDownloadPoll::Pending);
+                }
+                let offset = content_offset.ok_or_else(|| {
+                    ClientError::Protocol(
+                        "artifact response contained bytes without an offset".to_owned(),
+                    )
+                })?;
+                if offset != self.expected_offset {
+                    return Err(ClientError::Protocol(format!(
+                        "artifact chunk offset {offset} does not match expected {}",
+                        self.expected_offset
+                    )));
+                }
+                let bytes = BASE64_STANDARD
+                    .decode(content_base64.unwrap_or_default())
+                    .map_err(|error| {
+                        ClientError::Protocol(format!(
+                            "artifact response contains invalid base64: {error}"
+                        ))
+                    })?;
+                self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64);
+                self.finished = content_eof;
+                Ok(ArtifactDownloadPoll::Chunk {
+                    offset,
+                    bytes,
+                    eof: content_eof,
+                })
+            }
+            _ => Err(unexpected_response("artifact_download_stream")),
+        }
+    }
+
+    pub async fn cancel(mut self) -> Result<(), ClientError> {
+        if self.finished {
+            return Ok(());
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink {
+                artifact: self.artifact.clone(),
+                token_digest: self.token_digest.clone(),
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLinkRevoked {} => {
+                self.finished = true;
+                Ok(())
+            }
+            _ => Err(unexpected_response("artifact_download_link_revoked")),
+        }
+    }
+}
+
+fn unexpected_response(expected: &str) -> ClientError {
+    ClientError::Protocol(format!(
+        "expected typed response {expected}, received another response variant"
+    ))
+}

crates/clusterflux-client/src/protocol.rs

@@ -0,0 +1,403 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink,
+    LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId,
+};
+use serde::{Deserialize, Serialize};
+
+use crate::types::*;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum AuthenticatedRequest {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject {
+        project: ProjectId,
+        name: String,
+    },
+    SelectProject {
+        project: ProjectId,
+    },
+    ListProjects,
+    RegisterAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    ListAgentPublicKeys,
+    RotateAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        agent: AgentId,
+    },
+    CreateNodeEnrollmentGrant {
+        ttl_seconds: u64,
+    },
+    ListNodeSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    RevokeNodeCredential {
+        node: NodeId,
+    },
+    ListProcessSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    CancelProcess {
+        process: ProcessId,
+    },
+    AbortProcess {
+        process: ProcessId,
+        launch_attempt: Option<String>,
+    },
+    QuotaStatus,
+    ListTaskEvents {
+        process: Option<ProcessId>,
+    },
+    ListTaskSnapshots {
+        process: ProcessId,
+    },
+    ListRecentLogs {
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    },
+    RestartTask {
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+    },
+    CreateDebugEpoch {
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    ListArtifacts {
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    },
+    GetArtifact {
+        artifact: ArtifactId,
+    },
+    CreateArtifactDownloadLink {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        artifact: ArtifactId,
+        token_digest: Digest,
+    },
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum LoginRequest {
+    BeginWebBrowserLogin {},
+    CancelWebBrowserLogin {
+        transaction_id: String,
+    },
+    ExchangeWebLoginHandoff {
+        transaction_id: String,
+        handoff_code: String,
+    },
+}
+
+#[derive(Clone, Deserialize)]
+pub(crate) struct WireBrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub session_secret: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum WireResponse {
+    AuthStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        authenticated: bool,
+        account_status: String,
+        suspended: bool,
+        disabled: bool,
+        deleted: bool,
+        manual_review: bool,
+        sanitized_reason: Option<String>,
+        next_actions: Vec<String>,
+    },
+    CliSessionRevoked {},
+    ProjectCreated {
+        project: Project,
+    },
+    ProjectSelected {
+        project: Project,
+    },
+    Projects {
+        projects: Vec<Project>,
+    },
+    AgentPublicKey {
+        record: AgentPublicKey,
+    },
+    AgentPublicKeys {
+        records: Vec<AgentPublicKey>,
+    },
+    NodeEnrollmentGrantCreated {
+        tenant: TenantId,
+        project: ProjectId,
+        grant: String,
+        scope: String,
+        expires_at_epoch_seconds: u64,
+    },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+    },
+    NodeCredentialRevoked {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        descriptor_removed: bool,
+        queued_assignments_removed: usize,
+    },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+    },
+    ProcessCancellationRequested {
+        process: ProcessId,
+        cancelled_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessAborted {
+        process: ProcessId,
+        aborted_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    QuotaStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        policy_label: Option<String>,
+        limits: ResourceLimits,
+        window_seconds: BTreeMap<LimitKind, u64>,
+        usage: BTreeMap<LimitKind, u64>,
+        window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+    },
+    TaskEvents {
+        events: Vec<TaskCompletionEvent>,
+    },
+    TaskSnapshots {
+        snapshots: Vec<TaskAttemptSnapshot>,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
+    TaskRestart {
+        process: ProcessId,
+        task: TaskInstanceId,
+        restarted_task_instance: Option<TaskInstanceId>,
+        restarted_attempt_id: Option<String>,
+        actor: UserId,
+        accepted: bool,
+        clean_boundary_available: bool,
+        active_task: bool,
+        completed_event_observed: bool,
+        requires_whole_process_restart: bool,
+        message: String,
+        audit_event: DebugAuditEvent,
+    },
+    TaskFailureResolved {
+        process: ProcessId,
+        task: TaskInstanceId,
+        attempt_id: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+        actor: UserId,
+        authorization: Authorization,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpoch {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        affected_tasks: Vec<TaskCancellationTarget>,
+        all_stop_requested: bool,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpochStatus {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        expected_tasks: Vec<TaskCancellationTarget>,
+        acknowledgements: Vec<DebugParticipantAcknowledgement>,
+        fully_frozen: bool,
+        partially_frozen: bool,
+        fully_resumed: bool,
+        failed: bool,
+        failure_messages: Vec<String>,
+        audit_event: DebugAuditEvent,
+    },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
+    ArtifactDownloadLink {
+        link: DownloadLink,
+    },
+    ArtifactDownloadLinkRevoked {},
+    ArtifactDownloadStream {
+        #[serde(rename = "link")]
+        _link: DownloadLink,
+        content_bytes_available: bool,
+        content_offset: Option<u64>,
+        content_eof: bool,
+        content_base64: Option<String>,
+    },
+    WebBrowserLoginStarted {
+        transaction_id: String,
+        authorization_url: String,
+        expires_at_epoch_seconds: u64,
+    },
+    WebBrowserLoginCancelled {},
+    WebBrowserSession {
+        session: WireBrowserSession,
+    },
+    Error {
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: String,
+        retryable: bool,
+        request_id: String,
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use serde::Deserialize;
+    use serde_json::Value;
+
+    use super::*;
+
+    #[derive(Deserialize)]
+    struct OperationFixture {
+        operation: String,
+        boundary: String,
+        request: Value,
+        response: Value,
+    }
+
+    #[test]
+    fn website_operation_contract_fixtures_cover_every_typed_request() {
+        let fixtures: Vec<OperationFixture> =
+            serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap();
+        let expected = BTreeSet::from([
+            "abort_process",
+            "auth_status",
+            "begin_web_browser_login",
+            "cancel_web_browser_login",
+            "cancel_process",
+            "create_artifact_download_link",
+            "create_debug_epoch",
+            "create_node_enrollment_grant",
+            "create_project",
+            "debug_attach",
+            "exchange_web_login_handoff",
+            "get_artifact",
+            "inspect_debug_epoch",
+            "list_agent_public_keys",
+            "list_artifacts",
+            "list_node_summaries",
+            "list_process_summaries",
+            "list_projects",
+            "list_recent_logs",
+            "list_task_events",
+            "list_task_snapshots",
+            "open_artifact_download_stream",
+            "quota_status",
+            "register_agent_public_key",
+            "resolve_task_failure",
+            "restart_task",
+            "resume_debug_epoch",
+            "revoke_agent_public_key",
+            "revoke_artifact_download_link",
+            "revoke_cli_session",
+            "revoke_node_credential",
+            "rotate_agent_public_key",
+            "select_project",
+        ])
+        .into_iter()
+        .map(str::to_owned)
+        .collect();
+        let mut observed = BTreeSet::new();
+
+        for fixture in fixtures {
+            assert_eq!(fixture.request["type"], fixture.operation);
+            let round_trip = match fixture.boundary.as_str() {
+                "control" => {
+                    let typed: AuthenticatedRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                "login" => {
+                    let typed: LoginRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                boundary => panic!("unknown fixture boundary {boundary}"),
+            };
+            assert_eq!(round_trip, fixture.request);
+            let response: WireResponse = serde_json::from_value(fixture.response).unwrap();
+            assert!(
+                !matches!(response, WireResponse::Error { .. }),
+                "{} must carry its operation-specific success response fixture",
+                fixture.operation
+            );
+            assert!(observed.insert(fixture.operation));
+        }
+        assert_eq!(observed, expected);
+    }
+}

crates/clusterflux-client/src/transport.rs

@@ -0,0 +1,250 @@
+use std::collections::{BTreeMap, VecDeque};
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use clusterflux_control::{endpoint_identity, ControlSession};
+use thiserror::Error;
+
+pub type TransportFuture =
+    Pin<Box<dyn Future<Output = Result<TransportResponse, ClientTransportError>> + Send + 'static>>;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportRequest {
+    pub api_path: String,
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportResponse {
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ClientTransportError {
+    #[error("client transport failed: {0}")]
+    Failed(String),
+    #[error("client transport task failed: {0}")]
+    Task(String),
+}
+
+pub trait ClientTransport: Send + Sync + 'static {
+    fn send(&self, request: TransportRequest) -> TransportFuture;
+}
+
+type ControlSessionPool = Arc<Vec<Mutex<Option<ControlSession>>>>;
+
+pub struct ControlTransport {
+    endpoint: String,
+    connect_timeout: Duration,
+    io_timeout: Duration,
+    sessions: Arc<Mutex<BTreeMap<String, ControlSessionPool>>>,
+    next_session: Arc<AtomicU64>,
+}
+
+// A small pool allows independent HTMX reads to progress concurrently while
+// keeping connection and blocking-worker use strictly bounded.
+const SESSIONS_PER_API_PATH: usize = 4;
+
+impl ControlTransport {
+    pub fn new(endpoint: impl Into<String>) -> Result<Self, ClientTransportError> {
+        Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
+    }
+
+    pub fn with_timeouts(
+        endpoint: impl Into<String>,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ClientTransportError> {
+        let endpoint = endpoint.into();
+        endpoint_identity(&endpoint)
+            .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+        Ok(Self {
+            endpoint,
+            connect_timeout,
+            io_timeout,
+            sessions: Arc::new(Mutex::new(BTreeMap::new())),
+            next_session: Arc::new(AtomicU64::new(0)),
+        })
+    }
+}
+
+impl ClientTransport for ControlTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let endpoint = self.endpoint.clone();
+        let connect_timeout = self.connect_timeout;
+        let io_timeout = self.io_timeout;
+        let sessions = Arc::clone(&self.sessions);
+        let next_session = Arc::clone(&self.next_session);
+        Box::pin(async move {
+            let pool = {
+                let mut sessions = sessions.lock().map_err(|_| {
+                    ClientTransportError::Failed(
+                        "client transport pool lock was poisoned".to_owned(),
+                    )
+                })?;
+                Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| {
+                    Arc::new(
+                        (0..SESSIONS_PER_API_PATH)
+                            .map(|_| Mutex::new(None))
+                            .collect(),
+                    )
+                }))
+            };
+            let slot_index =
+                next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH;
+            tokio::task::spawn_blocking(move || {
+                let value = serde_json::from_slice(&request.body)
+                    .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+                let mut selected = None;
+                for offset in 0..SESSIONS_PER_API_PATH {
+                    let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                    match pool[index].try_lock() {
+                        Ok(session) if session.is_some() => {
+                            selected = Some(session);
+                            break;
+                        }
+                        Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {}
+                        Err(std::sync::TryLockError::Poisoned(_)) => {
+                            return Err(ClientTransportError::Failed(
+                                "client transport session lock was poisoned".to_owned(),
+                            ));
+                        }
+                    }
+                }
+                if selected.is_none() {
+                    for offset in 0..SESSIONS_PER_API_PATH {
+                        let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                        match pool[index].try_lock() {
+                            Ok(session) => {
+                                selected = Some(session);
+                                break;
+                            }
+                            Err(std::sync::TryLockError::WouldBlock) => {}
+                            Err(std::sync::TryLockError::Poisoned(_)) => {
+                                return Err(ClientTransportError::Failed(
+                                    "client transport session lock was poisoned".to_owned(),
+                                ));
+                            }
+                        }
+                    }
+                }
+                let mut session = match selected {
+                    Some(session) => session,
+                    None => pool[slot_index].lock().map_err(|_| {
+                        ClientTransportError::Failed(
+                            "client transport session lock was poisoned".to_owned(),
+                        )
+                    })?,
+                };
+                if session.is_none() {
+                    *session = Some(
+                        ControlSession::connect_to_api_path_with_timeouts(
+                            &endpoint,
+                            &request.api_path,
+                            connect_timeout,
+                            io_timeout,
+                        )
+                        .map_err(|error| ClientTransportError::Failed(error.to_string()))?,
+                    );
+                }
+                let response = session
+                    .as_mut()
+                    .expect("session was initialized for the requested API path")
+                    .request(&value);
+                match response {
+                    Ok(response) => serde_json::to_vec(&response)
+                        .map(|body| TransportResponse { body })
+                        .map_err(|error| ClientTransportError::Failed(error.to_string())),
+                    Err(error) => {
+                        *session = None;
+                        Err(ClientTransportError::Failed(error.to_string()))
+                    }
+                }
+            })
+            .await
+            .map_err(|error| ClientTransportError::Task(error.to_string()))?
+        })
+    }
+}
+
+#[derive(Clone, Default)]
+pub struct MockTransport {
+    state: Arc<Mutex<MockTransportState>>,
+}
+
+#[derive(Default)]
+struct MockTransportState {
+    responses: VecDeque<Result<Vec<u8>, ClientTransportError>>,
+    requests: Vec<TransportRequest>,
+}
+
+impl MockTransport {
+    pub fn from_json_responses(responses: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        let responses = responses
+            .into_iter()
+            .map(|response| Ok(response.into().into_bytes()))
+            .collect();
+        Self {
+            state: Arc::new(Mutex::new(MockTransportState {
+                responses,
+                requests: Vec::new(),
+            })),
+        }
+    }
+
+    pub fn push_json_response(&self, response: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Ok(response.into().into_bytes()));
+    }
+
+    pub fn push_error(&self, message: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Err(ClientTransportError::Failed(message.into())));
+    }
+
+    pub fn request_bodies(&self) -> Vec<String> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .iter()
+            .map(|request| String::from_utf8_lossy(&request.body).into_owned())
+            .collect()
+    }
+
+    pub fn requests(&self) -> Vec<TransportRequest> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .clone()
+    }
+}
+
+impl ClientTransport for MockTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let state = Arc::clone(&self.state);
+        Box::pin(async move {
+            let mut state = state.lock().map_err(|_| {
+                ClientTransportError::Failed("mock transport lock was poisoned".to_owned())
+            })?;
+            state.requests.push(request);
+            state
+                .responses
+                .pop_front()
+                .ok_or_else(|| {
+                    ClientTransportError::Failed("mock transport has no queued response".to_owned())
+                })?
+                .map(|body| TransportResponse { body })
+        })
+    }
+}

crates/clusterflux-client/src/types.rs

@@ -0,0 +1,468 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement,
+    ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy,
+    TaskInstanceId, TenantId, UserId, VfsPath,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AccountStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub authenticated: bool,
+    pub account_status: String,
+    pub suspended: bool,
+    pub disabled: bool,
+    pub deleted: bool,
+    pub manual_review: bool,
+    pub sanitized_reason: Option<String>,
+    pub next_actions: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Project {
+    pub id: ProjectId,
+    pub tenant: TenantId,
+    pub name: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AgentPublicKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub agent: AgentId,
+    pub public_key: String,
+    pub public_key_fingerprint: Digest,
+    pub version: u64,
+    pub revoked: bool,
+    pub scopes: Vec<String>,
+    pub human_account_creation_privilege: bool,
+    pub browser_interaction_required_each_run: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeEnrollmentGrant {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub grant: String,
+    pub scope: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodePage {
+    pub nodes: Vec<NodeSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeRevocation {
+    pub node: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub descriptor_removed: bool,
+    pub queued_assignments_removed: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessPage {
+    pub processes: Vec<ProcessSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskTerminalState {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskExecutor {
+    CoordinatorMain,
+    Node,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCompletionEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub node: NodeId,
+    pub executor: TaskExecutor,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub attempt_id: Option<String>,
+    pub placement: Option<Placement>,
+    pub terminal_state: TaskTerminalState,
+    pub status_code: Option<i32>,
+    pub stdout_bytes: u64,
+    pub stderr_bytes: u64,
+    pub stdout_tail: String,
+    pub stderr_tail: String,
+    pub stdout_truncated: bool,
+    pub stderr_truncated: bool,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub result: Option<TaskBoundaryValue>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskAttemptState {
+    Queued,
+    Running,
+    FailedAwaitingAction,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAttemptSnapshot {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub attempt_number: u32,
+    pub task_definition: TaskDefinitionId,
+    pub display_name: String,
+    pub state: TaskAttemptState,
+    pub current: bool,
+    pub node: Option<NodeId>,
+    pub environment_id: Option<String>,
+    pub environment_digest: Option<Digest>,
+    pub argument_summary: Vec<String>,
+    pub handle_summary: Vec<String>,
+    pub command_state: Option<String>,
+    pub vfs_checkpoint: String,
+    pub probe_symbol: Option<String>,
+    pub source_path: Option<String>,
+    pub source_line: Option<u32>,
+    pub restart_compatible: bool,
+    pub failure_policy: TaskFailurePolicy,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub status_code: Option<i32>,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogPage {
+    pub entries: Vec<RecentLogEntry>,
+    pub next_sequence: Option<u64>,
+    pub history_truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactPage {
+    pub artifacts: Vec<ArtifactSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct QuotaStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub policy_label: Option<String>,
+    pub limits: ResourceLimits,
+    pub window_seconds: BTreeMap<LimitKind, u64>,
+    pub usage: BTreeMap<LimitKind, u64>,
+    pub window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCancellationTarget {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessCancellation {
+    pub process: ProcessId,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub affected_nodes: Vec<NodeId>,
+    pub aborted: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAuditEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+    pub actor: UserId,
+    pub operation: String,
+    pub allowed: bool,
+    pub reason: String,
+    pub charged_debug_read_bytes: u64,
+    pub used_debug_read_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAttach {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub authorization: Authorization,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochControl {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub all_stop_requested: bool,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DebugAcknowledgementState {
+    Frozen,
+    Running,
+    Failed,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipantAcknowledgement {
+    pub node: NodeId,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub epoch: u64,
+    pub state: DebugAcknowledgementState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochStatus {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub expected_tasks: Vec<TaskCancellationTarget>,
+    pub acknowledgements: Vec<DebugParticipantAcknowledgement>,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+    pub failure_messages: Vec<String>,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskRestart {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub restarted_task_instance: Option<TaskInstanceId>,
+    pub restarted_attempt_id: Option<String>,
+    pub actor: UserId,
+    pub accepted: bool,
+    pub clean_boundary_available: bool,
+    pub active_task: bool,
+    pub completed_event_observed: bool,
+    pub requires_whole_process_restart: bool,
+    pub message: String,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskFailureResolutionResult {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub resolution: TaskFailureResolution,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskFailureResolution {
+    AcceptFailure,
+    Cancel,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskReplacementBundle {
+    pub bundle_digest: Digest,
+    pub wasm_module_base64: String,
+    pub source_snapshot: Option<Digest>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BrowserLoginStart {
+    pub transaction_id: String,
+    pub authorization_url: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct SessionCredential(pub(crate) String);
+
+impl SessionCredential {
+    pub fn from_secret(secret: impl Into<String>) -> Self {
+        Self(secret.into())
+    }
+
+    pub fn expose_secret(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Debug for SessionCredential {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("SessionCredential([REDACTED])")
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct BrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub credential: SessionCredential,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ArtifactDownloadPoll {
+    Pending,
+    Chunk {
+        offset: u64,
+        bytes: Vec<u8>,
+        eof: bool,
+    },
+}

crates/clusterflux-client/tests/client_contract.rs

@@ -0,0 +1,254 @@
+use std::net::TcpListener;
+use std::time::Duration;
+
+use clusterflux_client::{
+    ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError,
+    ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId,
+    UserId, CLIENT_API_VERSION,
+};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_coordinator::service::CoordinatorService;
+use serde_json::{json, Value};
+
+#[tokio::test]
+async fn mock_transport_exercises_typed_envelope_and_session_plumbing() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "projects",
+        "projects": [{
+            "id": "project-one",
+            "tenant": "tenant-one",
+            "name": "Project one"
+        }],
+        "actor": "user-one"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport.clone())
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, CONTROL_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION);
+    assert_eq!(envelope["request_id"], "client-1");
+    assert_eq!(envelope["operation"], "authenticated");
+    assert_eq!(envelope["payload"]["type"], "authenticated");
+    assert_eq!(envelope["payload"]["request"]["type"], "list_projects");
+    assert_eq!(envelope["payload"]["session_secret"], "test-session-secret");
+}
+
+#[tokio::test]
+async fn structured_errors_retain_machine_fields_and_originating_request_id() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "account_suspended",
+        "category": "authorization",
+        "message": "account access is suspended",
+        "retryable": false,
+        "request_id": "client-1"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Api(error) = client.account_status().await.unwrap_err() else {
+        panic!("expected typed API error");
+    };
+    assert_eq!(error.code, ApiErrorCode::AccountSuspended);
+    assert_eq!(error.category, ApiErrorCategory::Authorization);
+    assert_eq!(error.request_id, "client-1");
+    assert!(!error.retryable);
+}
+
+#[tokio::test]
+async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() {
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "web_browser_login_cancelled" }).to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport.clone());
+
+    client
+        .cancel_browser_login("login-transaction")
+        .await
+        .unwrap();
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, LOGIN_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["operation"], "cancel_web_browser_login");
+    assert_eq!(
+        envelope["payload"],
+        json!({
+            "type": "cancel_web_browser_login",
+            "transaction_id": "login-transaction"
+        })
+    );
+}
+
+#[tokio::test]
+async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "validation_error",
+        "category": "validation",
+        "message": "bad request",
+        "retryable": false,
+        "request_id": "another-request"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else {
+        panic!("expected protocol error");
+    };
+    assert!(message.contains("does not match client-1"));
+}
+
+#[test]
+fn session_credentials_are_redacted_from_debug_output() {
+    let credential = SessionCredential::from_secret("must-not-appear");
+    assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])");
+}
+
+#[tokio::test]
+async fn artifact_download_is_a_typed_bounded_chunk_stream() {
+    let link = json!({
+        "artifact": "artifact-one",
+        "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        "artifact_size_bytes": 5,
+        "source": { "RetainedNode": "node-one" },
+        "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one",
+        "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+        "expires_at_epoch_seconds": 1000,
+        "tenant": "tenant-one",
+        "project": "project-one",
+        "process": "process-one",
+        "actor": { "User": "user-one" },
+        "max_bytes": 1024,
+        "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+    });
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link.clone(),
+            "streamed_bytes": 0,
+            "charged_download_bytes": 0,
+            "content_bytes_available": false,
+            "content_eof": false
+        })
+        .to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link,
+            "streamed_bytes": 5,
+            "charged_download_bytes": 5,
+            "content_bytes_available": true,
+            "content_offset": 0,
+            "content_eof": true,
+            "content_base64": "aGVsbG8="
+        })
+        .to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+    let mut download = client
+        .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64)
+        .await
+        .unwrap();
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Pending
+    );
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Chunk {
+            offset: 0,
+            bytes: b"hello".to_vec(),
+            eof: true,
+        }
+    );
+}
+
+#[tokio::test]
+async fn typed_client_runs_against_the_real_strict_control_endpoint() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let mut service = CoordinatorService::new(7);
+        service
+            .issue_cli_session(
+                TenantId::from("tenant-one"),
+                ProjectId::from("project-one"),
+                UserId::from("user-one"),
+                "real-endpoint-session",
+                None,
+            )
+            .unwrap();
+        let (stream, _) = listener.accept().unwrap();
+        service.handle_stream(stream).unwrap();
+    });
+
+    let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}"))
+        .unwrap()
+        .with_session_credential(&SessionCredential::from_secret("real-endpoint-session"));
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+    let status = client.account_status().await.unwrap();
+    assert!(status.authenticated);
+    assert_eq!(status.actor, UserId::from("user-one"));
+
+    drop(client);
+    server.join().unwrap();
+}
+
+#[tokio::test]
+async fn concurrent_requests_expand_the_bounded_pool_without_serializing() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let handlers = (0..2)
+            .map(|_| {
+                let (stream, _) = listener.accept().unwrap();
+                std::thread::spawn(move || {
+                    let mut service = CoordinatorService::new(7);
+                    service
+                        .issue_cli_session(
+                            TenantId::from("tenant-one"),
+                            ProjectId::from("project-one"),
+                            UserId::from("user-one"),
+                            "concurrent-session",
+                            None,
+                        )
+                        .unwrap();
+                    service.handle_stream(stream).unwrap();
+                })
+            })
+            .collect::<Vec<_>>();
+        for handler in handlers {
+            handler.join().unwrap();
+        }
+    });
+
+    let transport = ControlTransport::with_timeouts(
+        format!("clusterflux+tcp://{address}"),
+        Duration::from_secs(2),
+        Duration::from_secs(5),
+    )
+    .unwrap();
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("concurrent-session"));
+    let (first, second) = tokio::join!(client.account_status(), client.account_status());
+    assert!(first.unwrap().authenticated);
+    assert!(second.unwrap().authenticated);
+
+    drop(client);
+    server.join().unwrap();
+}

crates/clusterflux-client/tests/fixtures/web_operations.json

@@ -0,0 +1,200 @@
+[
+  {
+    "operation": "begin_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "begin_web_browser_login" },
+    "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "cancel_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" },
+    "response": { "type": "web_browser_login_cancelled" }
+  },
+  {
+    "operation": "exchange_web_login_handoff",
+    "boundary": "login",
+    "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" },
+    "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } }
+  },
+  {
+    "operation": "auth_status",
+    "boundary": "control",
+    "request": { "type": "auth_status" },
+    "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] }
+  },
+  {
+    "operation": "revoke_cli_session",
+    "boundary": "control",
+    "request": { "type": "revoke_cli_session" },
+    "response": { "type": "cli_session_revoked" }
+  },
+  {
+    "operation": "create_project",
+    "boundary": "control",
+    "request": { "type": "create_project", "project": "project-new", "name": "New project" },
+    "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } }
+  },
+  {
+    "operation": "select_project",
+    "boundary": "control",
+    "request": { "type": "select_project", "project": "project-selected" },
+    "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } }
+  },
+  {
+    "operation": "list_projects",
+    "boundary": "control",
+    "request": { "type": "list_projects" },
+    "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] }
+  },
+  {
+    "operation": "register_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "list_agent_public_keys",
+    "boundary": "control",
+    "request": { "type": "list_agent_public_keys" },
+    "response": { "type": "agent_public_keys", "records": [] }
+  },
+  {
+    "operation": "rotate_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "revoke_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "create_node_enrollment_grant",
+    "boundary": "control",
+    "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 },
+    "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "list_node_summaries",
+    "boundary": "control",
+    "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 },
+    "response": { "type": "node_summaries", "nodes": [], "next_cursor": null }
+  },
+  {
+    "operation": "revoke_node_credential",
+    "boundary": "control",
+    "request": { "type": "revoke_node_credential", "node": "node-one" },
+    "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 }
+  },
+  {
+    "operation": "list_process_summaries",
+    "boundary": "control",
+    "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 },
+    "response": { "type": "process_summaries", "processes": [], "next_cursor": null }
+  },
+  {
+    "operation": "cancel_process",
+    "boundary": "control",
+    "request": { "type": "cancel_process", "process": "process-one" },
+    "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "abort_process",
+    "boundary": "control",
+    "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null },
+    "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "quota_status",
+    "boundary": "control",
+    "request": { "type": "quota_status" },
+    "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } }
+  },
+  {
+    "operation": "list_task_events",
+    "boundary": "control",
+    "request": { "type": "list_task_events", "process": "process-one" },
+    "response": { "type": "task_events", "events": [] }
+  },
+  {
+    "operation": "list_task_snapshots",
+    "boundary": "control",
+    "request": { "type": "list_task_snapshots", "process": "process-one" },
+    "response": { "type": "task_snapshots", "snapshots": [] }
+  },
+  {
+    "operation": "list_recent_logs",
+    "boundary": "control",
+    "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 },
+    "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false }
+  },
+  {
+    "operation": "restart_task",
+    "boundary": "control",
+    "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null },
+    "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resolve_task_failure",
+    "boundary": "control",
+    "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" },
+    "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" }
+  },
+  {
+    "operation": "debug_attach",
+    "boundary": "control",
+    "request": { "type": "debug_attach", "process": "process-one" },
+    "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "create_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resume_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "inspect_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "list_artifacts",
+    "boundary": "control",
+    "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 },
+    "response": { "type": "artifacts", "artifacts": [], "next_cursor": null }
+  },
+  {
+    "operation": "get_artifact",
+    "boundary": "control",
+    "request": { "type": "get_artifact", "artifact": "artifact-one" },
+    "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } }
+  },
+  {
+    "operation": "create_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 },
+    "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } }
+  },
+  {
+    "operation": "open_artifact_download_stream",
+    "boundary": "control",
+    "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 },
+    "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" }
+  },
+  {
+    "operation": "revoke_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
+    "response": { "type": "artifact_download_link_revoked" }
+  }
+]

crates/clusterflux-control/src/lib.rs

@@ -3,6 +3,8 @@ use std::io::{BufRead, BufReader, Read, Write};
 #[cfg(not(target_arch = "wasm32"))]
 use std::net::TcpStream;
 use std::net::{IpAddr, ToSocketAddrs};
+#[cfg(not(target_arch = "wasm32"))]
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
 
 use serde_json::Value;
@@ -59,13 +61,35 @@ impl ControlSession {
         endpoint: &str,
         api_path: &str,
     ) -> Result<Self, ControlTransportError> {
-        let mut session = Self::connect(endpoint)?;
+        Self::connect_to_api_path_with_timeouts(
+            endpoint,
+            api_path,
+            Duration::from_secs(10),
+            Duration::from_secs(30),
+        )
+    }
+
+    pub fn connect_to_api_path_with_timeouts(
+        endpoint: &str,
+        api_path: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ControlTransportError> {
+        let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?;
         #[cfg(not(target_arch = "wasm32"))]
+        {
+            let mut session = session;
             if let ControlTransport::Https { url, .. } = &mut session.transport {
                 *url = endpoint_api_url(endpoint, api_path)?;
             }
             Ok(session)
         }
+        #[cfg(target_arch = "wasm32")]
+        {
+            let _ = api_path;
+            Ok(session)
+        }
+    }
 
     pub fn connect_with_timeouts(
         endpoint: &str,
@@ -127,6 +151,7 @@ impl ControlSession {
             if encoded.len() > MAX_CONTROL_FRAME_BYTES {
                 return Err(ControlTransportError::FrameTooLarge);
             }
+            let inject_response_loss = should_inject_response_loss(value);
             let response = match &mut self.transport {
                 #[cfg(not(target_arch = "wasm32"))]
                 ControlTransport::Https { agent, url } => {
@@ -136,6 +161,9 @@ impl ControlSession {
                         .set("Accept", "application/json")
                         .send_bytes(&encoded)
                         .map_err(|error| ControlTransportError::Http(error.to_string()))?;
+                    if inject_response_loss {
+                        return Err(ControlTransportError::Closed);
+                    }
                     if response.status() != 200 {
                         return Err(ControlTransportError::Http(format!(
                             "coordinator returned HTTP {} {}",
@@ -157,6 +185,9 @@ impl ControlSession {
                     writer.write_all(&encoded)?;
                     writer.write_all(b"\n")?;
                     writer.flush()?;
+                    if inject_response_loss {
+                        return Err(ControlTransportError::Closed);
+                    }
                     let mut bytes = Vec::new();
                     reader
                         .take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
@@ -180,6 +211,24 @@ impl ControlSession {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+fn should_inject_response_loss(value: &Value) -> bool {
+    static INJECTED: AtomicBool = AtomicBool::new(false);
+    let Ok(expected_operation) = std::env::var("CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION")
+    else {
+        return false;
+    };
+    let operation = value
+        .pointer("/payload/request/type")
+        .or_else(|| value.pointer("/payload/type"))
+        .or_else(|| value.get("type"))
+        .and_then(Value::as_str);
+    operation == Some(expected_operation.as_str())
+        && INJECTED
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+}
+
 pub fn control_api_url(endpoint: &str) -> Result<String, ControlTransportError> {
     endpoint_api_url(endpoint, CONTROL_API_PATH)
 }

crates/clusterflux-coordinator/src/durable.rs

@@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
     pub enrollment_scope: String,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct NodeScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub node: NodeId,
+}
+
+impl NodeScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
+        Self {
+            tenant,
+            project,
+            node,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
+        Self::new(tenant.clone(), project.clone(), node.clone())
+    }
+
+    pub fn credential_subject(&self) -> String {
+        format!(
+            "node:{}:{}:{}:{}:{}:{}",
+            self.tenant.as_str().len(),
+            self.tenant,
+            self.project.as_str().len(),
+            self.project,
+            self.node.as_str().len(),
+            self.node
+        )
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct CredentialRecord {
     pub subject: String,
@@ -107,7 +140,7 @@ pub struct DurableState {
     pub tenants: BTreeMap<TenantId, TenantRecord>,
     pub users: BTreeMap<UserId, UserRecord>,
     pub projects: BTreeMap<ProjectId, ProjectRecord>,
-    pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
+    pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
     pub credentials: BTreeMap<String, CredentialRecord>,
     pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
     pub source_provider_configs:

crates/clusterflux-coordinator/src/lib.rs

@@ -13,7 +13,7 @@ pub mod service;
 mod sessions;
 pub use durable::{
     AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
-    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
+    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
     ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
     TenantRecord, UserRecord,
 };
@@ -33,6 +33,7 @@ pub use service::{
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ActiveProcess {
     pub id: ProcessId,
+    pub launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
     pub tenant: TenantId,
     pub project: ProjectId,
     pub connected_nodes: BTreeSet<NodeId>,
@@ -127,8 +128,9 @@ impl Coordinator {
         public_key: impl Into<String>,
         enrollment_scope: impl Into<String>,
     ) {
+        let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
         self.durable.node_identities.insert(
-            node.clone(),
+            key,
             NodeIdentityRecord {
                 id: node,
                 tenant,
@@ -180,10 +182,13 @@ impl Coordinator {
             public_key,
             credential.scope.clone(),
         );
+        let subject =
+            NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
+                .credential_subject();
         self.durable.credentials.insert(
-            format!("node:{node}"),
+            subject.clone(),
             CredentialRecord {
-                subject: format!("node:{node}"),
+                subject,
                 tenant: credential.tenant.clone(),
                 project: Some(credential.project.clone()),
                 kind: credential.credential_kind.clone(),
@@ -322,9 +327,20 @@ impl Coordinator {
         tenant: TenantId,
         project: ProjectId,
         id: ProcessId,
+    ) -> ActiveProcess {
+        self.start_process_for_launch_attempt(tenant, project, id, None)
+    }
+
+    pub fn start_process_for_launch_attempt(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        id: ProcessId,
+        launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
     ) -> ActiveProcess {
         let process = ActiveProcess {
             id: id.clone(),
+            launch_attempt,
             tenant,
             project,
             connected_nodes: BTreeSet::new(),
@@ -344,15 +360,12 @@ impl Coordinator {
         project: &ProjectId,
         process: &ProcessId,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
-            .ok_or(CoordinatorError::UnknownNode)?;
+        {
-        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::UnknownNode);
-            return Err(CoordinatorError::Unauthorized(
-                "node identity is outside the requested tenant/project scope".to_owned(),
-            ));
         }
         if !self
             .active_processes
@@ -367,14 +380,18 @@ impl Coordinator {
 
     pub fn reconnect_node(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         process: Option<(&ProcessId, u64)>,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
-            .ok_or(CoordinatorError::UnknownNode)?;
+        {
+            return Err(CoordinatorError::UnknownNode);
+        }
 
         if let Some((process_id, stale_epoch)) = process {
             if stale_epoch != self.coordinator_epoch {
@@ -383,11 +400,7 @@ impl Coordinator {
                     current_epoch: self.coordinator_epoch,
                 });
             }
-            let key = (
+            let key = (tenant.clone(), project.clone(), process_id.clone());
-                identity.tenant.clone(),
-                identity.project.clone(),
-                process_id.clone(),
-            );
             if let Some(active) = self.active_processes.get_mut(&key) {
                 active.connected_nodes.insert(node.clone());
             }
@@ -404,23 +417,27 @@ impl Coordinator {
         let identity = self
             .durable
             .node_identities
-            .get(node)
+            .get(&NodeScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                node,
+            ))
             .ok_or(CoordinatorError::UnknownNode)?
             .clone();
-        if identity.tenant != context.tenant || identity.project != context.project {
-            return Err(CoordinatorError::Unauthorized(
-                "node credential is outside the signed-in tenant/project scope".to_owned(),
-            ));
-        }
         if !matches!(context.actor, Actor::User(_)) {
             return Err(CoordinatorError::Unauthorized(
                 "node credential revocation requires a user identity".to_owned(),
             ));
         }
-        self.durable.node_identities.remove(node);
+        let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
-        self.durable.credentials.remove(&format!("node:{node}"));
+        self.durable.node_identities.remove(&key);
-        for active in self.active_processes.values_mut() {
+        self.durable.credentials.remove(&key.credential_subject());
-            active.connected_nodes.remove(node);
+        for active in self
+            .active_processes
+            .values_mut()
+            .filter(|active| active.tenant == context.tenant && active.project == context.project)
+        {
+            active.connected_nodes.remove(&key.node);
         }
         Ok(identity)
     }
@@ -531,6 +548,32 @@ impl Coordinator {
             .expect("active process was checked immediately before removal"))
     }
 
+    pub fn abort_process_for_launch_attempt(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        launch_attempt: &clusterflux_core::LaunchAttemptId,
+    ) -> Result<ActiveProcess, CoordinatorError> {
+        let key = (tenant.clone(), project.clone(), process.clone());
+        let active = self.active_processes.get(&key).ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "launch rollback requires an active virtual process".to_owned(),
+            )
+        })?;
+        if active.launch_attempt.as_ref() != Some(launch_attempt) {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "launch rollback denied: attempt {} does not own process {}",
+                launch_attempt.as_str(),
+                process.as_str()
+            )));
+        }
+        Ok(self
+            .active_processes
+            .remove(&key)
+            .expect("active process was checked immediately before removal"))
+    }
+
     pub fn active_process_count(&self) -> usize {
         self.active_processes.len()
     }
@@ -542,8 +585,31 @@ impl Coordinator {
             .count()
     }
 
-    pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
+    pub fn tenant_count(&self) -> usize {
-        self.durable.node_identities.get(id)
+        self.durable.tenants.len()
+    }
+
+    pub fn user_count(&self) -> usize {
+        self.durable.users.len()
+    }
+
+    pub fn project_count(&self) -> usize {
+        self.durable.projects.len()
+    }
+
+    pub fn node_identity_count(&self) -> usize {
+        self.durable.node_identities.len()
+    }
+
+    pub fn node_identity(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        id: &NodeId,
+    ) -> Option<&NodeIdentityRecord> {
+        self.durable
+            .node_identities
+            .get(&NodeScopeKey::from_refs(tenant, project, id))
     }
 
     pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@@ -645,12 +711,24 @@ mod tests {
             .contains_key(&TenantId::from("tenant")));
         assert!(restarted.durable.users.contains_key(&UserId::from("user")));
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         assert_eq!(
             restarted
                 .durable
                 .credentials
-                .get("node:node")
+                .get(&node_subject)
                 .map(|credential| &credential.kind),
             Some(&CredentialKind::NodeCredential)
         );
@@ -674,7 +752,12 @@ mod tests {
         );
         assert_eq!(rerun.coordinator_epoch, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), Some((&process, 2)))
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                Some((&process, 2)),
+            )
             .unwrap();
         assert!(restarted
             .active_process(
@@ -687,6 +770,68 @@ mod tests {
             .contains(&NodeId::from("node")));
     }
 
+    #[test]
+    fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
+        let store = InMemoryDurableStore::default();
+        let mut coordinator = Coordinator::boot(&store, 1);
+        let node = NodeId::from("shared-node");
+        let scopes = [
+            (TenantId::from("tenant-a"), ProjectId::from("project-a")),
+            (TenantId::from("tenant-b"), ProjectId::from("project-b")),
+            (TenantId::from("tenant-a"), ProjectId::from("project-c")),
+        ];
+        for (index, (tenant, project)) in scopes.iter().enumerate() {
+            let mut grant = coordinator.create_node_enrollment_grant(
+                tenant.clone(),
+                project.clone(),
+                format!("grant-{index}"),
+                "node:attach",
+                100,
+            );
+            coordinator
+                .exchange_node_enrollment_grant(
+                    &mut grant,
+                    node.clone(),
+                    &format!("public-key-{index}"),
+                    "node:attach",
+                    99,
+                )
+                .unwrap();
+        }
+
+        let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
+        let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
+        let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
+        assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
+        assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
+        assert_eq!(
+            coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
+            coordinator.durable.node_identities.get(&scope_a)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
+            coordinator.durable.node_identities.get(&scope_b)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
+            coordinator.durable.node_identities.get(&scope_c)
+        );
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_a.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_b.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_c.credential_subject()));
+        assert_eq!(coordinator.durable.node_identities.len(), 3);
+        assert_eq!(coordinator.durable.credentials.len(), 3);
+    }
+
     #[test]
     fn identical_process_ids_are_isolated_by_tenant_and_project() {
         let store = InMemoryDurableStore::default();
@@ -735,11 +880,18 @@ mod tests {
 
         let mut restarted = Coordinator::boot(&store, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), None)
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                None,
+            )
             .unwrap();
 
         let error = restarted
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -771,7 +923,13 @@ mod tests {
             .unwrap();
 
         assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
+        assert!(coordinator
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
     }
 
     #[test]
@@ -785,10 +943,16 @@ mod tests {
             "public-key",
             "node:attach",
         );
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         coordinator.durable.credentials.insert(
-            "node:node".to_owned(),
+            node_subject.clone(),
             CredentialRecord {
-                subject: "node:node".to_owned(),
+                subject: node_subject.clone(),
                 tenant: TenantId::from("tenant"),
                 project: Some(ProjectId::from("project")),
                 kind: CredentialKind::NodeCredential,
@@ -802,6 +966,8 @@ mod tests {
         );
         coordinator
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -817,7 +983,7 @@ mod tests {
                 &NodeId::from("node"),
             )
             .unwrap_err();
-        assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(foreign, CoordinatorError::UnknownNode));
 
         let revoked = coordinator
             .revoke_node_credential(
@@ -830,8 +996,14 @@ mod tests {
             )
             .unwrap();
         assert_eq!(revoked.id, NodeId::from("node"));
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
+        assert!(coordinator
-        assert!(!coordinator.durable.credentials.contains_key("node:node"));
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_none());
+        assert!(!coordinator.durable.credentials.contains_key(&node_subject));
         assert!(!coordinator
             .active_process(
                 &TenantId::from("tenant"),
@@ -890,7 +1062,7 @@ mod tests {
     }
 
     #[test]
-    fn account_policy_state_summarizes_private_admin_records_safely() {
+    fn account_policy_state_summarizes_sensitive_admin_records_safely() {
         let tenant = TenantId::from("tenant");
         let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
 
@@ -1046,7 +1218,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(error, CoordinatorError::UnknownNode));
     }
 
     #[test]

crates/clusterflux-coordinator/src/main.rs

@@ -5,17 +5,40 @@ use clusterflux_core::{ProjectId, TenantId, UserId};
 use serde_json::json;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux coordinator.\n\n\
+                 Usage: clusterflux-coordinator [OPTIONS]\n\n\
+                 Options:\n  \
+                   --listen <ADDRESS>                 [default: 127.0.0.1:0]\n  \
+                   --allow-local-trusted-loopback\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
+
     let mut listen = "127.0.0.1:0".to_owned();
     let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
         .ok()
         .as_deref()
         == Some("1");
-    let mut args = std::env::args().skip(1);
+    let mut args = raw_args.into_iter();
     while let Some(arg) = args.next() {
         if arg == "--listen" {
             listen = args.next().ok_or("--listen requires an address")?;
         } else if arg == "--allow-local-trusted-loopback" {
             allow_local_trusted = true;
+        } else {
+            return Err(format!("unknown argument: {arg}").into());
         }
     }
 

crates/clusterflux-coordinator/src/postgres_store.rs

@@ -4,8 +4,8 @@ use thiserror::Error;
 
 use crate::{
     AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
-    DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
+    DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
-    ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
+    ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
 };
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
             state.projects.insert(record.id.clone(), record);
         }
         for record in self.query_records::<NodeIdentityRecord>(
-            "SELECT record FROM clusterflux_node_identities ORDER BY node_id",
+            "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
         )? {
-            state.node_identities.insert(record.id.clone(), record);
+            state.node_identities.insert(
+                NodeScopeKey::new(
+                    record.tenant.clone(),
+                    record.project.clone(),
+                    record.id.clone(),
+                ),
+                record,
+            );
         }
         for record in self.query_records::<CredentialRecord>(
             "SELECT record FROM clusterflux_credentials ORDER BY subject",
@@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
 );
 
 CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
-    node_id TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
     project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
-    record JSONB NOT NULL
+    node_id TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, node_id)
 );
 
+DO $clusterflux_node_scope_migration$
+DECLARE
+    current_primary_key TEXT;
+    current_primary_key_columns TEXT[];
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_node_identities
+        WHERE record->>'id' IS DISTINCT FROM node_id
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration refused an internally inconsistent legacy row';
+    END IF;
+
+    SELECT
+        constraint_record.conname,
+        array_agg(attribute.attname ORDER BY key_column.ordinality)
+    INTO current_primary_key, current_primary_key_columns
+    FROM pg_constraint AS constraint_record
+    CROSS JOIN LATERAL unnest(constraint_record.conkey)
+        WITH ORDINALITY AS key_column(attribute_number, ordinality)
+    JOIN pg_attribute AS attribute
+      ON attribute.attrelid = constraint_record.conrelid
+     AND attribute.attnum = key_column.attribute_number
+    WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
+      AND constraint_record.contype = 'p'
+    GROUP BY constraint_record.conname;
+
+    IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
+        EXECUTE format(
+            'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
+            current_primary_key
+        );
+        ALTER TABLE clusterflux_node_identities
+            ADD PRIMARY KEY (tenant_id, project_id, node_id);
+    ELSIF current_primary_key_columns
+        IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
+    THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration found an unexpected primary key shape';
+    END IF;
+END
+$clusterflux_node_scope_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     subject TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     record JSONB NOT NULL
 );
 
+DO $clusterflux_node_credential_migration$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials
+        WHERE record->>'subject' IS DISTINCT FROM subject
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration refused an internally inconsistent legacy row';
+    END IF;
+
+    UPDATE clusterflux_credentials AS credential
+    SET subject = format(
+            'node:%s:%s:%s:%s:%s:%s',
+            octet_length(identity.tenant_id),
+            identity.tenant_id,
+            octet_length(identity.project_id),
+            identity.project_id,
+            octet_length(identity.node_id),
+            identity.node_id
+        ),
+        record = jsonb_set(
+            credential.record,
+            '{subject}',
+            to_jsonb(format(
+                'node:%s:%s:%s:%s:%s:%s',
+                octet_length(identity.tenant_id),
+                identity.tenant_id,
+                octet_length(identity.project_id),
+                identity.project_id,
+                octet_length(identity.node_id),
+                identity.node_id
+            )),
+            false
+        )
+    FROM clusterflux_node_identities AS identity
+    WHERE credential.subject = 'node:' || identity.node_id
+      AND credential.tenant_id = identity.tenant_id
+      AND credential.project_id = identity.project_id;
+
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials AS credential
+        WHERE credential.subject LIKE 'node:%'
+          AND NOT EXISTS (
+              SELECT 1
+              FROM clusterflux_node_identities AS identity
+              WHERE credential.tenant_id = identity.tenant_id
+                AND credential.project_id = identity.project_id
+                AND credential.subject = format(
+                    'node:%s:%s:%s:%s:%s:%s',
+                    octet_length(identity.tenant_id),
+                    identity.tenant_id,
+                    octet_length(identity.project_id),
+                    identity.project_id,
+                    octet_length(identity.node_id),
+                    identity.node_id
+                )
+          )
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration found an unscoped or orphaned node subject';
+    END IF;
+END
+$clusterflux_node_credential_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
     session_digest TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -529,7 +651,13 @@ mod tests {
         let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
 
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
         assert_eq!(restarted.active_process_count(), 0);
     }
 

crates/clusterflux-coordinator/src/service.rs

@@ -7,13 +7,14 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use clusterflux_core::{
-    Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
+    Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
-    NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
+    CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
-    RateLimit, TenantId, TransportError, UserId,
+    NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
+    TenantId, TransportError, UserId,
 };
 use thiserror::Error;
 
-use crate::{Coordinator, CoordinatorError};
+use crate::{Coordinator, CoordinatorError, NodeScopeKey};
 
 mod admin;
 mod artifacts;
@@ -34,6 +35,7 @@ mod quota;
 mod relay;
 mod routing;
 mod signed_nodes;
+mod summaries;
 mod tcp;
 mod wire_protocol;
 use authorization::authorize_authenticated_user_operation;
@@ -43,12 +45,14 @@ use keys::{
     ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
 };
 pub use protocol::{
-    ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
-    CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
-    DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
+    DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
-    TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
-    TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
+    RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
-    TaskTerminalState, VirtualProcessStatus, WorkflowActor,
+    TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
+    TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
+    VirtualProcessStatus, WorkflowActor,
 };
 pub use quota::CoordinatorQuotaConfiguration;
 pub use relay::{
@@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
 const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
 const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
 const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
-const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
+const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
 const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
 const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
+const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
+const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
+const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
+const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
+const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
+const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
 const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
 fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
     requested.clamp(1, maximum)
@@ -111,6 +121,24 @@ pub struct CoordinatorAdmission {
     pub max_artifact_download_ttl_seconds: u64,
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CoordinatorOperationalMetrics {
+    pub tenants: usize,
+    pub users: usize,
+    pub projects: usize,
+    pub enrolled_nodes: usize,
+    pub reported_nodes: usize,
+    pub live_nodes: usize,
+    pub active_processes: usize,
+    pub active_coordinator_mains: usize,
+    pub max_active_coordinator_mains: usize,
+    pub active_tasks: usize,
+    pub queued_tasks: usize,
+    pub artifacts: usize,
+    pub retained_download_links: usize,
+    pub relay: ArtifactRelayUsage,
+}
+
 impl Default for CoordinatorAdmission {
     fn default() -> Self {
         Self {
@@ -151,16 +179,139 @@ pub enum CoordinatorServiceError {
     Durable(String),
 }
 
+impl CoordinatorServiceError {
+    pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
+        let request_id = request_id.into();
+        let message = self.to_string();
+        let (code, category, retryable) = match self {
+            Self::Io(_) => (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Json(_)
+            | Self::CapabilityReport(_)
+            | Self::InvalidArtifactPath(_)
+            | Self::InvalidTaskLogTail(_) => (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            ),
+            Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
+                return ApiError::from_message(request_id, message);
+            }
+            Self::Coordinator(CoordinatorError::UnknownNode) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Coordinator(CoordinatorError::Enrollment(_)) => (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            ),
+            Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
+                (ApiErrorCode::Conflict, ApiErrorCategory::State, true)
+            }
+            Self::Download(DownloadError::NotFound) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Download(DownloadError::Unavailable)
+            | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Download(DownloadError::LimitExceeded { .. }) => (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            ),
+            Self::Download(DownloadError::Unauthorized(_))
+            | Self::Download(DownloadError::InvalidToken)
+            | Self::Download(DownloadError::Expired)
+            | Self::Download(DownloadError::Revoked) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Scheduler(_) => (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Transport(_) => (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Panel(PanelError::RateLimited) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Panel(PanelError::UnknownWidget(_)) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Panel(_) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Durable(_) => (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                true,
+            ),
+        };
+        ApiError::new(code, category, message, retryable, request_id)
+    }
+}
+
 pub struct CoordinatorService {
     coordinator: Coordinator,
     store: RuntimeDurableStore,
-    node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
+    node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
-    node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
+    node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
     node_stale_after_seconds: u64,
     debug_freeze_timeout: std::time::Duration,
     enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
     task_events: VecDeque<TaskCompletionEvent>,
     process_scope_history: VecDeque<ProcessControlKey>,
+    process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
+    process_summary_order: VecDeque<ProcessControlKey>,
+    next_process_summary_order: u64,
+    task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
+    recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
+    recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
+    recent_log_accounted_bytes: BTreeMap<
+        (
+            TenantId,
+            ProjectId,
+            ProcessId,
+            clusterflux_core::TaskInstanceId,
+            String,
+        ),
+        u64,
+    >,
+    recent_log_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    recent_log_quota_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    next_recent_log_sequence: u64,
     debug_audit_events: VecDeque<DebugAuditEvent>,
     debug_epochs: BTreeMap<ProcessControlKey, u64>,
     debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@@ -180,7 +331,7 @@ pub struct CoordinatorService {
     process_cancellations: BTreeSet<ProcessControlKey>,
     process_aborts: BTreeSet<ProcessControlKey>,
     agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
-    node_replay_nonces: BTreeMap<(NodeId, String), u64>,
+    node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
     panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
     stopped_panels: BTreeSet<PanelStopKey>,
     panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@@ -215,6 +366,29 @@ impl CoordinatorService {
         self.artifact_relay.usage()
     }
 
+    pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
+        CoordinatorOperationalMetrics {
+            tenants: self.coordinator.tenant_count(),
+            users: self.coordinator.user_count(),
+            projects: self.coordinator.project_count(),
+            enrolled_nodes: self.coordinator.node_identity_count(),
+            reported_nodes: self.node_descriptors.len(),
+            live_nodes: self
+                .node_descriptors
+                .keys()
+                .filter(|scope| self.node_is_live(scope))
+                .count(),
+            active_processes: self.coordinator.active_process_count(),
+            active_coordinator_mains: self.main_runtime.active_main_count(),
+            max_active_coordinator_mains: self.main_runtime.max_active_mains(),
+            active_tasks: self.active_tasks.len(),
+            queued_tasks: self.pending_task_launches.len(),
+            artifacts: self.artifact_registry.artifact_count(),
+            retained_download_links: self.artifact_registry.retained_download_link_count(),
+            relay: self.artifact_relay.usage(),
+        }
+    }
+
     fn commit_artifact_relay(
         &mut self,
         candidate: relay::ArtifactRelayLedger,
@@ -284,15 +458,10 @@ impl CoordinatorService {
         {
             let identity = self
                 .coordinator
-                .node_identity(node)
+                .node_identity(tenant, project, node)
                 .ok_or(CoordinatorError::UnknownNode)?;
-            if &identity.tenant != tenant || &identity.project != project {
+            debug_assert_eq!(&identity.tenant, tenant);
-                return Err(CoordinatorError::Unauthorized(
+            debug_assert_eq!(&identity.project, project);
-                    "node process-control request is outside its enrolled tenant/project scope"
-                        .to_owned(),
-                )
-                .into());
-            }
             return Ok(());
         }
         self.coordinator
@@ -409,6 +578,16 @@ impl CoordinatorService {
             enrollment_grants: BTreeMap::new(),
             task_events: VecDeque::new(),
             process_scope_history: VecDeque::new(),
+            process_summaries: BTreeMap::new(),
+            process_summary_order: VecDeque::new(),
+            next_process_summary_order: 1,
+            task_terminal_states: BTreeMap::new(),
+            recent_logs: BTreeMap::new(),
+            recent_log_dropped_through: BTreeMap::new(),
+            recent_log_accounted_bytes: BTreeMap::new(),
+            recent_log_truncated_streams: BTreeSet::new(),
+            recent_log_quota_truncated_streams: BTreeSet::new(),
+            next_recent_log_sequence: 1,
             debug_audit_events: VecDeque::new(),
             debug_epochs: BTreeMap::new(),
             debug_epoch_runtime: BTreeMap::new(),

crates/clusterflux-coordinator/src/service/artifacts.rs

@@ -8,7 +8,7 @@ use clusterflux_core::{
 };
 use sha2::{Digest as _, Sha256};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::relay::RelayFinishReason;
 use super::{
@@ -52,7 +52,11 @@ impl CoordinatorService {
         let action = self
             .artifact_registry
             .download_action(&context, &artifact, &policy)?;
-        self.ensure_download_source_connectivity(&action.source)?;
+        self.ensure_download_source_connectivity(
+            &context.tenant,
+            &context.project,
+            &action.source,
+        )?;
         let downloadable_size = self
             .artifact_registry
             .downloadable_size(&context, &artifact, &policy)?;
@@ -163,7 +167,11 @@ impl CoordinatorService {
             },
             &mut validation_meter,
         )?;
-        self.ensure_download_source_connectivity(&stream.link.source)?;
+        self.ensure_download_source_connectivity(
+            &stream.link.tenant,
+            &stream.link.project,
+            &stream.link.source,
+        )?;
         self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
 
         if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@@ -302,7 +310,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let transfer_id = generate_opaque_token("artifact_transfer")
             .map_err(CoordinatorServiceError::Protocol)?;
@@ -410,7 +418,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
         let destination =
@@ -682,15 +690,10 @@ impl CoordinatorService {
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
+        debug_assert_eq!(&identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(&identity.project, project);
-                "artifact reverse transfer node is outside its enrolled tenant/project scope"
-                    .to_owned(),
-            )
-            .into());
-        }
         Ok(())
     }
 
@@ -725,15 +728,12 @@ impl CoordinatorService {
     ) -> Result<NodeEndpoint, CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
+        debug_assert_eq!(&identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(&identity.project, project);
-                "artifact export node is outside the tenant/project scope".to_owned(),
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
-            )
+        let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
-            .into());
-        }
-        let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "node {node} has not reported export connectivity"
             ))
@@ -744,7 +744,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "node {node} is offline for artifact export"
@@ -769,17 +769,20 @@ impl CoordinatorService {
 
     fn ensure_download_source_connectivity(
         &self,
+        tenant: &TenantId,
+        project: &ProjectId,
         source: &StorageLocation,
     ) -> Result<(), clusterflux_core::DownloadError> {
         let StorageLocation::RetainedNode(node) = source else {
             return Ok(());
         };
-        let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
+        let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "retaining node {node} has not reported online status for artifact download"
             ))
         })?;
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "retaining node {node} is offline for artifact download"

crates/clusterflux-coordinator/src/service/authorization.rs

@@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation {
     RevokeAgentPublicKey,
     CreateNodeEnrollmentGrant,
     ListNodeDescriptors,
+    ListNodeSummaries,
     RevokeNodeCredential,
     StartProcess,
     ScheduleTask,
@@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation {
     CancelProcess,
     AbortProcess,
     ListProcesses,
+    ListProcessSummaries,
     QuotaStatus,
     RestartTask,
     ResolveTaskFailure,
@@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation {
     InspectDebugEpoch,
     ListTaskEvents,
     ListTaskSnapshots,
+    ListRecentLogs,
     JoinTask,
+    ListArtifacts,
+    GetArtifact,
     CreateArtifactDownloadLink,
     OpenArtifactDownloadStream,
     RevokeArtifactDownloadLink,
@@ -56,6 +61,7 @@ impl PublicUserOperation {
             Self::RevokeAgentPublicKey => "revoke_agent_public_key",
             Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
             Self::ListNodeDescriptors => "list_node_descriptors",
+            Self::ListNodeSummaries => "list_node_summaries",
             Self::RevokeNodeCredential => "revoke_node_credential",
             Self::StartProcess => "start_process",
             Self::ScheduleTask => "schedule_task",
@@ -63,6 +69,7 @@ impl PublicUserOperation {
             Self::CancelProcess => "cancel_process",
             Self::AbortProcess => "abort_process",
             Self::ListProcesses => "list_processes",
+            Self::ListProcessSummaries => "list_process_summaries",
             Self::QuotaStatus => "quota_status",
             Self::RestartTask => "restart_task",
             Self::ResolveTaskFailure => "resolve_task_failure",
@@ -74,7 +81,10 @@ impl PublicUserOperation {
             Self::InspectDebugEpoch => "inspect_debug_epoch",
             Self::ListTaskEvents => "list_task_events",
             Self::ListTaskSnapshots => "list_task_snapshots",
+            Self::ListRecentLogs => "list_recent_logs",
             Self::JoinTask => "join_task",
+            Self::ListArtifacts => "list_artifacts",
+            Self::GetArtifact => "get_artifact",
             Self::CreateArtifactDownloadLink => "create_artifact_download_link",
             Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
             Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
@@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
                 Self::CreateNodeEnrollmentGrant
             }
             AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries,
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
                 Self::RevokeNodeCredential
             }
@@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
             AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
             AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => {
+                Self::ListProcessSummaries
+            }
             AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
             AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
             AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
@@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
             AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
             AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
+            AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs,
             AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
+            AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts,
+            AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact,
             AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
                 Self::CreateArtifactDownloadLink
             }

crates/clusterflux-coordinator/src/service/debug.rs

@@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct DebugBreakpointPlan {
     pub(super) actor: UserId,
+    pub(super) revision: u64,
     pub(super) probe_symbols: BTreeSet<String>,
     pub(super) hit_epoch: Option<u64>,
     pub(super) hit_task: Option<TaskInstanceId>,
@@ -85,6 +86,7 @@ impl CoordinatorService {
         project: String,
         actor_user: String,
         process: String,
+        revision: u64,
         probe_symbols: Vec<String>,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let probe_symbols = validate_probe_symbols(probe_symbols)?;
@@ -115,10 +117,28 @@ impl CoordinatorService {
             ))
             .into());
         }
+        let key = process_control_key(&tenant, &project, &process);
+        if let Some(current) = self.debug_breakpoints.get(&key) {
+            if current.actor == actor && revision < current.revision {
+                return Ok(CoordinatorResponse::DebugBreakpoints {
+                    process,
+                    actor,
+                    revision: current.revision,
+                    probe_symbols: current.probe_symbols.iter().cloned().collect(),
+                    hit_epoch: current.hit_epoch,
+                    hit_task: current.hit_task.clone(),
+                    hit_probe_symbol: current.hit_probe_symbol.clone(),
+                    charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
+                    used_debug_read_bytes: audit_event.used_debug_read_bytes,
+                    audit_event,
+                });
+            }
+        }
         self.debug_breakpoints.insert(
-            process_control_key(&tenant, &project, &process),
+            key,
             DebugBreakpointPlan {
                 actor: actor.clone(),
+                revision,
                 probe_symbols: probe_symbols.iter().cloned().collect(),
                 hit_epoch: None,
                 hit_task: None,
@@ -128,6 +148,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision,
             probe_symbols,
             hit_epoch: None,
             hit_task: None,
@@ -190,6 +211,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision: plan.revision,
             probe_symbols: plan.probe_symbols.into_iter().collect(),
             hit_epoch: plan.hit_epoch,
             hit_task: plan.hit_task,

crates/clusterflux-coordinator/src/service/debug_requests.rs

@@ -33,12 +33,14 @@ impl CoordinatorService {
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant,
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             ),
             CoordinatorRequest::InspectDebugBreakpoints {
@@ -96,12 +98,14 @@ impl CoordinatorService {
             ),
             AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant.as_str().to_owned(),
                 project.as_str().to_owned(),
                 actor.as_str().to_owned(),
                 process,
+                revision,
                 probe_symbols,
             ),
             AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
@@ -222,24 +226,25 @@ impl CoordinatorService {
         } else if !completed_event_observed {
             "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
         } else if let Some(checkpoint) = checkpoint {
-            let vfs_available =
+            let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
-                checkpoint
+                true,
-                    .checkpoint
+                |available, (path, object)| {
-                    .vfs_manifest
+                    let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
-                    .objects
+                        CoordinatorServiceError::InvalidArtifactPath(error.to_string())
-                    .iter()
+                    })?;
-                    .all(|(path, object)| {
+                    Ok::<_, CoordinatorServiceError>(
-                        let artifact = super::keys::artifact_id_from_path(path);
+                        available
-                        self.artifact_registry
+                            && self
-                            .metadata(&artifact)
+                                .artifact_registry
+                                .metadata(&tenant, &project, &artifact)
                                 .is_some_and(|metadata| {
-                                metadata.tenant == tenant
+                                    metadata.digest == object.digest
-                                    && metadata.project == project
-                                    && metadata.digest == object.digest
                                         && metadata.size == object.size
                                         && !metadata.retaining_nodes.is_empty()
-                            })
+                                }),
-                    });
+                    )
+                },
+            )?;
             if !vfs_available {
                 "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
             } else {
@@ -285,6 +290,25 @@ impl CoordinatorService {
                             task.as_str()
                         );
                         if let Some(replacement) = replacement {
+                            if assignment.task_spec.source_snapshot.is_some() {
+                                let replacement_source = replacement_bundle
+                                    .as_ref()
+                                    .and_then(|bundle| bundle.source_snapshot.clone())
+                                    .ok_or_else(|| {
+                                        CoordinatorServiceError::Protocol(
+                                            "replacement task omitted the current SourceSnapshot for source-bound arguments"
+                                                .to_owned(),
+                                        )
+                                    })?;
+                                assignment
+                                    .task_spec
+                                    .rebind_source_snapshot(replacement_source)
+                                    .map_err(|error| {
+                                        CoordinatorServiceError::Protocol(format!(
+                                            "replacement task SourceSnapshot is invalid: {error}"
+                                        ))
+                                    })?;
+                            }
                             assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
                                 export: Some(replacement.export),
                                 abi: WasmExportAbi::TaskV1,
@@ -456,6 +480,7 @@ impl CoordinatorService {
         }
         self.record_task_completion_event(event.clone());
         self.notify_coordinator_main_waiters(&event);
+        self.maybe_retire_terminal_process(&tenant, &project, &process)?;
         Ok(CoordinatorResponse::TaskFailureResolved {
             process,
             task,

crates/clusterflux-coordinator/src/service/keys.rs

@@ -1,6 +1,7 @@
 use clusterflux_core::{
     ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
 };
+use thiserror::Error;
 
 pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
 pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
     (tenant.clone(), project.clone(), grant.to_owned())
 }
 
-pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub(super) enum ArtifactPathError {
+    #[error("path must start with the exact /vfs/artifacts/ prefix")]
+    WrongPrefix,
+    #[error("path must name an artifact after /vfs/artifacts/")]
+    EmptyArtifact,
+    #[error("mapped artifact identifier is invalid: {0}")]
+    InvalidArtifactId(#[from] clusterflux_core::IdParseError),
+}
+
+pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
     let value = path
         .as_str()
         .strip_prefix("/vfs/artifacts/")
-        .unwrap_or(path.as_str())
+        .ok_or(ArtifactPathError::WrongPrefix)?;
-        .replace('/', ":");
+    if value.is_empty() {
-    ArtifactId::new(value)
+        return Err(ArtifactPathError::EmptyArtifact);
+    }
+    ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn artifact_id_conversion_is_fallible_and_structural() {
+        assert_eq!(
+            artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
+            ArtifactId::from("build:output")
+        );
+        for path in [
+            "/vfs/other/output",
+            "/vfs/artifacts",
+            "/vfs/artifacts/bad artifact!",
+        ] {
+            let path = VfsPath::new(path).unwrap();
+            assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
+        }
+
+        let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
+        assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
+    }
 }

crates/clusterflux-coordinator/src/service/logs.rs

@@ -1,5 +1,5 @@
 use clusterflux_core::{
-    ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
+    ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
     TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
 };
 
@@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key};
 use super::protocol::TaskAttemptState;
 use super::{
     artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
-    TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
+    RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState,
+    MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES,
+    MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT,
+    MAX_TASK_LOG_TAIL_BYTES,
 };
 
 impl CoordinatorService {
@@ -36,23 +39,44 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         validate_task_log_tail("stdout_tail", &stdout_tail)?;
         validate_task_log_tail("stderr_tail", &stderr_tail)?;
-        let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota
+        let stdout_retained = self.accept_final_log_stream(
-            .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+            &tenant,
-        self.quota
+            &project,
-            .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+            &process,
+            &task,
+            TaskLogStream::Stdout,
+            stdout_bytes,
+            &stdout_tail,
+            stdout_truncated,
+            now_epoch_seconds,
+        )?;
+        let stderr_retained = self.accept_final_log_stream(
+            &tenant,
+            &project,
+            &process,
+            &task,
+            TaskLogStream::Stderr,
+            stderr_bytes,
+            &stderr_tail,
+            stderr_truncated,
+            now_epoch_seconds,
+        )?;
         Ok(CoordinatorResponse::TaskLogRecorded {
             process,
             task,
             stdout_bytes,
             stderr_bytes,
-            stdout_tail: if stdout_truncated {
+            stdout_tail: if !stdout_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stdout_truncated {
                 format!("{stdout_tail}\n... truncated")
             } else {
                 stdout_tail
             },
-            stderr_tail: if stderr_truncated {
+            stderr_tail: if !stderr_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stderr_truncated {
                 format!("{stderr_tail}\n... truncated")
             } else {
                 stderr_tail
@@ -61,6 +85,158 @@ impl CoordinatorService {
         })
     }
 
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_report_task_log_chunk(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stream: TaskLogStream,
+        offset: u64,
+        source_bytes: u64,
+        text: String,
+        truncated: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
+                "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}",
+                text.len()
+            )));
+        }
+        if source_bytes == 0 && !text.is_empty() && !truncated {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes must describe non-empty text".to_owned(),
+            ));
+        }
+        if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(),
+            ));
+        }
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
+        let expected = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let end = offset.checked_add(source_bytes).ok_or_else(|| {
+            CoordinatorServiceError::Protocol(
+                "live log chunk offset exceeds the supported range".to_owned(),
+            )
+        })?;
+        let state_marker = source_bytes == 0 && truncated;
+        if end < expected || (end == expected && !state_marker) {
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: expected,
+            });
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key, end);
+            }
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: end.max(expected),
+            });
+        }
+        let newly_accounted = end.saturating_sub(expected);
+        if self
+            .quota
+            .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)
+            .is_err()
+        {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key.clone(), end);
+            }
+            let sequence = self.mark_log_quota_truncated(
+                &tenant,
+                &project,
+                &process,
+                &task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence,
+                next_offset: end.max(expected),
+            });
+        }
+        if offset > expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!("[log output lost: {} bytes]", offset - expected),
+                true,
+                now_epoch_seconds,
+            );
+        } else if offset < expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!(
+                    "[log output overlap omitted: {} new source bytes]",
+                    end - expected
+                ),
+                true,
+                now_epoch_seconds,
+            );
+        }
+        let marker_is_new = if truncated {
+            self.recent_log_truncated_streams.insert(key.clone())
+        } else {
+            false
+        };
+        let text = if state_marker && text.is_empty() {
+            "[log output truncated at source]".to_owned()
+        } else {
+            text
+        };
+        let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
+            .then(|| {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream,
+                    text,
+                    truncated,
+                    now_epoch_seconds,
+                )
+            });
+        if end > expected {
+            self.recent_log_accounted_bytes.insert(key, end);
+        }
+        Ok(CoordinatorResponse::TaskLogChunkRecorded {
+            process,
+            task,
+            sequence,
+            next_offset: end,
+        })
+    }
+
     pub(super) fn handle_report_vfs_metadata(
         &mut self,
         tenant: String,
@@ -85,7 +261,9 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant,
                 project,
                 process: process.clone(),
@@ -177,20 +355,37 @@ impl CoordinatorService {
             artifact_size_bytes,
             result,
         };
-        let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota.can_charge_log_bytes(
+        let stdout_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stdout,
+            event.stdout_bytes,
+            &event.stdout_tail,
+            event.stdout_truncated,
             now_epoch_seconds,
         )?;
-        self.quota.charge_log_bytes(
+        let stderr_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stderr,
+            event.stderr_bytes,
+            &event.stderr_tail,
+            event.stderr_truncated,
             now_epoch_seconds,
         )?;
+        if !stdout_retained {
+            event.stdout_tail = "[log output truncated at project log quota]".to_owned();
+            event.stdout_truncated = true;
+        }
+        if !stderr_retained {
+            event.stderr_tail = "[log output truncated at project log quota]".to_owned();
+            event.stderr_truncated = true;
+        }
         let task_key = task_control_key(
             &event.tenant,
             &event.project,
@@ -203,7 +398,9 @@ impl CoordinatorService {
         event.placement = self.task_placements.remove(&task_key);
         if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant: event.tenant.clone(),
                 project: event.project.clone(),
                 process: event.process.clone(),
@@ -217,26 +414,22 @@ impl CoordinatorService {
         self.task_aborts.remove(&task_key);
         self.debug_commands.remove(&task_key);
         self.active_tasks.remove(&task_key);
-        let no_active_tasks =
+        self.clear_recent_log_offsets_for_task(
-            !self
+            &event.tenant,
-                .active_tasks
+            &event.project,
-                .iter()
+            &event.process,
-                .any(|(task_tenant, task_project, task_process, _, _)| {
+            &event.task,
-                    task_tenant == &event.tenant
+        );
-                        && task_project == &event.project
+        self.task_assignments.retain(|_, assignments| {
-                        && task_process == &event.process
+            assignments.retain(|assignment| {
+                assignment.tenant != event.tenant
+                    || assignment.project != event.project
+                    || assignment.process != event.process
+                    || assignment.node != event.node
+                    || assignment.task != event.task
+            });
+            !assignments.is_empty()
         });
-        if no_active_tasks {
-            self.process_aborts.remove(&process_key);
-            if self.process_cancellations.remove(&process_key)
-                && !self.main_runtime.controls.contains_key(&process_key)
-            {
-                self.coordinator
-                    .abort_process(&event.tenant, &event.project, &event.process)?;
-                self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
-                self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
-            }
-        }
         if process_was_aborted {
             let checkpoint_key = super::keys::task_restart_key(
                 &event.tenant,
@@ -253,10 +446,20 @@ impl CoordinatorService {
         if !awaiting_operator {
             self.notify_coordinator_main_waiters(&event);
         }
+        self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
+        let events_recorded = self
+            .task_events
+            .iter()
+            .filter(|recorded| {
+                recorded.tenant == event.tenant
+                    && recorded.project == event.project
+                    && recorded.process == event.process
+            })
+            .count();
         Ok(CoordinatorResponse::TaskRecorded {
             process: event.process,
             task: event.task,
-            events_recorded: self.task_events.len(),
+            events_recorded,
         })
     }
 
@@ -316,7 +519,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::TaskSnapshots { snapshots })
     }
 
-    fn authorize_task_event_process_scope(
+    pub(super) fn authorize_task_event_process_scope(
         &self,
         tenant: &TenantId,
         project: &ProjectId,
@@ -356,6 +559,47 @@ impl CoordinatorService {
         Ok(())
     }
 
+    pub(super) fn handle_list_recent_logs(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: Option<String>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = task.map(TaskInstanceId::new);
+        self.authorize_task_event_process_scope(&tenant, &project, &process)?;
+        let after_sequence = after_sequence.unwrap_or(0);
+        let retained = self.recent_logs.get(&(tenant.clone(), project.clone()));
+        let history_truncated = self
+            .recent_log_dropped_through
+            .get(&process_control_key(&tenant, &project, &process))
+            .is_some_and(|dropped_through| *dropped_through > after_sequence);
+        let entries = retained
+            .into_iter()
+            .flatten()
+            .filter(|entry| {
+                entry.process == process
+                    && entry.sequence > after_sequence
+                    && task.as_ref().is_none_or(|task| &entry.task == task)
+            })
+            .take(limit as usize)
+            .cloned()
+            .collect::<Vec<_>>();
+        let next_sequence = entries.last().map(|entry| entry.sequence);
+        Ok(CoordinatorResponse::RecentLogs {
+            entries,
+            next_sequence,
+            history_truncated,
+        })
+    }
+
     pub(super) fn handle_join_task(
         &mut self,
         tenant: String,
@@ -474,6 +718,22 @@ impl CoordinatorService {
     pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
         event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
         event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
+        match event.executor {
+            super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
+                &event.tenant,
+                &event.project,
+                &event.process,
+                event.task_definition.clone(),
+                event.task.clone(),
+                event.terminal_state.clone(),
+            ),
+            super::TaskExecutor::Node => {
+                self.task_terminal_states.insert(
+                    task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
+                    event.terminal_state.clone(),
+                );
+            }
+        }
         let process_scope = (
             event.tenant.clone(),
             event.project.clone(),
@@ -511,6 +771,321 @@ impl CoordinatorService {
         self.task_events.push_back(event);
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn record_recent_log(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        task: TaskInstanceId,
+        stream: TaskLogStream,
+        mut text: String,
+        mut truncated: bool,
+        server_timestamp_epoch_seconds: u64,
+    ) -> u64 {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES;
+            while !text.is_char_boundary(boundary) {
+                boundary -= 1;
+            }
+            text.truncate(boundary);
+            truncated = true;
+        }
+        let sequence = self.next_recent_log_sequence;
+        self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1);
+        let logs = self
+            .recent_logs
+            .entry((tenant.clone(), project.clone()))
+            .or_default();
+        let mut dropped = Vec::new();
+        while logs.iter().filter(|entry| entry.process == process).count()
+            >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS
+        {
+            let Some(index) = logs.iter().position(|entry| entry.process == process) else {
+                break;
+            };
+            if let Some(entry) = logs.remove(index) {
+                dropped.push(entry);
+            }
+        }
+        while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT
+            || logs
+                .iter()
+                .map(|entry| entry.text.len())
+                .sum::<usize>()
+                .saturating_add(text.len())
+                > MAX_RECENT_LOG_BYTES_PER_PROJECT
+        {
+            match logs.pop_front() {
+                Some(entry) => dropped.push(entry),
+                None => break,
+            }
+        }
+        logs.push_back(RecentLogEntry {
+            sequence,
+            process,
+            task,
+            stream,
+            text,
+            server_timestamp_epoch_seconds,
+            truncated,
+        });
+        for entry in dropped {
+            let key = process_control_key(&tenant, &project, &entry.process);
+            self.recent_log_dropped_through
+                .entry(key)
+                .and_modify(|dropped_through| {
+                    *dropped_through = (*dropped_through).max(entry.sequence);
+                })
+                .or_insert(entry.sequence);
+        }
+        sequence
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn accept_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| {
+            let stream_name = match stream {
+                TaskLogStream::Stdout => "stdout",
+                TaskLogStream::Stderr => "stderr",
+            };
+            CoordinatorServiceError::Protocol(format!(
+                "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted"
+            ))
+        })?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            return Ok(false);
+        }
+        if self
+            .quota
+            .charge_log_bytes(tenant, project, remaining, now_epoch_seconds)
+            .is_err()
+        {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            self.mark_log_quota_truncated(
+                tenant,
+                project,
+                process,
+                task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(false);
+        }
+        self.reconcile_final_log_stream(
+            tenant,
+            project,
+            process,
+            task,
+            stream,
+            total_source_bytes,
+            final_tail,
+            source_truncated,
+            now_epoch_seconds,
+        );
+        Ok(true)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn mark_log_quota_truncated(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: &TaskLogStream,
+        now_epoch_seconds: u64,
+    ) -> Option<u64> {
+        let key = recent_log_offset_key(tenant, project, process, task, stream);
+        if !self.recent_log_quota_truncated_streams.insert(key.clone()) {
+            return None;
+        }
+        self.recent_log_truncated_streams.insert(key);
+        Some(self.record_recent_log(
+            tenant.clone(),
+            project.clone(),
+            process.clone(),
+            task.clone(),
+            stream.clone(),
+            "[log output truncated at project log quota]".to_owned(),
+            true,
+            now_epoch_seconds,
+        ))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn reconcile_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let mut visible_truncation = false;
+        if total_source_bytes > accounted {
+            let missing = total_source_bytes - accounted;
+            if final_tail.is_empty() {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!("[log output unavailable: {missing} source bytes]"),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            } else if (final_tail.len() as u64) <= total_source_bytes {
+                let tail_source_start = total_source_bytes - final_tail.len() as u64;
+                if accounted < tail_source_start {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        format!(
+                            "[log output lost before final tail: {} source bytes]",
+                            tail_source_start - accounted
+                        ),
+                        true,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation = true;
+                }
+                let source_start = accounted.max(tail_source_start) - tail_source_start;
+                let mut byte_start = usize::try_from(source_start)
+                    .unwrap_or(final_tail.len())
+                    .min(final_tail.len());
+                while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
+                    byte_start += 1;
+                }
+                let suffix = &final_tail[byte_start..];
+                if !suffix.is_empty() {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        suffix.to_owned(),
+                        source_truncated || visible_truncation,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation |= source_truncated;
+                }
+            } else if accounted == 0 {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    final_tail.to_owned(),
+                    source_truncated,
+                    now_epoch_seconds,
+                );
+                visible_truncation |= source_truncated;
+            } else {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!(
+                        "[{missing} additional source bytes could not be merged without duplicating redacted output]"
+                    ),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            }
+            self.recent_log_accounted_bytes
+                .insert(key.clone(), total_source_bytes);
+        }
+
+        let marker_is_new = (source_truncated || visible_truncation)
+            && self.recent_log_truncated_streams.insert(key);
+        if marker_is_new && !visible_truncation {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream,
+                "[log output truncated at source]".to_owned(),
+                true,
+                now_epoch_seconds,
+            );
+        }
+    }
+
+    pub(super) fn clear_recent_log_offsets_for_task(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+    ) {
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _), _| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+    }
+
     fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
         let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
         let Some(attempt) = self
@@ -545,6 +1120,118 @@ impl CoordinatorService {
         awaiting_operator
     }
 
+    pub(super) fn maybe_retire_terminal_process(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let process_key = process_control_key(tenant, project, process);
+        if self.main_runtime.controls.contains_key(&process_key) {
+            return Ok(false);
+        }
+
+        let has_runnable_remote_work =
+            self.active_tasks
+                .iter()
+                .any(|(task_tenant, task_project, task_process, _, _)| {
+                    task_tenant == tenant && task_project == project && task_process == process
+                })
+                || self.pending_task_launches.iter().any(|pending| {
+                    &pending.tenant == tenant
+                        && &pending.project == project
+                        && &pending.process == process
+                })
+                || self.task_assignments.values().any(|assignments| {
+                    assignments.iter().any(|assignment| {
+                        &assignment.tenant == tenant
+                            && &assignment.project == project
+                            && &assignment.process == process
+                    })
+                })
+                || self.task_attempts.iter().any(
+                    |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                        attempt_tenant == tenant
+                            && attempt_project == project
+                            && attempt_process == process
+                            && attempts.iter().rev().any(|attempt| {
+                                attempt.current
+                                    && matches!(
+                                        attempt.state,
+                                        TaskAttemptState::Queued
+                                            | TaskAttemptState::Running
+                                            | TaskAttemptState::FailedAwaitingAction
+                                    )
+                            })
+                    },
+                );
+        if has_runnable_remote_work {
+            return Ok(false);
+        }
+
+        let main_completed = self
+            .process_summaries
+            .get(&process_key)
+            .and_then(|summary| summary.main_terminal_state.as_ref())
+            .is_some_and(|state| matches!(state, TaskTerminalState::Completed));
+        let cancellation_completed = self.process_cancellations.contains(&process_key);
+        if !main_completed && !cancellation_completed {
+            return Ok(false);
+        }
+
+        self.process_aborts.remove(&process_key);
+        self.process_cancellations.remove(&process_key);
+        if self
+            .coordinator
+            .active_process(tenant, project, process)
+            .is_none()
+        {
+            return Ok(false);
+        }
+        let final_result = if cancellation_completed {
+            super::ProcessFinalResult::Cancelled
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Failed
+            },
+        ) {
+            super::ProcessFinalResult::Failed
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Cancelled
+            },
+        ) {
+            super::ProcessFinalResult::Cancelled
+        } else {
+            super::ProcessFinalResult::Completed
+        };
+        self.record_process_terminal(
+            tenant,
+            project,
+            process,
+            final_result,
+            self.current_epoch_seconds()?,
+        );
+        self.coordinator.abort_process(tenant, project, process)?;
+        self.clear_debug_state_for_process(tenant, project, process);
+        self.clear_operator_panel_state(tenant, project, process);
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(tenant, project);
+        self.artifact_registry.enforce_project_metadata_limit(
+            tenant,
+            project,
+            &pinned,
+            &protected_processes,
+        );
+        Ok(true)
+    }
+
     fn flush_artifact_metadata(
         &mut self,
         flush: ArtifactFlush,
@@ -552,23 +1239,59 @@ impl CoordinatorService {
         let now_epoch_seconds = self.current_epoch_seconds()?;
         self.artifact_registry
             .expire_download_links(now_epoch_seconds);
-        let pinned = self
+        let tenant = flush.tenant.clone();
-            .task_restart_checkpoints
+        let project = flush.project.clone();
-            .values()
+        let (pinned, protected_processes) =
-            .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
+            self.artifact_retention_guards_for_project(&tenant, &project);
-            .chain(
-                self.pending_task_launches
-                    .iter()
-                    .flat_map(|pending| pending.task_spec.required_artifacts.iter()),
-            )
-            .cloned()
-            .collect::<std::collections::BTreeSet<ArtifactId>>();
         self.artifact_registry
-            .flush_metadata_bounded(flush, &pinned)
+            .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
             .map(|_| ())
             .map_err(CoordinatorServiceError::Protocol)
     }
 
+    fn artifact_retention_guards_for_project(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> (
+        std::collections::BTreeSet<ArtifactScopeKey>,
+        std::collections::BTreeSet<ProcessId>,
+    ) {
+        let mut pinned = std::collections::BTreeSet::new();
+        for checkpoint in self.task_restart_checkpoints.values() {
+            if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
+            {
+                continue;
+            }
+            for artifact in &checkpoint.assignment.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &checkpoint.assignment.tenant,
+                    &checkpoint.assignment.project,
+                    artifact,
+                ));
+            }
+        }
+        for pending in &self.pending_task_launches {
+            if &pending.tenant != tenant || &pending.project != project {
+                continue;
+            }
+            for artifact in &pending.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &pending.tenant,
+                    &pending.project,
+                    artifact,
+                ));
+            }
+        }
+        let protected_processes = self
+            .coordinator
+            .active_processes_for_project(tenant, project)
+            .into_iter()
+            .map(|process| process.id)
+            .collect();
+        (pinned, protected_processes)
+    }
+
     fn task_is_known_or_active(
         &self,
         tenant: &TenantId,
@@ -601,15 +1324,24 @@ impl CoordinatorService {
     }
 }
 
-fn checked_reported_log_bytes(
+fn recent_log_offset_key(
-    stdout_bytes: u64,
+    tenant: &TenantId,
-    stderr_bytes: u64,
+    project: &ProjectId,
-) -> Result<u64, CoordinatorServiceError> {
+    process: &ProcessId,
-    stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
+    task: &TaskInstanceId,
-        CoordinatorServiceError::Protocol(
+    stream: &TaskLogStream,
-            "reported task log byte counts exceed the supported range".to_owned(),
+) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        task.clone(),
+        match stream {
+            TaskLogStream::Stdout => "stdout",
+            TaskLogStream::Stderr => "stderr",
+        }
+        .to_owned(),
     )
-    })
 }
 
 fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
@@ -626,11 +1358,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
     if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
         return value;
     }
-    let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
+    let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES;
-    while !value.is_char_boundary(boundary) {
+    while boundary < value.len() && !value.is_char_boundary(boundary) {
-        boundary -= 1;
+        boundary += 1;
     }
-    value.truncate(boundary);
+    value.drain(..boundary);
     *truncated = true;
     value
 }

crates/clusterflux-coordinator/src/service/main_runtime.rs

@@ -30,14 +30,14 @@ use super::{
 };
 
 #[derive(Clone)]
-struct MainScope {
+pub(super) struct MainScope {
-    tenant: TenantId,
+    pub(super) tenant: TenantId,
-    project: ProjectId,
+    pub(super) project: ProjectId,
-    process: ProcessId,
+    pub(super) process: ProcessId,
-    task_definition: TaskDefinitionId,
+    pub(super) task_definition: TaskDefinitionId,
-    task_instance: TaskInstanceId,
+    pub(super) task_instance: TaskInstanceId,
-    epoch: u64,
+    pub(super) epoch: u64,
-    launch_id: u64,
+    pub(super) launch_id: u64,
 }
 
 enum MainCommand {
@@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime {
 }
 
 impl CoordinatorMainRuntime {
+    pub(super) fn active_main_count(&self) -> usize {
+        self.controls.len()
+    }
+
+    pub(super) fn max_active_mains(&self) -> usize {
+        self.max_active_mains
+    }
+
     pub(super) fn configure(
         &mut self,
         configuration: super::CoordinatorMainRuntimeConfiguration,
@@ -729,6 +737,13 @@ impl CoordinatorService {
                 &process,
                 "coordinator main launch failed admission or validation",
             );
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Failed,
+                self.liveness_now_epoch_seconds(),
+            );
             let _ = self.coordinator.abort_process(&tenant, &project, &process);
         }
         result
@@ -889,7 +904,7 @@ impl CoordinatorService {
         }
     }
 
-    fn record_coordinator_main_completion(
+    pub(super) fn record_coordinator_main_completion(
         &mut self,
         scope: MainScope,
         result: Result<WasmTaskResult, String>,
@@ -929,6 +944,7 @@ impl CoordinatorService {
             ),
             Err(error) => (TaskTerminalState::Failed, None, error),
         };
+        let main_completed = matches!(terminal_state, TaskTerminalState::Completed);
         let main_state = match terminal_state {
             TaskTerminalState::Completed => "completed",
             TaskTerminalState::Failed => "failed",
@@ -970,6 +986,12 @@ impl CoordinatorService {
             }
         }
         let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
+        self.main_runtime.controls.remove(&process_key);
+        if main_completed {
+            let _ =
+                self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
+            return;
+        }
         for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
             if task_tenant == scope.tenant
                 && task_project == scope.project
@@ -985,10 +1007,16 @@ impl CoordinatorService {
             }
         }
         self.process_aborts.insert(process_key.clone());
+        self.record_process_terminal(
+            &scope.tenant,
+            &scope.project,
+            &scope.process,
+            super::ProcessFinalResult::Failed,
+            self.liveness_now_epoch_seconds(),
+        );
         let _ = self
             .coordinator
             .abort_process(&scope.tenant, &scope.project, &scope.process);
-        self.main_runtime.controls.remove(&process_key);
         self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process);
         self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process);
     }
@@ -1157,6 +1185,120 @@ mod tests {
         assert!(!service.main_runtime.controls.contains_key(&process_key));
         assert!(!service.debug_epochs.contains_key(&process_key));
         assert!(!service.debug_epoch_runtime.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+    }
+
+    #[test]
+    fn completed_main_keeps_process_and_debug_state_for_active_children() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-current");
+        let main_task = TaskInstanceId::from("ti:vp-current:main");
+        let child_task = TaskInstanceId::from("ti:vp-current:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task.clone(),
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(
+            scope,
+            Ok(WasmTaskResult::completed(
+                main_task,
+                TaskBoundaryValue::SmallJson(serde_json::Value::Null),
+            )),
+        );
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_some());
+        assert!(!service.main_runtime.controls.contains_key(&process_key));
+        assert!(service.debug_epochs.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(!service.task_aborts.contains(&child_key));
+    }
+
+    #[test]
+    fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-failed-main");
+        let main_task = TaskInstanceId::from("ti:vp-failed-main:main");
+        let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task,
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(scope, Err("main crashed".to_owned()));
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_none());
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(service.task_aborts.contains(&child_key));
         assert!(service.process_aborts.contains(&process_key));
+        assert!(!service.debug_epochs.contains_key(&process_key));
+        assert!(service.task_events.iter().any(|event| {
+            event.process == process
+                && event.executor == TaskExecutor::CoordinatorMain
+                && event.terminal_state == TaskTerminalState::Failed
+        }));
     }
 }

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     SourceProviderKind, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{
     bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@@ -27,9 +27,9 @@ impl CoordinatorService {
         unix_timestamp_seconds()
     }
 
-    pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
+    pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
         self.node_last_seen_epoch_seconds
-            .get(node)
+            .get(scope)
             .is_some_and(|last_seen| {
                 self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
                     <= self.node_stale_after_seconds
@@ -41,7 +41,11 @@ impl CoordinatorService {
             .values()
             .cloned()
             .map(|mut descriptor| {
-                descriptor.online = self.node_is_live(&descriptor.id);
+                descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
+                    &descriptor.tenant,
+                    &descriptor.project,
+                    &descriptor.id,
+                ));
                 descriptor
             })
             .collect()
@@ -165,7 +169,11 @@ impl CoordinatorService {
             !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
         });
         self.coordinator.ensure_tenant_active(&tenant)?;
-        if self.coordinator.node_identity(&node).is_none() {
+        if self
+            .coordinator
+            .node_identity(&tenant, &project, &node)
+            .is_none()
+        {
             self.quota.ensure_node_admission(
                 &tenant,
                 self.coordinator.node_identity_count_for_tenant(&tenant),
@@ -197,12 +205,21 @@ impl CoordinatorService {
 
     pub(super) fn handle_node_heartbeat(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         node_signature: Option<NodeSignedRequest>,
         payload_digest: &Digest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
-        self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
+        self.authenticate_node_request(
+            &NodeScopeKey::new(tenant, project, node.clone()),
+            node_signature,
+            "node_heartbeat",
+            payload_digest,
+        )?;
         Ok(CoordinatorResponse::NodeHeartbeat {
             node,
             epoch: self.coordinator.coordinator_epoch(),
@@ -225,16 +242,13 @@ impl CoordinatorService {
         let tenant = TenantId::new(tenant);
         let project = ProjectId::new(project);
         let node = NodeId::new(node);
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "node capability report is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
         capabilities.validate_public_report()?;
         for (kind, count) in [
             ("cached environments", cached_environment_digests.len()),
@@ -274,16 +288,20 @@ impl CoordinatorService {
             .into_iter()
             .map(ArtifactId::new)
             .collect::<BTreeSet<_>>();
-        self.artifact_registry
+        self.artifact_registry.reconcile_node_retention(
-            .reconcile_node_retention(&node, &artifact_locations);
+            &tenant,
+            &project,
+            &node,
+            &artifact_locations,
+        );
 
-        let online = self.node_is_live(&node);
+        let online = self.node_is_live(&node_scope);
         self.node_descriptors.insert(
-            node.clone(),
+            node_scope,
             NodeDescriptor {
                 id: node.clone(),
-                tenant,
+                tenant: tenant.clone(),
-                project,
+                project: project.clone(),
                 capabilities,
                 cached_environments: cached_environment_digests.into_iter().collect(),
                 dependency_caches: dependency_cache_digests.into_iter().collect(),
@@ -293,9 +311,14 @@ impl CoordinatorService {
                 online,
             },
         );
+        let node_descriptors = self
+            .node_descriptors
+            .values()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .count();
         Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
             node,
-            node_descriptors: self.node_descriptors.len(),
+            node_descriptors,
         })
     }
 
@@ -333,9 +356,13 @@ impl CoordinatorService {
             actor: Actor::User(actor.clone()),
         };
         self.coordinator.revoke_node_credential(&context, &node)?;
-        let descriptor_removed = self.node_descriptors.remove(&node).is_some();
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
-        self.node_last_seen_epoch_seconds.remove(&node);
+        let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
-        self.artifact_registry.garbage_collect_node(&node);
+        self.node_last_seen_epoch_seconds.remove(&node_scope);
+        self.node_replay_nonces
+            .retain(|(retained_scope, _), _| retained_scope != &node_scope);
+        self.artifact_registry
+            .garbage_collect_node(&tenant, &project, &node);
         let queued_assignments_removed = self
             .task_assignments
             .remove(&(tenant.clone(), project.clone(), node.clone()))
@@ -369,14 +396,14 @@ impl CoordinatorService {
 
     pub(super) fn authenticate_node_request(
         &mut self,
-        node: &NodeId,
+        scope: &NodeScopeKey,
         node_signature: Option<NodeSignedRequest>,
         request_kind: &str,
         payload_digest: &Digest,
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(&scope.tenant, &scope.project, &scope.node)
             .ok_or(CoordinatorError::UnknownNode)?;
         let signature = node_signature.ok_or_else(|| {
             CoordinatorError::Unauthorized(
@@ -401,7 +428,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        let replay_key = (node.clone(), signature.nonce.clone());
+        let replay_key = (scope.clone(), signature.nonce.clone());
         self.node_replay_nonces.retain(|_, accepted_at| {
             now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
         });
@@ -413,7 +440,7 @@ impl CoordinatorService {
         }
         verify_node_request_signature(
             &identity.public_key,
-            node,
+            &scope.node,
             request_kind,
             payload_digest,
             &signature,
@@ -422,7 +449,7 @@ impl CoordinatorService {
         if self
             .node_replay_nonces
             .keys()
-            .filter(|(retained_node, _)| retained_node == node)
+            .filter(|(retained_scope, _)| retained_scope == scope)
             .count()
             >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
         {
@@ -436,8 +463,8 @@ impl CoordinatorService {
             .insert(replay_key, now_epoch_seconds);
         let seen_at = self.liveness_now_epoch_seconds();
         self.node_last_seen_epoch_seconds
-            .insert(node.clone(), seen_at);
+            .insert(scope.clone(), seen_at);
-        if let Some(descriptor) = self.node_descriptors.get_mut(node) {
+        if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
             descriptor.online = true;
         }
         Ok(())

crates/clusterflux-coordinator/src/service/panels.rs

@@ -254,7 +254,8 @@ impl CoordinatorService {
             .rev()
             .find_map(|event| event.artifact_path.as_ref())
         {
-            let artifact = artifact_id_from_path(path);
+            let artifact = artifact_id_from_path(path)
+                .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
             let context = clusterflux_core::AuthContext {
                 tenant,
                 project,

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -3,9 +3,9 @@ use std::collections::BTreeMap;
 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
 use clusterflux_core::{
     AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
-    NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskBoundaryValue, TaskCheckpoint,
+    NodeDescriptor, NodeId, Placement, PlacementError, PlacementRequest, ProcessId, ProjectId,
-    TaskDispatch, TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath,
+    Scheduler, TaskBoundaryValue, TaskCheckpoint, TaskDispatch, TaskInstanceId, TaskSpec, TenantId,
-    WasmTaskInvocation,
+    VfsManifest, VfsObject, VfsPath, WasmTaskInvocation,
 };
 
 use crate::CoordinatorError;
@@ -18,7 +18,76 @@ use super::{
 use super::processes::*;
 use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
 
+fn select_task_placement(
+    candidates: &[Placement],
+    active_by_node: &BTreeMap<NodeId, usize>,
+) -> Option<Placement> {
+    candidates
+        .iter()
+        .min_by(|left, right| {
+            right
+                .score
+                .cmp(&left.score)
+                .then_with(|| {
+                    active_by_node
+                        .get(&left.node)
+                        .copied()
+                        .unwrap_or_default()
+                        .cmp(&active_by_node.get(&right.node).copied().unwrap_or_default())
+                })
+                .then_with(|| left.node.cmp(&right.node))
+        })
+        .cloned()
+}
+
 impl CoordinatorService {
+    fn place_workflow_task(
+        &self,
+        nodes: &[NodeDescriptor],
+        request: &PlacementRequest,
+    ) -> Result<Placement, PlacementError> {
+        let candidates = nodes
+            .iter()
+            .filter_map(|node| {
+                DefaultScheduler
+                    .place(std::slice::from_ref(node), request)
+                    .ok()
+            })
+            .collect::<Vec<_>>();
+        if candidates.is_empty() {
+            return DefaultScheduler.place(nodes, request);
+        }
+        let active_by_node = self
+            .active_tasks
+            .iter()
+            .filter(|(tenant, project, _, _, _)| {
+                tenant == &request.tenant && project == &request.project
+            })
+            .fold(BTreeMap::<NodeId, usize>::new(), |mut counts, key| {
+                *counts.entry(key.3.clone()).or_default() += 1;
+                counts
+            });
+        let mut selected = select_task_placement(&candidates, &active_by_node)
+            .expect("one or more compatible task-placement candidates should remain");
+        let selected_load = active_by_node
+            .get(&selected.node)
+            .copied()
+            .unwrap_or_default();
+        if candidates.iter().any(|candidate| {
+            candidate.score == selected.score
+                && active_by_node
+                    .get(&candidate.node)
+                    .copied()
+                    .unwrap_or_default()
+                    > selected_load
+        }) {
+            selected.reasons.push(format!(
+                "least active equal-locality node ({selected_load} active assignment(s))"
+            ));
+        }
+        Ok(selected)
+    }
+
     pub(super) fn capture_task_restart_checkpoint(
         &mut self,
         assignment: &TaskAssignment,
@@ -49,14 +118,14 @@ impl CoordinatorService {
         let mut objects = BTreeMap::new();
         let mut missing_required_artifact = false;
         for artifact in &task_spec.required_artifacts {
-            let Some(metadata) = self.artifact_registry.metadata(artifact) else {
+            let Some(metadata) =
+                self.artifact_registry
+                    .metadata(&assignment.tenant, &assignment.project, artifact)
+            else {
                 missing_required_artifact = true;
                 continue;
             };
-            if metadata.tenant != assignment.tenant
+            if metadata.retaining_nodes.is_empty() {
-                || metadata.project != assignment.project
-                || metadata.retaining_nodes.is_empty()
-            {
                 missing_required_artifact = true;
                 continue;
             }
@@ -414,17 +483,14 @@ impl CoordinatorService {
         .validate()
         .map_err(CoordinatorServiceError::Protocol)?;
         for artifact in &task_spec.required_artifacts {
-            let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
+            let metadata = self
+                .artifact_registry
+                .metadata(&tenant, &project, artifact)
+                .ok_or_else(|| {
                     CoordinatorError::Unauthorized(format!(
-                    "required artifact {artifact} is unavailable or has expired"
+                        "required artifact {artifact} is unavailable or has expired in this tenant/project scope"
                     ))
                 })?;
-            if metadata.tenant != tenant || metadata.project != project {
-                return Err(CoordinatorError::Unauthorized(format!(
-                    "required artifact {artifact} is outside the task tenant/project scope"
-                ))
-                .into());
-            }
             if metadata.retaining_nodes.is_empty() {
                 return Err(CoordinatorError::Unauthorized(format!(
                     "required artifact {artifact} has no retaining node"
@@ -456,7 +522,7 @@ impl CoordinatorService {
             prefer_node: None,
         };
         let nodes = self.live_node_descriptors();
-        let placement = match DefaultScheduler.place(&nodes, &request) {
+        let placement = match self.place_workflow_task(&nodes, &request) {
             Ok(placement) => placement,
             Err(err) if wait_for_node => {
                 let reason = if err.message.is_empty() {
@@ -479,13 +545,22 @@ impl CoordinatorService {
                     task_spec,
                     wasm_module_base64,
                 });
+                let queued_tasks = self
+                    .pending_task_launches
+                    .iter()
+                    .filter(|pending| {
+                        pending.tenant == tenant
+                            && pending.project == project
+                            && pending.process == process
+                    })
+                    .count();
                 return Ok(CoordinatorResponse::TaskQueued {
                     process,
                     task,
                     actor,
                     reason,
                     charged_spawns,
-                    queued_tasks: self.pending_task_launches.len(),
+                    queued_tasks,
                 });
             }
             Err(err) => return Err(err.into()),
@@ -596,7 +671,9 @@ impl CoordinatorService {
                 ));
             };
             self.task_attempts.remove(&removable);
+            self.task_terminal_states.remove(&removable);
         }
+        self.task_terminal_states.remove(&key);
         let attempts = self.task_attempts.entry(key).or_default();
         for attempt in attempts.iter_mut() {
             attempt.current = false;
@@ -694,3 +771,42 @@ fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option<serde_json:
     let mut descriptors = super::main_runtime::task_descriptors(&module).ok()?;
     descriptors.remove(assignment.task_spec.task_definition.as_str())
 }
+
+#[cfg(test)]
+mod placement_tests {
+    use super::*;
+
+    fn placement(node: &str, score: i64) -> Placement {
+        Placement {
+            node: NodeId::from(node),
+            score,
+            reasons: Vec::new(),
+        }
+    }
+
+    #[test]
+    fn equal_locality_prefers_the_least_active_node() {
+        let candidates = vec![placement("busy", 40), placement("idle", 40)];
+        let active = BTreeMap::from([(NodeId::from("busy"), 1)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("idle")
+        );
+    }
+
+    #[test]
+    fn locality_score_remains_stronger_than_load_balancing() {
+        let candidates = vec![placement("warm", 50), placement("cold", 40)];
+        let active = BTreeMap::from([(NodeId::from("warm"), 2)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("warm")
+        );
+    }
+}

crates/clusterflux-coordinator/src/service/processes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::keys::{process_control_key, task_control_key};
 use super::{
@@ -47,14 +47,10 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "task assignment poll is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
         let assignment_key = (tenant.clone(), project.clone(), node.clone());
         let assignment = self
             .task_assignments
@@ -77,7 +73,11 @@ impl CoordinatorService {
         project: &ProjectId,
         node: &NodeId,
     ) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
-        let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
+        let Some(descriptor) = self
+            .node_descriptors
+            .get(&NodeScopeKey::from_refs(tenant, project, node))
+            .cloned()
+        else {
             return Ok(None);
         };
         let mut remaining = VecDeque::new();
@@ -233,16 +233,14 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "source preparation completion is outside the enrolled tenant/project scope"
+        let descriptor = self
-                    .to_owned(),
+            .node_descriptors
-            )
+            .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
-            .into());
+            .ok_or_else(|| {
-        }
-        let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
                 CoordinatorError::Unauthorized(
                     "source preparation completion requires a node capability report".to_owned(),
                 )
@@ -273,6 +271,7 @@ impl CoordinatorService {
         agent_signature: Option<AgentSignedRequest>,
         request_payload_digest: Option<&Digest>,
         process: String,
+        launch_attempt: Option<String>,
         restart: bool,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let tenant = TenantId::new(tenant);
@@ -383,6 +382,10 @@ impl CoordinatorService {
                     || attempt_project != &project
                     || attempt_process != &process
             });
+        self.task_terminal_states
+            .retain(|(task_tenant, task_project, task_process, _), _| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
         self.restart_launches
             .retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
                 attempt_tenant != &tenant
@@ -392,10 +395,19 @@ impl CoordinatorService {
         self.debug_audit_events.retain(|event| {
             event.tenant != tenant || event.project != project || event.process != process
         });
-        self.coordinator
+        let active = self.coordinator.start_process_for_launch_attempt(
-            .start_process(tenant, project, process.clone());
+            tenant.clone(),
+            project.clone(),
+            process.clone(),
+            launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
+        );
+        self.record_process_started(&tenant, &project, &process, now_epoch_seconds);
         Ok(CoordinatorResponse::ProcessStarted {
             process,
+            launch_attempt: active
+                .launch_attempt
+                .as_ref()
+                .map(|attempt| attempt.as_str().to_owned()),
             epoch: self.coordinator.coordinator_epoch(),
             actor,
             charged_spawns,
@@ -404,14 +416,18 @@ impl CoordinatorService {
 
     pub(super) fn handle_reconnect_node(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         process: String,
         epoch: u64,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
         let process = ProcessId::new(process);
         self.coordinator
-            .reconnect_node(&node, Some((&process, epoch)))?;
+            .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
         Ok(CoordinatorResponse::NodeReconnected { node, process })
     }
 
@@ -502,6 +518,13 @@ impl CoordinatorService {
         }
         let process_key = process_control_key(&tenant, &project, &process);
         if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Cancelled,
+                self.current_epoch_seconds()?,
+            );
             self.coordinator
                 .abort_process(&tenant, &project, &process)?;
             self.clear_operator_panel_state(&tenant, &project, &process);
@@ -519,6 +542,7 @@ impl CoordinatorService {
         project: String,
         actor_user: String,
         process: String,
+        launch_attempt: Option<String>,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let tenant = TenantId::new(tenant);
         let project = ProjectId::new(project);
@@ -534,6 +558,17 @@ impl CoordinatorService {
             })?;
         debug_assert_eq!(active.tenant, tenant);
         debug_assert_eq!(active.project, project);
+        let launch_attempt = launch_attempt.map(clusterflux_core::LaunchAttemptId::new);
+        if let Some(expected) = launch_attempt.as_ref() {
+            if active.launch_attempt.as_ref() != Some(expected) {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "launch rollback denied: attempt {} does not own process {}",
+                    expected.as_str(),
+                    process.as_str()
+                ))
+                .into());
+            }
+        }
 
         let process_key = process_control_key(&tenant, &project, &process);
         self.process_cancellations.remove(&process_key);
@@ -575,8 +610,24 @@ impl CoordinatorService {
             }
         }
 
+        self.record_process_terminal(
+            &tenant,
+            &project,
+            &process,
+            super::ProcessFinalResult::Cancelled,
+            self.current_epoch_seconds()?,
+        );
+        if let Some(launch_attempt) = launch_attempt.as_ref() {
+            self.coordinator.abort_process_for_launch_attempt(
+                &tenant,
+                &project,
+                &process,
+                launch_attempt,
+            )?;
+        } else {
             self.coordinator
                 .abort_process(&tenant, &project, &process)?;
+        }
         let active_restart_tasks = aborted_tasks
             .iter()
             .map(|target| target.task.clone())
@@ -624,10 +675,18 @@ impl CoordinatorService {
             .map(|active| {
                 let process_key = process_control_key(&active.tenant, &active.project, &active.id);
                 let main = self.main_runtime.controls.get(&process_key);
+                let stored = self.process_summaries.get(&process_key);
+                let stored_main_state = stored
+                    .and_then(|summary| summary.main_terminal_state.as_ref())
+                    .map(|state| match state {
+                        super::TaskTerminalState::Completed => "completed",
+                        super::TaskTerminalState::Failed => "failed",
+                        super::TaskTerminalState::Cancelled => "cancelled",
+                    });
                 let state = if self.process_cancellations.contains(&process_key) {
                     "cancelling"
                 } else {
-                    main.map_or("running", |main| main.state.as_str())
+                    "running"
                 };
                 let main_wait_state = main.and_then(|main| {
                     if main.state != "running" {
@@ -652,9 +711,15 @@ impl CoordinatorService {
                 VirtualProcessStatus {
                     process: active.id,
                     state: state.to_owned(),
-                    main_task_definition: main.map(|main| main.task_definition.clone()),
+                    main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
-                    main_task_instance: main.map(|main| main.task_instance.clone()),
+                        || stored.and_then(|summary| summary.main_task_definition.clone()),
-                    main_state: main.map(|main| main.state.clone()),
+                    ),
+                    main_task_instance: main
+                        .map(|main| main.task_instance.clone())
+                        .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
+                    main_state: main
+                        .map(|main| main.state.clone())
+                        .or_else(|| stored_main_state.map(str::to_owned)),
                     main_wait_state,
                     main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
                     connected_nodes: active.connected_nodes.into_iter().collect(),

crates/clusterflux-coordinator/src/service/protocol/responses.rs

@@ -183,6 +183,121 @@ pub struct VirtualProcessStatus {
     pub coordinator_epoch: u64,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum SourcePreparationDisposition {
     Pending { reason: String },
@@ -213,7 +328,7 @@ pub enum CoordinatorResponse {
         manual_review: bool,
         sanitized_reason: Option<String>,
         next_actions: Vec<String>,
-        private_moderation_details_exposed: bool,
+        sensitive_moderation_details_exposed: bool,
         signup_failure_details_exposed: bool,
     },
     AdminStatus {
@@ -282,6 +397,11 @@ pub enum CoordinatorResponse {
         descriptors: Vec<NodeDescriptor>,
         actor: UserId,
     },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     NodeCredentialRevoked {
         node: NodeId,
         tenant: TenantId,
@@ -344,6 +464,8 @@ pub enum CoordinatorResponse {
     },
     ProcessStarted {
         process: ProcessId,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        launch_attempt: Option<String>,
         epoch: u64,
         actor: WorkflowActor,
         charged_spawns: u64,
@@ -371,6 +493,11 @@ pub enum CoordinatorResponse {
         processes: Vec<VirtualProcessStatus>,
         actor: UserId,
     },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     QuotaStatus {
         tenant: TenantId,
         project: ProjectId,
@@ -428,6 +555,7 @@ pub enum CoordinatorResponse {
     DebugBreakpoints {
         process: ProcessId,
         actor: UserId,
+        revision: u64,
         probe_symbols: Vec<String>,
         hit_epoch: Option<u64>,
         hit_task: Option<TaskInstanceId>,
@@ -480,6 +608,17 @@ pub enum CoordinatorResponse {
         stderr_tail: String,
         backpressured: bool,
     },
+    TaskLogChunkRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        sequence: Option<u64>,
+        next_offset: u64,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
     VfsMetadataRecorded {
         process: ProcessId,
         task: TaskInstanceId,
@@ -516,6 +655,13 @@ pub enum CoordinatorResponse {
     ArtifactDownloadLink {
         link: DownloadLink,
     },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
     ArtifactDownloadLinkRevoked {
         link: DownloadLink,
     },
@@ -540,6 +686,24 @@ pub enum CoordinatorResponse {
         artifact_size_bytes: u64,
     },
     Error {
-        message: String,
+        #[serde(flatten)]
+        error: clusterflux_core::ApiError,
     },
 }
+
+impl CoordinatorResponse {
+    pub fn error(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        Self::Error {
+            error: clusterflux_core::ApiError::from_message(request_id, message),
+        }
+    }
+
+    pub fn service_error(
+        request_id: impl Into<String>,
+        error: &crate::service::CoordinatorServiceError,
+    ) -> Self {
+        Self::Error {
+            error: error.api_error(request_id),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/quota.rs

@@ -260,22 +260,6 @@ impl CoordinatorQuota {
         self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
     }
 
-    pub(super) fn can_charge_log_bytes(
-        &self,
-        tenant: &TenantId,
-        project: &ProjectId,
-        bytes: u64,
-        now_epoch_seconds: u64,
-    ) -> Result<(), LimitError> {
-        self.can_charge(
-            tenant,
-            project,
-            LimitKind::LogBytes,
-            bytes,
-            now_epoch_seconds,
-        )
-    }
-
     pub(super) fn charge_log_bytes(
         &mut self,
         tenant: &TenantId,

crates/clusterflux-coordinator/src/service/relay.rs

@@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage {
     pub egress_bytes: u64,
     pub abandoned_or_failed_bytes: u64,
     pub reserved_bytes: u64,
+    pub lifetime_ingress_bytes: u64,
+    pub lifetime_egress_bytes: u64,
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    pub completed_transfers: u64,
+    pub failed_transfers: u64,
+    pub cancelled_transfers: u64,
+    pub expired_transfers: u64,
+    pub tracked_scopes: usize,
+    pub max_active_global: usize,
+    pub max_tracked_scopes: usize,
 }
 
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
@@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState {
     pub ingress_used: u64,
     pub egress_used: u64,
     pub abandoned_or_failed_used: u64,
+    #[serde(default)]
+    pub lifetime_ingress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_egress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    #[serde(default)]
+    pub completed_transfers: u64,
+    #[serde(default)]
+    pub failed_transfers: u64,
+    #[serde(default)]
+    pub cancelled_transfers: u64,
+    #[serde(default)]
+    pub expired_transfers: u64,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger {
     ingress_used: u64,
     egress_used: u64,
     abandoned_or_failed_used: u64,
+    lifetime_ingress_bytes: u64,
+    lifetime_egress_bytes: u64,
+    lifetime_abandoned_or_failed_bytes: u64,
+    completed_transfers: u64,
+    failed_transfers: u64,
+    cancelled_transfers: u64,
+    expired_transfers: u64,
 }
 
 impl Default for ArtifactRelayLedger {
@@ -165,6 +196,13 @@ impl ArtifactRelayLedger {
             ingress_used: 0,
             egress_used: 0,
             abandoned_or_failed_used: 0,
+            lifetime_ingress_bytes: 0,
+            lifetime_egress_bytes: 0,
+            lifetime_abandoned_or_failed_bytes: 0,
+            completed_transfers: 0,
+            failed_transfers: 0,
+            cancelled_transfers: 0,
+            expired_transfers: 0,
         }
     }
 
@@ -218,6 +256,13 @@ impl ArtifactRelayLedger {
             ingress_used: state.ingress_used,
             egress_used: state.egress_used,
             abandoned_or_failed_used: state.abandoned_or_failed_used,
+            lifetime_ingress_bytes: state.lifetime_ingress_bytes,
+            lifetime_egress_bytes: state.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: state.completed_transfers,
+            failed_transfers: state.failed_transfers,
+            cancelled_transfers: state.cancelled_transfers,
+            expired_transfers: state.expired_transfers,
         }
     }
 
@@ -260,6 +305,13 @@ impl ArtifactRelayLedger {
             ingress_used: self.ingress_used,
             egress_used: self.egress_used,
             abandoned_or_failed_used: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
         }
     }
 
@@ -491,9 +543,11 @@ impl ArtifactRelayLedger {
         if ingress {
             reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
             self.ingress_used = self.ingress_used.saturating_add(bytes);
+            self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
         } else {
             reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
             self.egress_used = self.egress_used.saturating_add(bytes);
+            self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
         }
         let project_key = (
             reservation.scope.tenant.clone(),
@@ -554,10 +608,29 @@ impl ArtifactRelayLedger {
     pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
         if let Some(reservation) = self.reservations.remove(key) {
             if reason != RelayFinishReason::Completed {
+                let abandoned_or_failed_bytes = reservation
+                    .ingress_bytes
+                    .saturating_add(reservation.egress_bytes);
                 self.abandoned_or_failed_used = self
                     .abandoned_or_failed_used
-                    .saturating_add(reservation.ingress_bytes)
+                    .saturating_add(abandoned_or_failed_bytes);
-                    .saturating_add(reservation.egress_bytes);
+                self.lifetime_abandoned_or_failed_bytes = self
+                    .lifetime_abandoned_or_failed_bytes
+                    .saturating_add(abandoned_or_failed_bytes);
+            }
+            match reason {
+                RelayFinishReason::Completed => {
+                    self.completed_transfers = self.completed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Failed => {
+                    self.failed_transfers = self.failed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Cancelled => {
+                    self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Expired => {
+                    self.expired_transfers = self.expired_transfers.saturating_add(1);
+                }
             }
         }
     }
@@ -580,6 +653,18 @@ impl ArtifactRelayLedger {
             ingress_bytes: self.ingress_used,
             egress_bytes: self.egress_used,
             abandoned_or_failed_bytes: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
+            tracked_scopes: self.project_used.len()
+                + self.tenant_used.len()
+                + self.account_used.len(),
+            max_active_global: self.configuration.max_active_global,
+            max_tracked_scopes: self.configuration.max_tracked_scopes,
             reserved_bytes: self
                 .reservations
                 .values()
@@ -588,3 +673,68 @@ impl ArtifactRelayLedger {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
+        let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
+        ledger
+            .reserve(
+                "transfer".to_owned(),
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                16,
+                16,
+                100,
+                1,
+            )
+            .unwrap();
+        ledger.charge_ingress("transfer", 24, 1).unwrap();
+        ledger.charge_egress("transfer", 24, 1).unwrap();
+        ledger.finish("transfer", RelayFinishReason::Completed);
+
+        let usage = ledger.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+
+        ledger.prepare_period(u64::MAX);
+        let usage = ledger.usage();
+        assert_eq!(usage.ingress_bytes, 0);
+        assert_eq!(usage.egress_bytes, 0);
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+
+        let restored = ArtifactRelayLedger::from_durable(
+            ArtifactRelayConfiguration::unlimited(),
+            ledger.durable_state(),
+        );
+        let usage = restored.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+    }
+
+    #[test]
+    fn older_durable_relay_state_defaults_new_metrics() {
+        let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
+            "reservations": {},
+            "period": 1,
+            "project_used": [],
+            "tenant_used": [],
+            "account_used": [],
+            "ingress_used": 3,
+            "egress_used": 4,
+            "abandoned_or_failed_used": 2
+        }))
+        .unwrap();
+
+        assert_eq!(state.lifetime_ingress_bytes, 0);
+        assert_eq!(state.lifetime_egress_bytes, 0);
+        assert_eq!(state.completed_transfers, 0);
+    }
+}

crates/clusterflux-coordinator/src/service/routing.rs

@@ -5,6 +5,9 @@ impl CoordinatorService {
         &mut self,
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        request
+            .validate_external_identifiers()
+            .map_err(CoordinatorServiceError::Protocol)?;
         self.pump_main_runtime_commands();
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
@@ -42,7 +45,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -271,9 +274,17 @@ impl CoordinatorService {
                 enrollment_grant,
             ),
             CoordinatorRequest::NodeHeartbeat {
+                tenant,
+                project,
                 node,
                 node_signature,
-            } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
+            } => self.handle_node_heartbeat(
+                tenant,
+                project,
+                node,
+                node_signature,
+                &request_payload_digest,
+            ),
             CoordinatorRequest::SignedNode {
                 node,
                 node_signature,
@@ -287,6 +298,13 @@ impl CoordinatorService {
                 project,
                 actor_user,
             } => self.handle_list_node_descriptors(tenant, project, actor_user),
+            CoordinatorRequest::ListNodeSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::RevokeNodeCredential {
                 tenant,
                 project,
@@ -370,6 +388,7 @@ impl CoordinatorService {
                 agent_public_key_fingerprint,
                 agent_signature,
                 process,
+                launch_attempt,
                 restart,
             } => self.handle_start_process(
                 tenant,
@@ -380,6 +399,7 @@ impl CoordinatorService {
                 agent_signature,
                 Some(&request_payload_digest),
                 process,
+                launch_attempt,
                 restart,
             ),
             CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
@@ -401,12 +421,20 @@ impl CoordinatorService {
                 project,
                 actor_user,
                 process,
-            } => self.handle_abort_process(tenant, project, actor_user, process),
+                launch_attempt,
+            } => self.handle_abort_process(tenant, project, actor_user, process, launch_attempt),
             CoordinatorRequest::ListProcesses {
                 tenant,
                 project,
                 actor_user,
             } => self.handle_list_processes(tenant, project, actor_user),
+            CoordinatorRequest::ListProcessSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::QuotaStatus {
                 tenant,
                 project,
@@ -452,7 +480,8 @@ impl CoordinatorService {
             CoordinatorRequest::PollDebugCommand { .. }
             | CoordinatorRequest::ReportDebugState { .. }
             | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
-            CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportTaskLog { .. }
+            | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ListTaskEvents {
@@ -467,6 +496,23 @@ impl CoordinatorService {
                 actor_user,
                 process,
             } => self.handle_list_task_snapshots(tenant, project, actor_user, process),
+            CoordinatorRequest::ListRecentLogs {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             CoordinatorRequest::JoinTask {
                 tenant,
                 project,
@@ -513,6 +559,20 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            CoordinatorRequest::ListArtifacts {
+                tenant,
+                project,
+                actor_user,
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit),
+            CoordinatorRequest::GetArtifact {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+            } => self.handle_get_artifact(tenant, project, actor_user, artifact),
             CoordinatorRequest::OpenArtifactDownloadStream {
                 tenant,
                 project,
@@ -595,7 +655,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -682,6 +742,14 @@ impl CoordinatorService {
                     context.project.as_str().to_owned(),
                     actor.as_str().to_owned(),
                 ),
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self
+                .handle_list_node_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
                 .handle_revoke_node_credential(
                     context.tenant.as_str().to_owned(),
@@ -689,8 +757,11 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     node,
                 ),
-            AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
+            AuthenticatedCoordinatorRequest::StartProcess {
-                .handle_start_process(
+                process,
+                launch_attempt,
+                restart,
+            } => self.handle_start_process(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
                 Some(actor.as_str().to_owned()),
@@ -699,6 +770,7 @@ impl CoordinatorService {
                 None,
                 None,
                 process,
+                launch_attempt,
                 restart,
             ),
             request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
@@ -714,17 +786,29 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     process,
                 ),
-            AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
+            AuthenticatedCoordinatorRequest::AbortProcess {
+                process,
+                launch_attempt,
+            } => self.handle_abort_process(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
                 actor.as_str().to_owned(),
                 process,
+                launch_attempt,
             ),
             AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
                 actor.as_str().to_owned(),
             ),
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self
+                .handle_list_process_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -780,6 +864,20 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     process,
                 ),
+            AuthenticatedCoordinatorRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -799,6 +897,24 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            AuthenticatedCoordinatorRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                cursor,
+                limit,
+            ),
+            AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+            ),
             AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
                 artifact,
                 max_bytes,

crates/clusterflux-coordinator/src/service/signed_nodes.rs

@@ -1,6 +1,6 @@
-use clusterflux_core::{NodeId, NodeSignedRequest};
+use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
 
@@ -12,22 +12,24 @@ impl CoordinatorService {
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let request_kind = signed_node_request_kind(&request)?;
-        let request_node = signed_node_request_node(&request)?;
+        let request_scope = signed_node_request_scope(&request)?;
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
                 "failed to canonicalize signed node request: {error}"
             ))
         })?;
         let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
-        let signed_node = NodeId::new(signed_node);
+        let signed_node = NodeId::try_new(signed_node).map_err(|error| {
-        if request_node != signed_node {
+            CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
+        })?;
+        if request_scope.node != signed_node {
             return Err(CoordinatorError::Unauthorized(
                 "signed node request node does not match the wrapped request node".to_owned(),
             )
             .into());
         }
         self.authenticate_node_request(
-            &signed_node,
+            &request_scope,
             Some(node_signature),
             request_kind,
             &payload_digest,
@@ -144,11 +146,26 @@ impl CoordinatorService {
                 provider,
                 source_snapshot,
             ),
+            CoordinatorRequest::RequestRendezvous {
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_request_rendezvous(
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            ),
             CoordinatorRequest::ReconnectNode {
+                tenant,
+                project,
                 node,
                 process,
                 epoch,
-            } => self.handle_reconnect_node(node, process, epoch),
+            } => self.handle_reconnect_node(tenant, project, node, process, epoch),
             CoordinatorRequest::PollTaskControl {
                 tenant,
                 project,
@@ -236,6 +253,29 @@ impl CoordinatorService {
                 stderr_truncated,
                 backpressured,
             ),
+            CoordinatorRequest::ReportTaskLogChunk {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            } => self.handle_report_task_log_chunk(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            ),
             CoordinatorRequest::ReportVfsMetadata {
                 tenant,
                 project,
@@ -322,12 +362,14 @@ fn signed_node_request_kind(
         CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
         CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
         CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
+        CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"),
         CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
         CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
         CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
         CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
         CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
         CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
+        CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"),
         CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
         CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
         _ => Err(CoordinatorError::Unauthorized(
@@ -337,29 +379,135 @@ fn signed_node_request_kind(
     }
 }
 
-fn signed_node_request_node(
+fn signed_node_request_scope(
     request: &CoordinatorRequest,
-) -> Result<NodeId, CoordinatorServiceError> {
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
     match request {
-        CoordinatorRequest::ReportNodeCapabilities { node, .. }
+        CoordinatorRequest::ReportNodeCapabilities {
-        | CoordinatorRequest::PollTaskAssignment { node, .. }
+            tenant,
-        | CoordinatorRequest::PollArtifactTransfer { node, .. }
+            project,
-        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
+            node,
-        | CoordinatorRequest::FailArtifactTransfer { node, .. }
+            ..
-        | CoordinatorRequest::LaunchChildTask { node, .. }
+        }
-        | CoordinatorRequest::JoinChildTask { node, .. }
+        | CoordinatorRequest::PollTaskAssignment {
-        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
+            tenant,
-        | CoordinatorRequest::ReconnectNode { node, .. }
+            project,
-        | CoordinatorRequest::PollTaskControl { node, .. }
+            node,
-        | CoordinatorRequest::PollDebugCommand { node, .. }
+        }
-        | CoordinatorRequest::ReportDebugState { node, .. }
+        | CoordinatorRequest::PollArtifactTransfer {
-        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
+            tenant,
-        | CoordinatorRequest::ReportTaskLog { node, .. }
+            project,
-        | CoordinatorRequest::ReportVfsMetadata { node, .. }
+            node,
-        | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
+        }
+        | CoordinatorRequest::UploadArtifactTransferChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::FailArtifactTransfer {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::LaunchChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::JoinChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::CompleteSourcePreparation {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReconnectNode {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollTaskControl {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollDebugCommand {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugState {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugProbeHit {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLog {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLogChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportVfsMetadata {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::TaskCompleted {
+            tenant,
+            project,
+            node,
+            ..
+        } => node_scope_from_strings(tenant, project, node),
+        CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
+            scope.tenant.clone(),
+            scope.project.clone(),
+            source.node.clone(),
+        )),
         _ => Err(CoordinatorError::Unauthorized(
             "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
         )
         .into()),
     }
 }
+
+fn node_scope_from_strings(
+    tenant: &str,
+    project: &str,
+    node: &str,
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
+    let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
+    })?;
+    let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
+    })?;
+    let node = NodeId::try_new(node.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
+    })?;
+    Ok(NodeScopeKey::new(tenant, project, node))
+}

crates/clusterflux-coordinator/src/service/summaries.rs

@@ -0,0 +1,565 @@
+use std::collections::BTreeSet;
+use std::time::Instant;
+
+use clusterflux_core::{
+    ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId,
+};
+
+use super::keys::{process_control_key, ProcessControlKey};
+use super::{
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse,
+    CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
+    TaskAttemptState,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct StoredProcessSummary {
+    pub(super) started_at_epoch_seconds: u64,
+    pub(super) ended_at_epoch_seconds: Option<u64>,
+    pub(super) final_result: Option<ProcessFinalResult>,
+    pub(super) connected_nodes: Vec<NodeId>,
+    pub(super) main_task_definition: Option<TaskDefinitionId>,
+    pub(super) main_task_instance: Option<TaskInstanceId>,
+    pub(super) main_terminal_state: Option<super::TaskTerminalState>,
+    pub(super) order: u64,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_list_node_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = cursor.as_deref();
+        let mut nodes = self
+            .node_descriptors
+            .iter()
+            .filter(|(scope, descriptor)| {
+                scope.tenant == tenant
+                    && scope.project == project
+                    && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor)
+            })
+            .map(|(scope, descriptor)| {
+                let online = self.node_is_live(scope);
+                NodeSummary {
+                    id: descriptor.id.clone(),
+                    display_name: descriptor.id.as_str().to_owned(),
+                    online,
+                    stale: !online,
+                    last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(),
+                    capabilities: descriptor.capabilities.clone(),
+                    direct_connectivity: descriptor.direct_connectivity,
+                }
+            })
+            .collect::<Vec<_>>();
+        nodes.sort_by(|left, right| left.id.cmp(&right.id));
+        let has_more = nodes.len() > limit as usize;
+        nodes.truncate(limit as usize);
+        let next_cursor = has_more
+            .then(|| nodes.last().map(|node| node.id.as_str().to_owned()))
+            .flatten();
+        Ok(CoordinatorResponse::NodeSummaries {
+            nodes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    pub(super) fn record_process_started(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) {
+            logs.retain(|entry| &entry.process != process);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(tenant.clone(), project.clone()));
+            }
+        }
+        self.recent_log_dropped_through.remove(&key);
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, _, _), _| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.process_summary_order
+            .retain(|retained| retained != &key);
+        self.evict_process_summaries_for_project(tenant, project);
+        self.evict_process_summaries_total();
+        let order = self.next_process_summary_order;
+        self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+        self.process_summaries.insert(
+            key.clone(),
+            StoredProcessSummary {
+                started_at_epoch_seconds: now_epoch_seconds,
+                ended_at_epoch_seconds: None,
+                final_result: None,
+                connected_nodes: Vec::new(),
+                main_task_definition: None,
+                main_task_instance: None,
+                main_terminal_state: None,
+                order,
+            },
+        );
+        self.process_summary_order.push_back(key);
+    }
+
+    pub(super) fn record_process_terminal(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        final_result: ProcessFinalResult,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        let connected_nodes = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or_default();
+        let order = self.next_process_summary_order;
+        let entry = self
+            .process_summaries
+            .entry(key.clone())
+            .or_insert_with(|| {
+                self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+                self.process_summary_order.push_back(key.clone());
+                StoredProcessSummary {
+                    started_at_epoch_seconds: now_epoch_seconds,
+                    ended_at_epoch_seconds: None,
+                    final_result: None,
+                    connected_nodes: Vec::new(),
+                    main_task_definition: None,
+                    main_task_instance: None,
+                    main_terminal_state: None,
+                    order,
+                }
+            });
+        entry.ended_at_epoch_seconds = Some(now_epoch_seconds);
+        entry.final_result = Some(final_result);
+        entry.connected_nodes = connected_nodes;
+    }
+
+    pub(super) fn record_main_terminal_state(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task_definition: TaskDefinitionId,
+        task_instance: TaskInstanceId,
+        terminal_state: super::TaskTerminalState,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if !self.process_summaries.contains_key(&key) {
+            self.record_process_started(
+                tenant,
+                project,
+                process,
+                self.liveness_now_epoch_seconds(),
+            );
+        }
+        if let Some(summary) = self.process_summaries.get_mut(&key) {
+            summary.main_task_definition = Some(task_definition);
+            summary.main_task_instance = Some(task_instance);
+            summary.main_terminal_state = Some(terminal_state);
+        }
+    }
+
+    pub(super) fn handle_list_process_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = parse_order_cursor(cursor.as_deref(), "process")?;
+        let mut stored = self
+            .process_summaries
+            .iter()
+            .filter(|((entry_tenant, entry_project, _), summary)| {
+                entry_tenant == &tenant
+                    && entry_project == &project
+                    && cursor.is_none_or(|cursor| summary.order < cursor)
+            })
+            .map(|(key, summary)| (key.clone(), summary.clone()))
+            .collect::<Vec<_>>();
+        stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order));
+        let has_more = stored.len() > limit as usize;
+        stored.truncate(limit as usize);
+        let processes = stored
+            .into_iter()
+            .map(|(key, stored)| self.process_summary_from_stored(&key, stored))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| processes.last().map(|process| process.order_cursor.clone()))
+            .flatten();
+        Ok(CoordinatorResponse::ProcessSummaries {
+            processes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    fn process_summary_from_stored(
+        &self,
+        key: &ProcessControlKey,
+        stored: StoredProcessSummary,
+    ) -> ProcessSummary {
+        let (tenant, project, process) = key;
+        let active = self.coordinator.active_process(tenant, project, process);
+        let process_key = process_control_key(tenant, project, process);
+        let main_wait_state = active.and_then(|_| {
+            if self.pending_task_launches.iter().any(|pending| {
+                &pending.tenant == tenant
+                    && &pending.project == project
+                    && &pending.process == process
+            }) {
+                Some("waiting_for_node".to_owned())
+            } else if self
+                .main_runtime
+                .is_waiting_for_task(tenant, project, process)
+            {
+                Some("waiting_for_task".to_owned())
+            } else {
+                None
+            }
+        });
+        let current_debug_epoch = self.debug_epoch_summary(&process_key);
+        let awaiting_action = self.task_attempts.iter().any(
+            |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                attempt_tenant == tenant
+                    && attempt_project == project
+                    && attempt_process == process
+                    && attempts.iter().any(|attempt| {
+                        attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction
+                    })
+            },
+        );
+        let activity = if let Some(result) = &stored.final_result {
+            match result {
+                ProcessFinalResult::Completed => ProcessActivityState::Completed,
+                ProcessFinalResult::Failed => ProcessActivityState::Failed,
+                ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled,
+            }
+        } else if self.process_cancellations.contains(&process_key) {
+            ProcessActivityState::Cancelling
+        } else if current_debug_epoch
+            .as_ref()
+            .is_some_and(|epoch| epoch.partially_frozen)
+        {
+            ProcessActivityState::DebugEpochPartial
+        } else if awaiting_action {
+            ProcessActivityState::AwaitingAction
+        } else {
+            match main_wait_state.as_deref() {
+                Some("waiting_for_node") => ProcessActivityState::WaitingForNode,
+                Some("waiting_for_task") => ProcessActivityState::WaitingForTask,
+                _ => ProcessActivityState::Running,
+            }
+        };
+        let connected_nodes = active
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or(stored.connected_nodes);
+        ProcessSummary {
+            process: process.clone(),
+            lifecycle: if active.is_some() {
+                ProcessLifecycleState::Active
+            } else {
+                ProcessLifecycleState::RecentTerminal
+            },
+            activity,
+            main_wait_state,
+            started_at_epoch_seconds: stored.started_at_epoch_seconds,
+            ended_at_epoch_seconds: stored.ended_at_epoch_seconds,
+            final_result: stored.final_result,
+            connected_nodes,
+            current_debug_epoch,
+            order_cursor: format!("process:{}", stored.order),
+        }
+    }
+
+    fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option<DebugEpochSummary> {
+        let runtime = self.debug_epoch_runtime.get(key)?;
+        let acknowledgements = runtime.acknowledgements.values().collect::<Vec<_>>();
+        let all_acknowledged = !runtime.expected.is_empty()
+            && runtime
+                .expected
+                .iter()
+                .all(|participant| runtime.acknowledgements.contains_key(participant));
+        let fully_frozen = runtime.command == "freeze"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Frozen);
+        let freeze_deadline_elapsed =
+            runtime.command == "freeze" && Instant::now() >= runtime.deadline;
+        let frozen_count = acknowledgements
+            .iter()
+            .filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
+            .count();
+        let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen;
+        let fully_resumed = runtime.command == "resume"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Running);
+        let failed = acknowledgements
+            .iter()
+            .any(|ack| ack.state == DebugAcknowledgementState::Failed)
+            || (freeze_deadline_elapsed
+                && runtime
+                    .expected
+                    .iter()
+                    .any(|participant| !runtime.acknowledgements.contains_key(participant)));
+        Some(DebugEpochSummary {
+            epoch: runtime.epoch,
+            command: runtime.command.clone(),
+            fully_frozen,
+            partially_frozen,
+            fully_resumed,
+            failed,
+        })
+    }
+
+    pub(super) fn handle_list_artifacts(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: Option<String>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = process.map(ProcessId::new);
+        if let Some(process) = &process {
+            self.authorize_task_event_process_scope(&tenant, &project, process)?;
+        }
+        let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?;
+        let mut metadata = self
+            .artifact_registry
+            .metadata_for_project(&tenant, &project)
+            .filter(|metadata| {
+                process
+                    .as_ref()
+                    .is_none_or(|process| &metadata.process == process)
+                    && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor)
+            })
+            .cloned()
+            .collect::<Vec<_>>();
+        metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch));
+        let has_more = metadata.len() > limit as usize;
+        metadata.truncate(limit as usize);
+        let artifacts = metadata
+            .into_iter()
+            .map(|metadata| self.artifact_summary(metadata))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| {
+                artifacts
+                    .last()
+                    .map(|artifact| artifact.order_cursor.clone())
+            })
+            .flatten();
+        Ok(CoordinatorResponse::Artifacts {
+            artifacts,
+            next_cursor,
+        })
+    }
+
+    pub(super) fn handle_get_artifact(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let metadata = self
+            .artifact_registry
+            .metadata(&tenant, &project, &artifact)
+            .cloned()
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        Ok(CoordinatorResponse::Artifact {
+            artifact: self.artifact_summary(metadata),
+        })
+    }
+
+    fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary {
+        let live_retaining_nodes = metadata
+            .retaining_nodes
+            .iter()
+            .filter(|node| {
+                self.node_is_live(&crate::NodeScopeKey::from_refs(
+                    &metadata.tenant,
+                    &metadata.project,
+                    node,
+                ))
+            })
+            .cloned()
+            .collect::<BTreeSet<_>>();
+        let safe_node = live_retaining_nodes
+            .iter()
+            .next()
+            .cloned()
+            .or_else(|| metadata.retaining_nodes.iter().next().cloned());
+        let explicit_storage = !metadata.explicit_locations.is_empty();
+        let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage;
+        let availability = if downloadable_now {
+            ArtifactAvailability::Available
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactAvailability::NodeOffline
+        } else {
+            ArtifactAvailability::Unavailable
+        };
+        let retention_state = if explicit_storage {
+            ArtifactRetentionState::ExplicitStorage
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactRetentionState::NodeRetained
+        } else {
+            ArtifactRetentionState::Lost
+        };
+        let display_suffix = metadata.id.as_str().replace(':', "/");
+        let display_path = format!("/vfs/artifacts/{display_suffix}");
+        let display_name = display_suffix
+            .rsplit('/')
+            .next()
+            .unwrap_or(metadata.id.as_str())
+            .to_owned();
+        ArtifactSummary {
+            id: metadata.id,
+            display_path,
+            display_name,
+            process: metadata.process,
+            producer_task: metadata.producer_task,
+            safe_node,
+            digest: metadata.digest,
+            size_bytes: metadata.size,
+            availability,
+            downloadable_now,
+            retention_state,
+            explicit_storage,
+            order_cursor: format!("artifact:{}", metadata.flushed_epoch),
+        }
+    }
+
+    fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) {
+        while self
+            .process_summaries
+            .keys()
+            .filter(|(entry_tenant, entry_project, _)| {
+                entry_tenant == tenant && entry_project == project
+            })
+            .count()
+            >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT
+        {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                &key.0 == tenant
+                    && &key.1 == project
+                    && self
+                        .process_summaries
+                        .get(*key)
+                        .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn evict_process_summaries_total(&mut self) {
+        while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                self.process_summaries
+                    .get(*key)
+                    .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
+        self.process_summaries.remove(key);
+        self.recent_log_dropped_through.remove(key);
+        if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
+            logs.retain(|entry| entry.process != key.2);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
+            }
+        }
+        self.recent_log_accounted_bytes
+            .retain(|(tenant, project, process, _, _), _| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_quota_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.process_summary_order
+            .retain(|retained| retained != key);
+    }
+}
+
+fn parse_order_cursor(
+    cursor: Option<&str>,
+    expected_kind: &str,
+) -> Result<Option<u64>, CoordinatorServiceError> {
+    cursor
+        .map(|cursor| {
+            let (kind, order) = cursor.split_once(':').ok_or_else(|| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })?;
+            if kind != expected_kind {
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                )));
+            }
+            order.parse::<u64>().map_err(|_| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })
+        })
+        .transpose()
+}

crates/clusterflux-coordinator/src/service/tcp.rs

@@ -79,17 +79,15 @@ impl CoordinatorService {
                 continue;
             }
             let response = match decode_wire_request(&line) {
-                Ok(request) => match authorize_client_request(&request, authority_mode)
+                Ok((request_id, request)) => {
+                    match authorize_client_request(&request, authority_mode)
                         .and_then(|()| self.handle_request(request))
                     {
                         Ok(response) => response,
-                    Err(err) => CoordinatorResponse::Error {
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                        message: err.to_string(),
+                    }
-                    },
+                }
-                },
+                Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
-                Err(err) => CoordinatorResponse::Error {
-                    message: err.to_string(),
-                },
             };
             serde_json::to_writer(&mut writer, &response)?;
             writer.write_all(b"\n")?;
@@ -114,25 +112,19 @@ fn handle_shared_stream(
             continue;
         }
         let response = match decode_wire_request(&line) {
-            Ok(request) => match authorize_client_request(&request, authority_mode) {
+            Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) {
                 Ok(()) => match service.lock() {
                     Ok(mut service) => match service.handle_request(request) {
                         Ok(response) => response,
-                        Err(err) => CoordinatorResponse::Error {
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                            message: err.to_string(),
                     },
+                    Err(_) => {
+                        CoordinatorResponse::error(request_id, "coordinator service lock poisoned")
+                    }
                 },
-                    Err(_) => CoordinatorResponse::Error {
+                Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                        message: "coordinator service lock poisoned".to_owned(),
-                    },
-                },
-                Err(err) => CoordinatorResponse::Error {
-                    message: err.to_string(),
-                },
-            },
-            Err(err) => CoordinatorResponse::Error {
-                message: err.to_string(),
             },
+            Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
         };
         serde_json::to_writer(&mut writer, &response)?;
         writer.write_all(b"\n")?;
@@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato
     Ok((listener, addr))
 }
 
-fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
+fn decode_wire_request(
+    line: &str,
+) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> {
     serde_json::from_str::<super::CoordinatorWireRequest>(line)?
-        .into_request()
+        .into_parts()
         .map_err(CoordinatorServiceError::Protocol)
 }
 
+fn wire_request_id_hint(line: &str) -> String {
+    serde_json::from_str::<serde_json::Value>(line)
+        .ok()
+        .and_then(|value| {
+            value
+                .get("request_id")
+                .and_then(|value| value.as_str())
+                .map(str::to_owned)
+        })
+        .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok())
+        .unwrap_or_else(|| "unavailable".to_owned())
+}
+
 fn authorize_client_request(
     request: &CoordinatorRequest,
     authority_mode: ClientAuthorityMode,
@@ -180,15 +187,21 @@ fn authorize_client_request(
         }
         | CoordinatorRequest::AdminStatus { .. }
         | CoordinatorRequest::SuspendTenant { .. } => Ok(()),
-        _ => Err(CoordinatorServiceError::Protocol(
+        _ => Err(crate::CoordinatorError::Unauthorized(
             "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
                 .to_owned(),
-        )),
+        )
+        .into()),
     }
 }
 
 #[cfg(test)]
 mod transport_boundary_tests {
+    use std::io::{BufRead as _, BufReader};
+
+    use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId};
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -197,4 +210,99 @@ mod transport_boundary_tests {
         let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
         assert!(error.to_string().contains("restricted to loopback"));
     }
+
+    #[test]
+    fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() {
+        let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
+        let mut coordinator = CoordinatorService::new(11);
+        coordinator
+            .issue_cli_session(
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                "healthy-session",
+                None,
+            )
+            .unwrap();
+        let shared = Arc::new(Mutex::new(coordinator));
+        let server_shared = Arc::clone(&shared);
+        let server = std::thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap();
+        });
+
+        let mut stream = TcpStream::connect(addr).unwrap();
+        let mut reader = BufReader::new(stream.try_clone().unwrap());
+        for (index, malformed_process) in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0process".to_owned(),
+            "bad process!".to_owned(),
+            "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1),
+        ]
+        .into_iter()
+        .enumerate()
+        {
+            let malformed = coordinator_wire_request(
+                format!("malformed-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": {
+                        "type": "abort_process",
+                        "process": malformed_process,
+                        "launch_attempt": "valid-attempt"
+                    }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &malformed).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            let CoordinatorResponse::Error { error } =
+                serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
+            else {
+                panic!("malformed identifier request unexpectedly succeeded");
+            };
+            assert!(
+                error.message.contains("malformed external identifier")
+                    && error.message.contains("request.request.process"),
+                "unexpected malformed identifier response: {}",
+                error.message
+            );
+            assert_eq!(error.request_id, format!("malformed-{index}"));
+            assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
+
+            let valid = coordinator_wire_request(
+                format!("healthy-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": { "type": "auth_status" }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &valid).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            line.clear();
+            reader.read_line(&mut line).unwrap();
+            assert!(
+                matches!(
+                    serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
+                    CoordinatorResponse::AuthStatus {
+                        authenticated: true,
+                        ..
+                    }
+                ),
+                "valid authenticated traffic failed after malformed request {index}"
+            );
+        }
+
+        stream.shutdown(std::net::Shutdown::Both).unwrap();
+        server.join().unwrap();
+        assert!(!shared.is_poisoned());
+    }
 }

crates/clusterflux-coordinator/src/service/wire_protocol.rs

@@ -1,4 +1,4 @@
-use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
+use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest {
 
 impl CoordinatorWireRequest {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         match self {
-            Self::Envelope(envelope) => envelope.into_request(),
+            Self::Envelope(envelope) => envelope.into_parts(),
         }
     }
 }
@@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope {
 
 impl CoordinatorRequestEnvelope {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
             return Err(format!(
                 "unsupported coordinator wire request type {}; expected {}",
@@ -44,9 +52,9 @@ impl CoordinatorRequestEnvelope {
                 self.protocol_version, COORDINATOR_PROTOCOL_VERSION
             ));
         }
-        if self.request_id.trim().is_empty() {
+        RequestId::try_new(self.request_id.clone())
-            return Err("coordinator wire request_id must be non-empty".to_owned());
+            .map_err(|error| format!("malformed coordinator wire request_id: {error}"))?;
-        }
+        self.payload.validate_external_identifiers()?;
         let payload_operation = self.payload.operation()?;
         if self.operation != payload_operation {
             return Err(format!(
@@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope {
                 self.operation, payload_operation
             ));
         }
-        Ok(self.payload)
+        Ok((self.request_id, self.payload))
     }
 }

crates/clusterflux-core/src/api_error.rs

@@ -0,0 +1,296 @@
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCode {
+    Unauthenticated,
+    SessionExpired,
+    AccountSuspended,
+    Forbidden,
+    ValidationError,
+    NotFound,
+    Conflict,
+    ActiveProcessExists,
+    NodeOffline,
+    NoCapableNode,
+    TaskNotRestartable,
+    ArtifactUnavailable,
+    ArtifactLimitExceeded,
+    QuotaExceeded,
+    TemporaryCapacity,
+    DebugEpochPartial,
+    InternalError,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCategory {
+    Authentication,
+    Authorization,
+    Validation,
+    State,
+    Availability,
+    Resource,
+    Internal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ApiError {
+    pub code: ApiErrorCode,
+    pub category: ApiErrorCategory,
+    pub message: String,
+    pub retryable: bool,
+    pub request_id: String,
+}
+
+impl fmt::Display for ApiError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            formatter,
+            "{} (code {:?}, request {})",
+            self.message, self.code, self.request_id
+        )
+    }
+}
+
+impl std::error::Error for ApiError {}
+
+impl ApiError {
+    pub fn new(
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: impl Into<String>,
+        retryable: bool,
+        request_id: impl Into<String>,
+    ) -> Self {
+        Self {
+            code,
+            category,
+            message: message.into(),
+            retryable,
+            request_id: request_id.into(),
+        }
+    }
+
+    pub fn from_message(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        let request_id = request_id.into();
+        let message = message.into();
+        let normalized = message.to_ascii_lowercase();
+        let (code, category, retryable) = if normalized.contains("session credential has expired")
+            || normalized.contains("session expired")
+        {
+            (
+                ApiErrorCode::SessionExpired,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("tenant is suspended")
+            || normalized.contains("account is suspended")
+            || normalized.contains("suspended by hosted")
+        {
+            (
+                ApiErrorCode::AccountSuspended,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("session credential")
+            || normalized.contains("no authenticated")
+            || normalized.contains("not authenticated")
+            || normalized.contains("credential is not")
+        {
+            (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("project already has active virtual process") {
+            (
+                ApiErrorCode::ActiveProcessExists,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("no capable node") {
+            (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("node offline")
+            || normalized.contains("node is not live")
+            || normalized.contains("source node is not connected")
+            || normalized.contains("direct connectivity unavailable")
+        {
+            (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("restart")
+            && (normalized.contains("not restartable")
+                || normalized.contains("requires whole")
+                || normalized.contains("clean boundary"))
+        {
+            (
+                ApiErrorCode::TaskNotRestartable,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("exceeds download limit")
+                || normalized.contains("download session limit")
+                || normalized.contains("artifact limit"))
+        {
+            (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("does not exist")
+                || normalized.contains("not found")
+                || normalized.contains("unknown artifact"))
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("artifact")
+            && (normalized.contains("unavailable") || normalized.contains("retention"))
+        {
+            (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("resource limit")
+            || normalized.contains("quota")
+            || normalized.contains("limit exceeded")
+        {
+            (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            )
+        } else if normalized.contains("capacity")
+            || normalized.contains("temporarily full")
+            || normalized.contains("replay window is full")
+        {
+            (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("partial debug epoch")
+            || normalized.contains("debug epoch is partially")
+        {
+            (
+                ApiErrorCode::DebugEpochPartial,
+                ApiErrorCategory::State,
+                true,
+            )
+        } else if normalized.contains("malformed")
+            || normalized.contains("invalid ")
+            || normalized.contains("protocol")
+            || normalized.contains("unknown field")
+            || normalized.contains("missing field")
+            || normalized.contains("must ")
+        {
+            (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            )
+        } else if normalized.contains("outside")
+            || normalized.contains("unauthorized")
+            || normalized.contains("denied")
+            || normalized.contains("requires an authenticated")
+            || normalized.contains("may only")
+        {
+            (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("not found")
+            || normalized.contains("does not exist")
+            || normalized.contains("unknown ")
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("already")
+            || normalized.contains("conflict")
+            || normalized.contains("requires an active")
+        {
+            (ApiErrorCode::Conflict, ApiErrorCategory::State, false)
+        } else {
+            (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                false,
+            )
+        };
+        Self::new(code, category, message, retryable, request_id)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn required_machine_codes_are_stably_serialized() {
+        let required = [
+            ApiErrorCode::Unauthenticated,
+            ApiErrorCode::SessionExpired,
+            ApiErrorCode::AccountSuspended,
+            ApiErrorCode::Forbidden,
+            ApiErrorCode::ValidationError,
+            ApiErrorCode::NotFound,
+            ApiErrorCode::Conflict,
+            ApiErrorCode::ActiveProcessExists,
+            ApiErrorCode::NodeOffline,
+            ApiErrorCode::NoCapableNode,
+            ApiErrorCode::TaskNotRestartable,
+            ApiErrorCode::ArtifactUnavailable,
+            ApiErrorCode::ArtifactLimitExceeded,
+            ApiErrorCode::QuotaExceeded,
+            ApiErrorCode::TemporaryCapacity,
+            ApiErrorCode::DebugEpochPartial,
+        ];
+        let serialized = required
+            .into_iter()
+            .map(|code| serde_json::to_value(code).unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            serialized,
+            vec![
+                "unauthenticated",
+                "session_expired",
+                "account_suspended",
+                "forbidden",
+                "validation_error",
+                "not_found",
+                "conflict",
+                "active_process_exists",
+                "node_offline",
+                "no_capable_node",
+                "task_not_restartable",
+                "artifact_unavailable",
+                "artifact_limit_exceeded",
+                "quota_exceeded",
+                "temporary_capacity",
+                "debug_epoch_partial",
+            ]
+        );
+    }
+
+    #[test]
+    fn message_classification_keeps_request_identity() {
+        let error = ApiError::from_message(
+            "request-17",
+            "CLI session credential has expired; run login again",
+        );
+        assert_eq!(error.code, ApiErrorCode::SessionExpired);
+        assert_eq!(error.category, ApiErrorCategory::Authentication);
+        assert_eq!(error.request_id, "request-17");
+        assert!(!error.retryable);
+    }
+}

crates/clusterflux-core/src/artifact.rs

@@ -10,7 +10,7 @@ use crate::{
 
 const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
 const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
-const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
+const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
     pub coordinator_has_large_bytes: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct ArtifactScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub artifact: ArtifactId,
+}
+
+impl ArtifactScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
+        Self {
+            tenant,
+            project,
+            artifact,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
+        Self::new(tenant.clone(), project.clone(), artifact.clone())
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ArtifactFlush {
     pub id: ArtifactId,
@@ -148,7 +169,7 @@ pub struct ArtifactFlush {
 
 #[derive(Clone, Debug, Default)]
 pub struct ArtifactRegistry {
-    artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
+    artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
     issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
     next_epoch: u64,
 }
@@ -162,42 +183,46 @@ impl ArtifactRegistry {
     pub fn flush_metadata_bounded(
         &mut self,
         flush: ArtifactFlush,
-        pinned: &BTreeSet<ArtifactId>,
+        pinned: &BTreeSet<ArtifactScopeKey>,
     ) -> Result<ArtifactMetadata, String> {
-        let replacing_existing = self.artifacts.contains_key(&flush.id);
+        self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
-        while !replacing_existing
-            && self
-                .artifacts
-                .values()
-                .filter(|metadata| {
-                    metadata.tenant == flush.tenant
-                        && metadata.project == flush.project
-                        && metadata.process == flush.process
-                })
-                .count()
-                >= MAX_ARTIFACT_METADATA_PER_PROCESS
-        {
-            let candidate = self
-                .artifacts
-                .values()
-                .filter(|metadata| {
-                    metadata.tenant == flush.tenant
-                        && metadata.project == flush.project
-                        && metadata.process == flush.process
-                        && !pinned.contains(&metadata.id)
-                        && !self.issued_download_links.values().any(|issued| {
-                            issued.link.artifact == metadata.id
-                        })
-                })
-                .min_by_key(|metadata| metadata.flushed_epoch)
-                .map(|metadata| metadata.id.clone())
-                .ok_or_else(|| {
-                    "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
-                        .to_owned()
-                })?;
-            self.artifacts.remove(&candidate);
     }
-        self.next_epoch += 1;
+
+    pub fn flush_metadata_with_protected_processes(
+        &mut self,
+        flush: ArtifactFlush,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> Result<ArtifactMetadata, String> {
+        let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
+        let replacing = self.artifacts.contains_key(&key);
+        let retained_for_project = self
+            .artifacts
+            .values()
+            .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
+            .count();
+        let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
+            None
+        } else {
+            let mut protected_processes = protected_processes.clone();
+            protected_processes.insert(flush.process.clone());
+            Some(
+                self.project_metadata_eviction_candidate(
+                    &flush.tenant,
+                    &flush.project,
+                    pinned,
+                    &protected_processes,
+                )
+                .ok_or_else(|| {
+                    format!(
+                        "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
+                         exhausted by active or retained artifacts"
+                    )
+                })?,
+            )
+        };
+
+        self.next_epoch = self.next_epoch.saturating_add(1);
         let metadata = ArtifactMetadata {
             id: flush.id.clone(),
             tenant: flush.tenant,
@@ -212,43 +237,137 @@ impl ArtifactRegistry {
             explicit_locations: Vec::new(),
             coordinator_has_large_bytes: false,
         };
-        self.artifacts.insert(flush.id, metadata.clone());
+        if let Some(eviction) = eviction {
+            self.artifacts.remove(&eviction);
+        }
+        self.artifacts.insert(key, metadata.clone());
         Ok(metadata)
     }
 
+    pub fn enforce_project_metadata_limit(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> usize {
+        let mut evicted = 0;
+        while self
+            .artifacts
+            .values()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+            .count()
+            > MAX_ARTIFACT_METADATA_PER_PROJECT
+        {
+            let candidate = self.project_metadata_eviction_candidate(
+                tenant,
+                project,
+                pinned,
+                protected_processes,
+            );
+            let Some(candidate) = candidate else {
+                break;
+            };
+            self.artifacts.remove(&candidate);
+            evicted += 1;
+        }
+        evicted
+    }
+
+    fn project_metadata_eviction_candidate(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> Option<ArtifactScopeKey> {
+        self.artifacts
+            .values()
+            .filter(|metadata| {
+                &metadata.tenant == tenant
+                    && &metadata.project == project
+                    && !pinned.contains(&ArtifactScopeKey::from_refs(
+                        &metadata.tenant,
+                        &metadata.project,
+                        &metadata.id,
+                    ))
+                    && !protected_processes.contains(&metadata.process)
+                    && metadata.explicit_locations.is_empty()
+                    && !self.issued_download_links.values().any(|issued| {
+                        !issued.revoked
+                            && issued.link.tenant == metadata.tenant
+                            && issued.link.project == metadata.project
+                            && issued.link.artifact == metadata.id
+                    })
+            })
+            .min_by_key(|metadata| metadata.flushed_epoch)
+            .map(|metadata| {
+                ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
+            })
+    }
+
     pub fn sync_to_explicit_store(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         artifact: &ArtifactId,
         location: impl Into<String>,
     ) -> Result<(), ArtifactUnavailable> {
         let metadata = self
             .artifacts
-            .get_mut(artifact)
+            .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
             .ok_or(ArtifactUnavailable)?;
         metadata.explicit_locations.push(location.into());
         Ok(())
     }
 
-    pub fn garbage_collect_node(&mut self, node: &NodeId) {
+    pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
-        for metadata in self.artifacts.values_mut() {
+        for metadata in self
+            .artifacts
+            .values_mut()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+        {
             metadata.retaining_nodes.remove(node);
         }
     }
 
     pub fn reconcile_node_retention(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         retained_artifacts: &BTreeSet<ArtifactId>,
     ) {
-        for (artifact, metadata) in &mut self.artifacts {
+        for (key, metadata) in &mut self.artifacts {
-            if !retained_artifacts.contains(artifact) {
+            if &key.tenant != tenant || &key.project != project {
+                continue;
+            }
+            if retained_artifacts.contains(&key.artifact) {
+                metadata.retaining_nodes.insert(node.clone());
+            } else {
                 metadata.retaining_nodes.remove(node);
             }
         }
     }
 
-    pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
+    pub fn metadata(
-        self.artifacts.get(artifact)
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        artifact: &ArtifactId,
+    ) -> Option<&ArtifactMetadata> {
+        self.artifacts
+            .get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
+    }
+
+    pub fn metadata_for_project<'a>(
+        &'a self,
+        tenant: &'a TenantId,
+        project: &'a ProjectId,
+    ) -> impl Iterator<Item = &'a ArtifactMetadata> + 'a {
+        self.artifacts
+            .values()
+            .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project)
     }
 
     pub fn download_action(
@@ -259,7 +378,11 @@ impl ArtifactRegistry {
     ) -> Result<DownloadAction, DownloadError> {
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let scope = Scope {
             tenant: metadata.tenant.clone(),
@@ -313,7 +436,11 @@ impl ArtifactRegistry {
         self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         Ok(metadata.size)
     }
@@ -331,7 +458,11 @@ impl ArtifactRegistry {
         if self
             .issued_download_links
             .values()
-            .filter(|issued| issued.link.artifact == *artifact)
+            .filter(|issued| {
+                issued.link.tenant == context.tenant
+                    && issued.link.project == context.project
+                    && issued.link.artifact == *artifact
+            })
             .count()
             >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
         {
@@ -343,7 +474,11 @@ impl ArtifactRegistry {
         let action = self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
         let policy_context_digest =
@@ -396,7 +531,11 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact || issued.link.actor != context.actor {
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
+            || issued.link.actor != context.actor
+        {
             return Err(DownloadError::InvalidToken);
         }
         self.download_action(
@@ -432,7 +571,9 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
             || issued.link.max_bytes != policy.max_bytes
             || issued.link.actor != context.actor
         {
@@ -450,7 +591,11 @@ impl ArtifactRegistry {
         }
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if download_policy_context_digest(metadata, &action.source, policy)
             != issued.link.policy_context_digest
@@ -475,7 +620,11 @@ impl ArtifactRegistry {
     ) -> Result<(), DownloadError> {
         let metadata = self
             .artifacts
-            .get(&stream.link.artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &stream.link.tenant,
+                &stream.link.project,
+                &stream.link.artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if !source_is_available(metadata, &stream.link.source) {
             return Err(DownloadError::Unavailable);
@@ -594,7 +743,13 @@ mod tests {
     #[test]
     fn flush_publishes_metadata_without_coordinator_bytes() {
         let registry = registry_with_artifact();
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(!metadata.coordinator_has_large_bytes);
         assert_eq!(metadata.id, ArtifactId::from("artifact"));
@@ -613,7 +768,11 @@ mod tests {
     #[test]
     fn unsynced_node_loss_surfaces_as_unavailable() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -634,20 +793,68 @@ mod tests {
     #[test]
     fn signed_node_retention_inventory_removes_garbage_collected_location() {
         let mut registry = registry_with_artifact();
-        registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+            &BTreeSet::new(),
+        );
 
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(metadata.retaining_nodes.is_empty());
     }
 
+    #[test]
+    fn signed_node_retention_inventory_restores_readvertised_location() {
+        let mut registry = registry_with_artifact();
+        let node = NodeId::from("node");
+        let artifact = ArtifactId::from("artifact");
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+        );
+
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+            &BTreeSet::from([artifact.clone()]),
+        );
+
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &artifact,
+            )
+            .unwrap();
+        assert!(metadata.retaining_nodes.contains(&node));
+    }
+
     #[test]
     fn explicit_user_storage_location_survives_node_retention_loss() {
         let mut registry = registry_with_artifact();
         registry
-            .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
+            .sync_to_explicit_store(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+                "s3://bucket/app",
+            )
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -668,7 +875,11 @@ mod tests {
         );
         assert!(
             !registry
-                .metadata(&ArtifactId::from("artifact"))
+                .metadata(
+                    &TenantId::from("tenant"),
+                    &ProjectId::from("project"),
+                    &ArtifactId::from("artifact"),
+                )
                 .unwrap()
                 .coordinator_has_large_bytes
         );
@@ -699,7 +910,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
     }
 
     #[test]
@@ -719,13 +930,206 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
+    }
+
+    #[test]
+    fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
+        let artifact = ArtifactId::from("shared-artifact");
+        let tenant_a = TenantId::from("tenant-a");
+        let project_a = ProjectId::from("project-a");
+        let tenant_b = TenantId::from("tenant-b");
+        let project_b = ProjectId::from("project-b");
+        let project_c = ProjectId::from("project-c");
+        let node_a = NodeId::from("node-a");
+        let node_b = NodeId::from("node-b");
+        let node_c = NodeId::from("node-c");
+        let mut registry = ArtifactRegistry::default();
+        for (tenant, project, process, node, content, size) in [
+            (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
+            (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
+            (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
+        ] {
+            registry.flush_metadata(ArtifactFlush {
+                id: artifact.clone(),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: ProcessId::from(process),
+                producer_task: TaskInstanceId::from("producer"),
+                retaining_node: node.clone(),
+                digest: Digest::sha256(content),
+                size,
+            });
+        }
+
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_c, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-c")
+        );
+        registry.flush_metadata(ArtifactFlush {
+            id: artifact.clone(),
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            process: ProcessId::from("process-a"),
+            producer_task: TaskInstanceId::from("producer-repeat"),
+            retaining_node: node_a.clone(),
+            digest: Digest::sha256("bytes-a-repeat"),
+            size: 18,
+        });
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a-repeat")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+
+        registry
+            .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
+            .unwrap();
+        registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
+        let context_a = AuthContext {
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let context_b = AuthContext {
+            tenant: tenant_b.clone(),
+            project: project_b.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let policy = DownloadPolicy { max_bytes: 100 };
+        assert_eq!(
+            registry
+                .download_action(&context_a, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::ExplicitStore("store://tenant-a".to_owned())
+        );
+        assert_eq!(
+            registry
+                .download_action(&context_b, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::RetainedNode(node_b)
+        );
+
+        let first_a = registry
+            .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        let first_b = registry
+            .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        assert_ne!(
+            first_a.scoped_token_digest, first_b.scoped_token_digest,
+            "the full artifact scope must contribute to download tokens"
+        );
+        for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
+            registry
+                .create_download_link(
+                    &context_a,
+                    &artifact,
+                    &policy,
+                    &format!("tenant-a-{index}"),
+                    10,
+                    60,
+                )
+                .unwrap();
+        }
+        assert!(matches!(
+            registry.create_download_link(
+                &context_a,
+                &artifact,
+                &policy,
+                "tenant-a-over-limit",
+                10,
+                60,
+            ),
+            Err(DownloadError::Usage(_))
+        ));
+        registry
+            .create_download_link(
+                &context_b,
+                &artifact,
+                &policy,
+                "tenant-b-still-independent",
+                10,
+                60,
+            )
+            .unwrap();
+        registry
+            .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
+            .unwrap();
+
+        let limits = ResourceLimits {
+            limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
+        };
+        let mut meter = ResourceMeter::default();
+        registry
+            .open_download_stream(
+                DownloadStreamRequest {
+                    context: &context_b,
+                    artifact: &artifact,
+                    policy: &policy,
+                    presented_token_digest: &first_b.scoped_token_digest,
+                    now_epoch_seconds: 11,
+                    limits: &limits,
+                },
+                &mut meter,
+            )
+            .expect("revoking tenant A's link must not revoke tenant B's link");
+        assert_eq!(
+            registry
+                .open_download_stream(
+                    DownloadStreamRequest {
+                        context: &context_b,
+                        artifact: &artifact,
+                        policy: &policy,
+                        presented_token_digest: &first_a.scoped_token_digest,
+                        now_epoch_seconds: 11,
+                        limits: &limits,
+                    },
+                    &mut meter,
+                )
+                .unwrap_err(),
+            DownloadError::InvalidToken
+        );
     }
 
     #[test]
     fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -1002,51 +1406,200 @@ mod tests {
     }
 
     #[test]
-    fn artifact_metadata_is_bounded_without_evicting_pins() {
+    fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
         let mut registry = ArtifactRegistry::default();
-        let mut pinned = BTreeSet::new();
+        let tenant = TenantId::from("tenant");
-        for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
+        let project = ProjectId::from("project");
-            let id = ArtifactId::new(format!("artifact-{index}"));
+        registry.flush_metadata(ArtifactFlush {
-            pinned.insert(id.clone());
+            id: ArtifactId::from("artifact-0"),
-            registry
+            tenant: TenantId::from("other-tenant"),
-                .flush_metadata_bounded(
+            project: ProjectId::from("other-project"),
-                    ArtifactFlush {
-                        id,
-                        tenant: TenantId::from("tenant"),
-                        project: ProjectId::from("project"),
             process: ProcessId::from("process"),
+            producer_task: TaskInstanceId::from("other-task"),
+            retaining_node: NodeId::from("other-node"),
+            digest: Digest::sha256("other-content"),
+            size: 1,
+        });
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
+            let id = ArtifactId::new(format!("artifact-{index}"));
+            registry.flush_metadata(ArtifactFlush {
+                id,
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: ProcessId::new(if index == 3 {
+                    "active-process-3".to_owned()
+                } else {
+                    format!("completed-process-{index}")
+                }),
                 producer_task: TaskInstanceId::new(format!("task-{index}")),
                 retaining_node: NodeId::from("node"),
                 digest: Digest::sha256(format!("content-{index}")),
                 size: 1,
-                    },
+            });
-                    &pinned,
+        }
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        let pinned = BTreeSet::from([ArtifactScopeKey::new(
+            tenant.clone(),
+            project.clone(),
+            ArtifactId::from("artifact-0"),
+        )]);
+        registry
+            .sync_to_explicit_store(
+                &tenant,
+                &project,
+                &ArtifactId::from("artifact-1"),
+                "store://retained-export",
             )
             .unwrap();
-        }
+        let context = AuthContext {
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        registry
+            .create_download_link(
+                &context,
+                &ArtifactId::from("artifact-2"),
+                &DownloadPolicy { max_bytes: 1 },
+                "active-download",
+                10,
+                60,
+            )
+            .unwrap();
+        let protected_processes = BTreeSet::from([
+            ProcessId::from("active-process-3"),
+            ProcessId::from("active-process"),
+        ]);
         let next = ArtifactFlush {
             id: ArtifactId::from("artifact-next"),
-            tenant: TenantId::from("tenant"),
+            tenant: tenant.clone(),
-            project: ProjectId::from("project"),
+            project: project.clone(),
-            process: ProcessId::from("process"),
+            process: ProcessId::from("active-process"),
             producer_task: TaskInstanceId::from("task-next"),
             retaining_node: NodeId::from("node"),
             digest: Digest::sha256("next"),
             size: 1,
         };
+        registry
+            .flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
+            .unwrap();
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
+            assert!(
+                registry
+                    .metadata(&tenant, &project, &ArtifactId::from(id))
+                    .is_some(),
+                "{id} was evicted despite being pinned, exported, downloaded, or live"
+            );
+        }
+        assert!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
+                .is_none(),
+            "the oldest completed unprotected metadata should be evicted"
+        );
         assert!(registry
-            .flush_metadata_bounded(next.clone(), &pinned)
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
-            .unwrap_err()
-            .contains("pinned"));
-
-        pinned.remove(&ArtifactId::from("artifact-0"));
-        registry.flush_metadata_bounded(next, &pinned).unwrap();
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
-        assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
-        assert!(registry
-            .metadata(&ArtifactId::from("artifact-next"))
             .is_some());
+        assert_eq!(
+            registry
+                .metadata(
+                    &TenantId::from("other-tenant"),
+                    &ProjectId::from("other-project"),
+                    &ArtifactId::from("artifact-0"),
+                )
+                .unwrap()
+                .digest,
+            Digest::sha256("other-content"),
+            "eviction and pins in one scope must not affect an equal ID in another scope"
+        );
+    }
+
+    #[test]
+    fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
+        let mut registry = ArtifactRegistry::default();
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let active_process = ProcessId::from("active-process");
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
+            registry.flush_metadata(ArtifactFlush {
+                id: ArtifactId::new(format!("artifact-{index}")),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::new(format!("task-{index}")),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256(format!("content-{index}")),
+                size: 1,
+            });
+        }
+
+        let replacement = ArtifactFlush {
+            id: ArtifactId::from("artifact-0"),
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: active_process.clone(),
+            producer_task: TaskInstanceId::from("replacement-task"),
+            retaining_node: NodeId::from("node"),
+            digest: Digest::sha256("replacement"),
+            size: 2,
+        };
+        registry
+            .flush_metadata_with_protected_processes(
+                replacement,
+                &BTreeSet::new(),
+                &BTreeSet::from([active_process.clone()]),
+            )
+            .expect("replacement must remain possible at the metadata bound");
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement")
+        );
+
+        let epoch_before_rejection = registry.next_epoch;
+        let result = registry.flush_metadata_with_protected_processes(
+            ArtifactFlush {
+                id: ArtifactId::from("artifact-rejected"),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::from("rejected-task"),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256("rejected"),
+                size: 3,
+            },
+            &BTreeSet::new(),
+            &BTreeSet::from([active_process]),
+        );
+
+        assert!(result
+            .unwrap_err()
+            .contains("exhausted by active or retained artifacts"));
+        assert_eq!(registry.next_epoch, epoch_before_rejection);
+        assert_eq!(
+            registry.metadata_for_project(&tenant, &project).count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT
+        );
+        assert!(registry
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
+            .is_none());
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement"),
+            "rejection must not modify existing metadata"
+        );
     }
 
     #[test]
@@ -1135,7 +1688,11 @@ mod tests {
         registry
             .stream_download_chunk(&mut stream, &limits, &mut meter, 16)
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
 
         assert_eq!(
             registry

crates/clusterflux-core/src/auth.rs

@@ -364,7 +364,13 @@ pub fn agent_workflow_request_scope_from_payload(
                 .get("task_instance")
                 .and_then(Value::as_str)
                 .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
-            (process, Some(TaskInstanceId::from(task)))
+            (
+                process,
+                Some(
+                    TaskInstanceId::try_new(task)
+                        .map_err(|error| format!("malformed launch_task task instance: {error}"))?,
+                ),
+            )
         }
         _ => {
             return Err(format!(
@@ -373,10 +379,13 @@ pub fn agent_workflow_request_scope_from_payload(
         }
     };
     AgentWorkflowRequestScope::new(
-        TenantId::from(tenant),
+        TenantId::try_new(tenant)
-        ProjectId::from(project),
+            .map_err(|error| format!("malformed agent workflow tenant: {error}"))?,
+        ProjectId::try_new(project)
+            .map_err(|error| format!("malformed agent workflow project: {error}"))?,
         request_kind,
-        ProcessId::from(process),
+        ProcessId::try_new(process)
+            .map_err(|error| format!("malformed agent workflow process: {error}"))?,
         task,
     )
 }

crates/clusterflux-core/src/execution.rs

@@ -300,6 +300,27 @@ impl TaskBoundaryValue {
             _ => Vec::new(),
         }
     }
+
+    fn rebind_source_snapshot(&mut self, replacement: &Digest) -> usize {
+        match self {
+            Self::SourceSnapshot(snapshot) => {
+                *snapshot = replacement.clone();
+                1
+            }
+            Self::Structured(structured) => structured
+                .handles
+                .iter_mut()
+                .map(|handle| match handle {
+                    TaskBoundaryHandle::SourceSnapshot(snapshot) => {
+                        *snapshot = replacement.clone();
+                        1
+                    }
+                    _ => 0,
+                })
+                .sum(),
+            _ => 0,
+        }
+    }
 }
 
 pub const WASM_TASK_ABI_VERSION: u32 = 1;
@@ -846,6 +867,23 @@ impl TaskSpec {
         }
         Ok(())
     }
+
+    pub fn rebind_source_snapshot(&mut self, replacement: Digest) -> Result<(), String> {
+        self.validate_boundary_authority()?;
+        if self.source_snapshot.is_none() {
+            return Err("task has no SourceSnapshot argument handle to rebind".to_owned());
+        }
+        let rebound = self
+            .args
+            .iter_mut()
+            .map(|argument| argument.rebind_source_snapshot(&replacement))
+            .sum::<usize>();
+        if rebound == 0 {
+            return Err("task source dependency has no canonical argument handle".to_owned());
+        }
+        self.source_snapshot = Some(replacement);
+        self.validate_boundary_authority()
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -1044,6 +1082,11 @@ mod tests {
             bundle_digest: Some(Digest::sha256("bundle")),
         };
         spec.validate_boundary_authority().unwrap();
+        let replacement_source = Digest::sha256("replacement-source");
+        spec.rebind_source_snapshot(replacement_source.clone())
+            .unwrap();
+        assert_eq!(spec.source_snapshot, Some(replacement_source.clone()));
+        assert_eq!(spec.args[0].source_snapshots(), vec![replacement_source]);
         spec.required_artifacts = vec![ArtifactId::from("forged.bin")];
         assert!(spec
             .validate_boundary_authority()

crates/clusterflux-core/src/ids.rs

@@ -1,18 +1,102 @@
+#[cfg(not(target_arch = "wasm32"))]
+use serde::{de::Error as _, Deserializer};
 use serde::{Deserialize, Serialize};
 
+pub const MAX_EXTERNAL_ID_BYTES: usize = 255;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OpaqueTokenError {
+    reason: String,
+}
+
+impl std::fmt::Display for OpaqueTokenError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.reason)
+    }
+}
+
+impl std::error::Error for OpaqueTokenError {}
+
+pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> {
+    let reason = if value.trim().is_empty() {
+        Some("value must not be empty or whitespace-only".to_owned())
+    } else if value.len() > max_bytes {
+        Some(format!("value exceeds the {max_bytes}-byte limit"))
+    } else if value.chars().any(char::is_control) {
+        Some("control characters are forbidden".to_owned())
+    } else {
+        None
+    };
+    reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason }))
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IdParseError {
+    id_type: &'static str,
+    reason: &'static str,
+}
+
+impl IdParseError {
+    fn new(id_type: &'static str, reason: &'static str) -> Self {
+        Self { id_type, reason }
+    }
+}
+
+impl std::fmt::Display for IdParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} is invalid: {}", self.id_type, self.reason)
+    }
+}
+
+impl std::error::Error for IdParseError {}
+
+fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> {
+    if value.is_empty() || value.trim().is_empty() {
+        return Err(IdParseError::new(
+            id_type,
+            "value must not be empty or whitespace-only",
+        ));
+    }
+    if value.len() > MAX_EXTERNAL_ID_BYTES {
+        return Err(IdParseError::new(
+            id_type,
+            "value exceeds the 255-byte limit",
+        ));
+    }
+    if value.chars().any(char::is_control) {
+        return Err(IdParseError::new(
+            id_type,
+            "control characters are forbidden",
+        ));
+    }
+    if !value.bytes().all(|byte| {
+        byte.is_ascii_alphanumeric()
+            || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+')
+    }) {
+        return Err(IdParseError::new(
+            id_type,
+            "only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed",
+        ));
+    }
+    Ok(())
+}
+
 macro_rules! id_type {
     ($name:ident) => {
-        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+        #[cfg_attr(target_arch = "wasm32", derive(Deserialize))]
+        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
         pub struct $name(String);
 
         impl $name {
-            pub fn new(value: impl Into<String>) -> Self {
+            pub fn try_new(value: impl Into<String>) -> Result<Self, IdParseError> {
                 let value = value.into();
-                assert!(
+                validate_id(&value, stringify!($name))?;
-                    !value.trim().is_empty(),
+                Ok(Self(value))
-                    concat!(stringify!($name), " cannot be empty")
+            }
-                );
+
-                Self(value)
+            /// Constructs an identifier from a trusted, internally generated value.
+            pub fn new(value: impl Into<String>) -> Self {
+                Self::try_new(value).unwrap_or_else(|error| panic!("{error}"))
             }
 
             pub fn as_str(&self) -> &str {
@@ -26,6 +110,17 @@ macro_rules! id_type {
             }
         }
 
+        #[cfg(not(target_arch = "wasm32"))]
+        impl<'de> Deserialize<'de> for $name {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                let value = String::deserialize(deserializer)?;
+                Self::try_new(value).map_err(D::Error::custom)
+            }
+        }
+
         impl std::fmt::Display for $name {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                 f.write_str(&self.0)
@@ -38,8 +133,79 @@ id_type!(AgentId);
 id_type!(ArtifactId);
 id_type!(NodeId);
 id_type!(ProcessId);
+id_type!(LaunchAttemptId);
 id_type!(ProjectId);
 id_type!(TaskDefinitionId);
 id_type!(TaskInstanceId);
 id_type!(TenantId);
 id_type!(UserId);
+
+pub type DebugSessionId = ProcessId;
+pub type RequestId = LaunchAttemptId;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_hostile_values<T>(parse: impl Fn(String) -> Result<T, IdParseError>) {
+        for value in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0id".to_owned(),
+            "bad id".to_owned(),
+            "x".repeat(MAX_EXTERNAL_ID_BYTES + 1),
+        ] {
+            assert!(parse(value).is_err());
+        }
+    }
+
+    #[test]
+    fn every_external_identifier_type_rejects_hostile_values() {
+        assert_hostile_values(AgentId::try_new);
+        assert_hostile_values(ArtifactId::try_new);
+        assert_hostile_values(DebugSessionId::try_new);
+        assert_hostile_values(NodeId::try_new);
+        assert_hostile_values(ProcessId::try_new);
+        assert_hostile_values(LaunchAttemptId::try_new);
+        assert_hostile_values(ProjectId::try_new);
+        assert_hostile_values(RequestId::try_new);
+        assert_hostile_values(TaskDefinitionId::try_new);
+        assert_hostile_values(TaskInstanceId::try_new);
+        assert_hostile_values(TenantId::try_new);
+        assert_hostile_values(UserId::try_new);
+    }
+
+    #[test]
+    fn every_identifier_type_validates_during_deserialization() {
+        macro_rules! assert_deserialization {
+            ($id:ty) => {
+                assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok());
+                let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err();
+                assert!(
+                    error.to_string().contains("is invalid"),
+                    "{} produced an unexpected error: {error}",
+                    stringify!($id)
+                );
+            };
+        }
+
+        assert_deserialization!(AgentId);
+        assert_deserialization!(ArtifactId);
+        assert_deserialization!(NodeId);
+        assert_deserialization!(ProcessId);
+        assert_deserialization!(LaunchAttemptId);
+        assert_deserialization!(ProjectId);
+        assert_deserialization!(TaskDefinitionId);
+        assert_deserialization!(TaskInstanceId);
+        assert_deserialization!(TenantId);
+        assert_deserialization!(UserId);
+    }
+
+    #[test]
+    fn opaque_tokens_are_bounded_without_using_identifier_syntax() {
+        validate_opaque_token("opaque secret/+==", 64).unwrap();
+        assert!(validate_opaque_token("", 64).is_err());
+        assert!(validate_opaque_token("bad\0token", 64).is_err());
+        assert!(validate_opaque_token(&"x".repeat(65), 64).is_err());
+    }
+}

crates/clusterflux-core/src/lib.rs

@@ -1,3 +1,4 @@
+mod api_error;
 pub mod artifact;
 pub mod auth;
 pub mod bundle;
@@ -18,10 +19,11 @@ pub mod transport;
 pub mod vfs;
 pub mod wire;
 
+pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode};
 pub use artifact::{
     ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
-    ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
+    ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
-    DownloadStreamRequest, RetentionPolicy, StorageLocation,
+    DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
 };
 pub use auth::{
     admin_request_proof, admin_request_proof_from_token_digest,
@@ -66,8 +68,9 @@ pub use execution::{
     WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
 };
 pub use ids::{
-    AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
+    validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId,
-    UserId,
+    NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId, MAX_EXTERNAL_ID_BYTES,
 };
 pub use limits::{
     LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,

crates/clusterflux-core/src/vfs.rs

@@ -1,18 +1,48 @@
 use std::collections::BTreeMap;
 
-use serde::{Deserialize, Serialize};
+use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
 use thiserror::Error;
 
 use crate::{Digest, NodeId, TaskInstanceId};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub const MAX_VFS_PATH_BYTES: usize = 4096;
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 pub struct VfsPath(String);
 
 impl VfsPath {
     pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
         let path = path.into();
         if !path.starts_with("/vfs/") {
-            return Err(VfsError::InvalidPath(path));
+            return Err(VfsError::invalid(path, "path must start with /vfs/"));
+        }
+        if path.len() > MAX_VFS_PATH_BYTES {
+            return Err(VfsError::invalid(
+                path,
+                "path exceeds the 4096-byte protocol limit",
+            ));
+        }
+        if path.chars().any(char::is_control) {
+            return Err(VfsError::invalid(path, "control characters are forbidden"));
+        }
+        if path.contains('\\') {
+            return Err(VfsError::invalid(path, "backslashes are forbidden"));
+        }
+        let relative = &path["/vfs/".len()..];
+        if relative.is_empty() {
+            return Err(VfsError::invalid(
+                path,
+                "path after /vfs/ must not be empty",
+            ));
+        }
+        if relative
+            .split('/')
+            .any(|component| component.is_empty() || matches!(component, "." | ".."))
+        {
+            return Err(VfsError::invalid(
+                path,
+                "empty, '.', and '..' path components are forbidden",
+            ));
         }
         Ok(Self(path))
     }
@@ -22,6 +52,16 @@ impl VfsPath {
     }
 }
 
+impl<'de> Deserialize<'de> for VfsPath {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let path = String::deserialize(deserializer)?;
+        Self::new(path).map_err(D::Error::custom)
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct VfsObject {
     pub path: VfsPath,
@@ -63,12 +103,18 @@ pub enum ReuseDecision {
 
 #[derive(Clone, Debug, Error, PartialEq, Eq)]
 pub enum VfsError {
-    #[error("VFS path must start with /vfs/: {0}")]
+    #[error("invalid VFS path {path:?}: {reason}")]
-    InvalidPath(String),
+    InvalidPath { path: String, reason: &'static str },
     #[error("path is not visible in the published VFS manifest: {0}")]
     NotVisible(String),
 }
 
+impl VfsError {
+    fn invalid(path: String, reason: &'static str) -> Self {
+        Self::InvalidPath { path, reason }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct VfsOverlay {
     task: TaskInstanceId,
@@ -238,4 +284,39 @@ mod tests {
 
         assert_eq!(overlay.pending_len(), 0);
     }
+
+    #[test]
+    fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
+        for invalid in [
+            "vfs/artifacts/app".to_owned(),
+            "/vfs/".to_owned(),
+            "/vfs/artifacts//app".to_owned(),
+            "/vfs/artifacts/app/".to_owned(),
+            "/vfs/artifacts/./app".to_owned(),
+            "/vfs/artifacts/../app".to_owned(),
+            "/vfs/artifacts\\app".to_owned(),
+            "/vfs/artifacts/bad\0app".to_owned(),
+            format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
+        ] {
+            assert!(
+                VfsPath::new(&invalid).is_err(),
+                "hostile VFS path unexpectedly passed: {invalid:?}"
+            );
+            assert!(
+                serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
+                "hostile VFS path unexpectedly deserialized"
+            );
+        }
+    }
+
+    #[test]
+    fn vfs_path_accepts_the_4096_byte_boundary() {
+        let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
+        assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
+        assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
+
+        let overlong = format!("{path}x");
+        assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
+        assert!(VfsPath::new(overlong).is_err());
+    }
 }

crates/clusterflux-dap/src/breakpoints.rs

@@ -231,10 +231,27 @@ pub(crate) fn stopped_thread_for_breakpoint(state: &AdapterState) -> i64 {
 
 pub(crate) fn position_confirmed_breakpoint_stop(state: &mut AdapterState, stopped_thread: i64) {
     let line = state
+        .stopped_probe_symbol
+        .as_deref()
+        .and_then(|symbol| symbol.strip_prefix("clusterflux.probe."))
+        .and_then(|function| {
+            state
+                .debug_probes
+                .iter()
+                .find(|probe| probe.function == function)
+                .and_then(|probe| {
+                    state.breakpoints.iter().copied().find(|line| {
+                        *line >= i64::from(probe.line_start) && *line <= i64::from(probe.line_end)
+                    })
+                })
+        })
+        .or_else(|| {
+            state
                 .breakpoints
                 .iter()
                 .copied()
                 .find(|line| thread_for_source_line(state, *line) == stopped_thread)
+        })
         .or_else(|| state.breakpoints.first().copied());
     if let (Some(line), Some(thread)) = (line, state.threads.get_mut(&stopped_thread)) {
         thread.line = line;
@@ -258,11 +275,29 @@ pub(crate) fn next_breakpoint_after(state: &AdapterState, thread_id: i64) -> Opt
 
 fn thread_for_source_line(state: &AdapterState, line: i64) -> i64 {
     if let Some(probe) = debug_probe_for_line(state, line) {
+        if state.runtime_backend == crate::virtual_model::RuntimeBackend::Simulated {
+            let function = probe.function.to_ascii_lowercase();
+            if function.contains("linux") {
+                return LINUX_THREAD;
+            }
+            if function.contains("windows") {
+                return WINDOWS_THREAD;
+            }
+            if function.contains("package") {
+                return PACKAGE_THREAD;
+            }
+            return MAIN_THREAD;
+        }
         return state
+            .stopped_task
+            .as_ref()
+            .and_then(|task| {
+                state
                     .threads
                     .values()
-            .find(|thread| thread.task_definition == probe.task)
+                    .find(|thread| &thread.task == task)
                     .map(|thread| thread.id)
+            })
             .unwrap_or(MAIN_THREAD);
     }
 

crates/clusterflux-dap/src/main.rs

@@ -22,7 +22,7 @@ use breakpoints::{
 #[cfg(test)]
 use dap_protocol::{initialize_capabilities, read_message};
 #[cfg(test)]
-use runtime_client::{client_user_request, parse_task_restart_response};
+use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code};
 #[cfg(test)]
 use variables::variables_response;
 #[cfg(test)]
@@ -36,6 +36,26 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
 use virtual_model::{process_id, RuntimeBackend};
 
 fn main() -> Result<()> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux Debug Adapter Protocol server.\n\n\
+                 Usage: clusterflux-debug-dap\n\n\
+                 The adapter communicates over standard input and output.\n\n\
+                 Options:\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        [] => {}
+        [argument, ..] => anyhow::bail!("unknown argument: {argument}"),
+    }
     adapter::run_adapter()
 }
 

crates/clusterflux-dap/src/runtime_client/debug_protocol.rs

@@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestart
         restarted_task_instance: response
             .get("restarted_task_instance")
             .and_then(Value::as_str)
-            .map(TaskInstanceId::new),
+            .map(TaskInstanceId::try_new)
+            .transpose()
+            .map_err(|error| anyhow!("malformed restarted task instance: {error}"))?,
         restarted_attempt_id: response
             .get("restarted_attempt_id")
             .and_then(Value::as_str)

crates/clusterflux-dap/src/runtime_client/transport.rs

@@ -21,10 +21,19 @@ pub(crate) fn client_user_request(state: &AdapterState, mut request: Value) -> V
     })
 }
 
-pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
+pub(super) struct CoordinatorSession {
-    let mut session = ControlSession::connect(addr)?;
+    session: ControlSession,
-    let wire_request = coordinator_wire_request("dap-1", request);
+}
-    let response = session.request(&wire_request)?;
+
+impl CoordinatorSession {
+    pub(super) fn connect(addr: &str) -> Result<Self> {
+        Ok(Self {
+            session: ControlSession::connect(addr)?,
+        })
+    }
+
+    pub(super) fn request(&mut self, request: Value) -> Result<Value> {
+        let response = self.request_allow_error(request)?;
         if response.get("type").and_then(Value::as_str) == Some("error") {
             return Err(anyhow!(
                 "{}",
@@ -36,3 +45,17 @@ pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
         }
         Ok(response)
     }
+
+    pub(super) fn request_allow_error(&mut self, request: Value) -> Result<Value> {
+        let wire_request = coordinator_wire_request("dap-1", request);
+        Ok(self.session.request(&wire_request)?)
+    }
+}
+
+pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
+    CoordinatorSession::connect(addr)?.request(request)
+}
+
+pub(super) fn coordinator_request_allow_error(addr: &str, request: Value) -> Result<Value> {
+    CoordinatorSession::connect(addr)?.request_allow_error(request)
+}

crates/clusterflux-dap/src/source.rs

@@ -7,12 +7,12 @@ use serde_json::{json, Value};
 use crate::virtual_model::AdapterState;
 
 pub(crate) fn infer_source_path(project: &str) -> String {
-    if Path::new(project).join("src/build.rs").is_file() {
+    if Path::new(project).join("src/lib.rs").is_file() {
-        "src/build.rs".to_owned()
+        "src/lib.rs".to_owned()
     } else if Path::new(project).join("src/main.rs").is_file() {
         "src/main.rs".to_owned()
-    } else if Path::new(project).join("src/lib.rs").is_file() {
+    } else if Path::new(project).join("src/build.rs").is_file() {
-        "src/lib.rs".to_owned()
+        "src/build.rs".to_owned()
     } else {
         "src/main.rs".to_owned()
     }

crates/clusterflux-dap/src/tests.rs

@@ -45,6 +45,13 @@ fn initialize_capabilities_use_standard_dap_flags() {
         .all(|key| key.starts_with("supports") && !key.contains("clusterflux")));
 }
 
+#[test]
+fn stale_breakpoint_success_and_failure_completions_are_not_current() {
+    assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9));
+}
+
 #[test]
 fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() {
     assert_eq!(
@@ -189,10 +196,19 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
     state.apply_runtime_record(RuntimeLaunchRecord {
         coordinator: "127.0.0.1:7999".to_owned(),
         node: "node".to_owned(),
-        node_report: json!({ "terminal_event": {
+        node_report: json!({
+            "task_snapshots": { "snapshots": [{
+                "task": "compile-linux-1",
+                "task_definition": "compile-linux",
+                "attempt_id": "attempt-1",
+                "current": true,
+                "state": "failed_awaiting_action"
+            }] },
+            "terminal_event": {
                 "task": "compile-linux-1",
                 "task_definition": "compile-linux"
-        } }),
+            }
+        }),
         task_events: json!({ "events": [] }),
         placed_task_launched: true,
         status_code: Some(1),
@@ -209,20 +225,25 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
         stopped_probe_symbol: None,
         all_participants_frozen: false,
     });
-    let linux = state
+    let exact_runtime_thread = state
         .threads
-        .get(&LINUX_THREAD)
+        .values()
-        .expect("linux task should exist");
+        .find(|thread| thread.task == TaskInstanceId::from("compile-linux-1"))
+        .expect("the exact terminal task instance should have its own thread");
 
     assert!(state.last_task_failed);
     assert!(state
         .command_status
         .contains("failed through local services"));
-    assert!(matches!(linux.state, DebugRuntimeState::Failed(_)));
+    assert!(matches!(
-    assert!(linux
+        exact_runtime_thread.state,
+        DebugRuntimeState::Failed(_)
+    ));
+    assert!(exact_runtime_thread
         .recent_output
         .iter()
         .any(|line| line.contains("task failed")));
+    assert_eq!(state.threads.len(), 1);
 }
 
 #[test]
@@ -275,7 +296,7 @@ fn breakpoint_resolution_uses_bundle_debug_probe_metadata() {
     let mut state = AdapterState::default();
     state.debug_probes = vec![BundleDebugProbe {
         id: "probe-linux".to_owned(),
-        source_path: "src/build.rs".to_owned(),
+        source_path: "src/lib.rs".to_owned(),
         line_start: 20,
         line_end: 30,
         function: "compile_linux".to_owned(),
@@ -303,7 +324,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
 
     let mut state = AdapterState::default();
     state.project = project.path().to_string_lossy().into_owned();
-    state.source_path = "src/build.rs".to_owned();
+    state.source_path = "src/lib.rs".to_owned();
     state.breakpoints = vec![1];
 
     let requested_source = source_dir.join("main.rs");
@@ -311,9 +332,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
 
     assert_eq!(resolved.len(), 1);
     assert!(!resolved[0].verified);
-    assert!(resolved[0]
+    assert!(resolved[0].message.contains("configured for `src/lib.rs`"));
-        .message
-        .contains("configured for `src/build.rs`"));
     assert_eq!(state.breakpoints, vec![1]);
 }
 
@@ -358,7 +377,11 @@ fn windows_thread_runtime_variables_share_virtual_process() {
         variable["name"] == "virtual_process_id" && variable["value"] == state.process.to_string()
     }));
     assert!(variables.iter().any(|variable| {
-        variable["name"] == "virtual_thread" && variable["value"] == "compile windows"
+        variable["name"] == "virtual_thread" && variable["value"] == "compile-windows"
+    }));
+    assert!(variables.iter().any(|variable| {
+        variable["name"] == "task_attempt_id"
+            && variable["value"] == format!("demo-attempt-{WINDOWS_THREAD}")
     }));
 }
 
@@ -369,10 +392,19 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
     state.apply_runtime_record(RuntimeLaunchRecord {
         coordinator: "127.0.0.1:7999".to_owned(),
         node: "node".to_owned(),
-        node_report: json!({ "terminal_event": {
+        node_report: json!({
+            "task_snapshots": { "snapshots": [{
+                "task": "compile-linux-1",
+                "task_definition": "compile-linux",
+                "attempt_id": "attempt-1",
+                "current": true,
+                "state": "running"
+            }] },
+            "terminal_event": {
                 "task": "compile-linux-1",
                 "task_definition": "compile-linux"
-        } }),
+            }
+        }),
         task_events: json!({ "events": [] }),
         placed_task_launched: true,
         status_code: Some(0),
@@ -389,21 +421,28 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
         stopped_probe_symbol: None,
         all_participants_frozen: false,
     });
+    let exact_thread_id = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("compile-linux-1"))
+        .map(|thread| thread.id)
+        .expect("the exact terminal task instance should have its own thread");
     {
-        let linux = state.threads.get_mut(&LINUX_THREAD).unwrap();
+        let runtime_thread = state.threads.get_mut(&exact_thread_id).unwrap();
-        linux.runtime_task_args = vec![("source".to_owned(), "source://checkout".to_owned())];
+        runtime_thread.runtime_task_args =
-        linux.runtime_handles = vec![(
+            vec![("source".to_owned(), "source://checkout".to_owned())];
+        runtime_thread.runtime_handles = vec![(
             "artifact".to_owned(),
             "artifact://runtime/output".to_owned(),
         )];
-        linux.runtime_command_status = Some("frozen native command pid 42".to_owned());
+        runtime_thread.runtime_command_status = Some("frozen native command pid 42".to_owned());
     }
-    let linux = state
+    let runtime_thread = state
         .threads
-        .get(&LINUX_THREAD)
+        .get(&exact_thread_id)
-        .expect("linux task should exist");
+        .expect("exact runtime thread should exist");
 
-    let args = variables_response(&state, linux.args_ref);
+    let args = variables_response(&state, runtime_thread.args_ref);
     let args = args["variables"]
         .as_array()
         .expect("variables response should be an array");
@@ -418,7 +457,7 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
             && variable["type"] == "runtime-handle"
     }));
 
-    let runtime = variables_response(&state, linux.runtime_ref);
+    let runtime = variables_response(&state, runtime_thread.runtime_ref);
     let runtime = runtime["variables"].as_array().unwrap();
     let command_ref = runtime
         .iter()
@@ -634,14 +673,73 @@ fn terminal_events_do_not_resurrect_threads_absent_from_current_snapshots() {
 }
 
 #[test]
-fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
+fn terminal_record_without_snapshots_clears_active_threads() {
     let mut state = AdapterState::default();
-    let project = Path::new(env!("CARGO_MANIFEST_DIR"))
+    state.runtime_backend = RuntimeBackend::LiveServices;
-        .join("../../examples/launch-build-demo")
+    assert!(!state.threads.is_empty());
-        .canonicalize()
+
+    state.apply_runtime_record(RuntimeLaunchRecord {
+        coordinator: "127.0.0.1:9443".to_owned(),
+        node: "coordinator-main".to_owned(),
+        node_report: json!({ "terminal_event": { "terminal_state": "completed" } }),
+        task_events: json!({ "events": [] }),
+        placed_task_launched: true,
+        status_code: Some(0),
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        artifact_path: None,
+        event_count: 0,
+        debug_epoch: None,
+        stopped_task: None,
+        stopped_probe_symbol: None,
+        all_participants_frozen: false,
+    });
+
+    assert!(state.threads.is_empty());
+}
+
+#[test]
+fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-source-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        r#"async fn run_build_workflow() {
+    let source = prepare_source_snapshot();
+    let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
+        .name("compile linux")
+        .env(linux_env())
+        .start()
+        .await
         .unwrap();
+    let linux_thread = linux.virtual_thread_id();
+    let linux_artifact = linux.join().await.unwrap();
+    let package = clusterflux::spawn::async_task_with_arg(
+        vec![linux_artifact.clone()],
+        package_release,
+    )
+    .name("package artifacts")
+    .env(linux_env())
+    .start()
+    .await
+    .unwrap();
+    let package_artifact = package.join().await.unwrap();
+}
+"#,
+    )
+    .unwrap();
+
+    let mut state = AdapterState::default();
     state.project = project.to_string_lossy().into_owned();
-    state.source_path = "src/build.rs".to_owned();
+    state.source_path = "src/lib.rs".to_owned();
     let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
     let inspection_line = source
         .lines()
@@ -675,6 +773,8 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
         variable["name"] == "unavailable-local-diagnostic"
             && variable["type"] == "unavailable-local"
     }));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -794,12 +894,21 @@ fn package_release(inputs: Vec<Artifact>) -> Artifact {
 
 #[test]
 fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-wasm-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n    input + 1\n}\n",
+    )
+    .unwrap();
+
     let mut state = AdapterState::default();
-    state.project = Path::new(env!("CARGO_MANIFEST_DIR"))
+    state.project = project.to_string_lossy().into_owned();
-        .join("../../examples/launch-build-demo")
+    state.source_path = "src/lib.rs".to_owned();
-        .to_string_lossy()
-        .into_owned();
-    state.source_path = "src/build.rs".to_owned();
     let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
     state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
         .lines()
@@ -826,6 +935,8 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
     assert!(!locals
         .iter()
         .any(|variable| variable["name"] == "wasm-local-diagnostic"));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -852,3 +963,188 @@ fn detects_current_failed_attempt_awaiting_operator_action() {
         Some(("task-current", "attempt-current"))
     );
 }
+
+#[test]
+fn whole_process_terminal_status_covers_every_current_logical_task_attempt() {
+    let snapshots = |items: serde_json::Value| json!({ "snapshots": items });
+
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful main and final child must succeed"
+    );
+    for (state, status_code) in [("failed", 23), ("cancelled", 1)] {
+        assert_eq!(
+            whole_process_status_code(
+                Some(0),
+                &snapshots(json!([
+                    {
+                        "task": "child",
+                        "current": true,
+                        "state": state,
+                        "status_code": status_code
+                    }
+                ]))
+            ),
+            Some(status_code),
+            "a terminal child must determine the whole-process result"
+        );
+    }
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "accepted-failure",
+                    "current": true,
+                    "state": "failed",
+                    "command_state": "failure_accepted",
+                    "status_code": 17
+                }
+            ]))
+        ),
+        Some(17),
+        "accepting an AwaitOperator failure releases the process but does not turn failure into success"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-1",
+                    "current": false,
+                    "state": "failed",
+                    "status_code": 7
+                },
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-2",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful replacement attempt is authoritative over stale failed attempts"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(9),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(9),
+        "a failed coordinator main cannot be masked by successful children"
+    );
+}
+
+#[test]
+fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
+    let mut state = AdapterState::default();
+    state.runtime_backend = RuntimeBackend::LiveServices;
+    let record = |snapshots: serde_json::Value| RuntimeLaunchRecord {
+        coordinator: "127.0.0.1:9443".to_owned(),
+        node: "node-a".to_owned(),
+        node_report: json!({
+            "task_snapshots": { "snapshots": snapshots },
+            "process_status": null
+        }),
+        task_events: json!({ "events": [] }),
+        placed_task_launched: true,
+        status_code: None,
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        artifact_path: None,
+        event_count: 0,
+        debug_epoch: None,
+        stopped_task: None,
+        stopped_probe_symbol: None,
+        all_participants_frozen: false,
+    };
+    state.apply_attach_record(record(json!([
+        {
+            "task": "lane-stable",
+            "task_definition": "build_lane",
+            "attempt_id": "stable-1",
+            "current": true,
+            "state": "running"
+        },
+        {
+            "task": "lane-recovering",
+            "task_definition": "build_lane",
+            "attempt_id": "recovering-1",
+            "current": true,
+            "state": "failed_awaiting_action"
+        }
+    ])));
+    let stable_id = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
+        .unwrap()
+        .id;
+    let recovering_id = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-recovering"))
+        .unwrap()
+        .id;
+    assert_ne!(stable_id, recovering_id);
+
+    state.apply_runtime_record(record(json!([
+        {
+            "task": "lane-new",
+            "task_definition": "build_lane",
+            "attempt_id": "new-1",
+            "current": true,
+            "state": "running"
+        },
+        {
+            "task": "lane-stable",
+            "task_definition": "build_lane",
+            "attempt_id": "stable-2",
+            "current": true,
+            "state": "running"
+        }
+    ])));
+
+    let stable = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
+        .unwrap();
+    let new_lane = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-new"))
+        .unwrap();
+    assert_eq!(stable.id, stable_id);
+    assert_eq!(stable.attempt_id, "stable-2");
+    assert!(state
+        .threads
+        .values()
+        .all(|thread| thread.task != TaskInstanceId::from("lane-recovering")));
+    assert!(new_lane.id > recovering_id);
+}

crates/clusterflux-dap/src/variables.rs

@@ -149,7 +149,7 @@ pub(crate) fn variables_response(state: &AdapterState, reference: i64) -> Value
                 },
                 {
                     "name": "virtual_thread",
-                    "value": thread.name,
+                    "value": thread.task.to_string(),
                     "variablesReference": 0
                 },
                 {

crates/clusterflux-dap/src/virtual_model.rs

@@ -66,11 +66,16 @@ pub(crate) struct AdapterState {
     pub(crate) last_task_failed: bool,
     pub(crate) runtime_artifact_path: Option<String>,
     pub(crate) runtime_event_count: usize,
+    pub(crate) runtime_snapshot_fingerprint: String,
+    pub(crate) next_thread_id: i64,
     pub(crate) coordinator_debug_epoch: Option<u64>,
     pub(crate) debug_all_threads_stopped: bool,
     pub(crate) stopped_task: Option<TaskInstanceId>,
+    pub(crate) stopped_probe_symbol: Option<String>,
     pub(crate) debug_probes: Vec<BundleDebugProbe>,
     pub(crate) breakpoints: Vec<i64>,
+    pub(crate) breakpoints_installed: bool,
+    pub(crate) breakpoint_revision: u64,
     pub(crate) threads: BTreeMap<i64, VirtualThread>,
 }
 
@@ -107,9 +112,9 @@ impl Default for AdapterState {
             threads: launch_threads(&entry),
             entry,
             project,
-            source_path: "src/build.rs".to_owned(),
+            source_path: "src/lib.rs".to_owned(),
             runtime_backend: RuntimeBackend::Simulated,
-            coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(),
+            coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(),
             tenant: TenantId::from("tenant"),
             project_id: ProjectId::from("project"),
             actor_user: UserId::from("dap"),
@@ -120,11 +125,16 @@ impl Default for AdapterState {
             last_task_failed: false,
             runtime_artifact_path: None,
             runtime_event_count: 0,
+            runtime_snapshot_fingerprint: String::new(),
+            next_thread_id: LINUX_THREAD,
             coordinator_debug_epoch: None,
             debug_all_threads_stopped: true,
             stopped_task: None,
+            stopped_probe_symbol: None,
             debug_probes: Vec::new(),
             breakpoints: Vec::new(),
+            breakpoints_installed: false,
+            breakpoint_revision: 0,
         }
     }
 }
@@ -190,18 +200,24 @@ impl AdapterState {
             }
             self.coordinator_endpoint =
                 crate::view_state::normalize_coordinator_endpoint(&session.coordinator);
-            self.tenant = TenantId::new(session.tenant);
+            self.tenant = TenantId::try_new(session.tenant)
-            self.project_id = ProjectId::new(session.project);
+                .map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?;
-            self.actor_user = UserId::new(session.user);
+            self.project_id = ProjectId::try_new(session.project)
+                .map_err(|error| anyhow!("invalid project in CLI session: {error}"))?;
+            self.actor_user = UserId::try_new(session.user)
+                .map_err(|error| anyhow!("invalid user in CLI session: {error}"))?;
             self.client_session_secret = Some(session.session_secret);
         } else {
             self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(
                 coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"),
             );
             if let Some(scope) = project_scope {
-                self.tenant = TenantId::new(scope.tenant);
+                self.tenant = TenantId::try_new(scope.tenant)
-                self.project_id = ProjectId::new(scope.project);
+                    .map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?;
-                self.actor_user = UserId::new(scope.user);
+                self.project_id = ProjectId::try_new(scope.project)
+                    .map_err(|error| anyhow!("invalid project in project scope: {error}"))?;
+                self.actor_user = UserId::try_new(scope.user)
+                    .map_err(|error| anyhow!("invalid user in project scope: {error}"))?;
             }
             self.client_session_secret = None;
         }
@@ -211,13 +227,18 @@ impl AdapterState {
         self.last_task_failed = false;
         self.runtime_artifact_path = None;
         self.runtime_event_count = 0;
+        self.runtime_snapshot_fingerprint.clear();
+        self.next_thread_id = LINUX_THREAD;
         self.coordinator_debug_epoch = None;
         self.debug_all_threads_stopped = true;
         self.stopped_task = None;
+        self.stopped_probe_symbol = None;
         self.debug_probes =
             crate::breakpoints::load_bundle_debug_probes(&self.project, &self.source_path);
         self.epoch = 0;
         self.breakpoints.clear();
+        self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated;
+        self.breakpoint_revision = 0;
         self.threads = if self.runtime_backend == RuntimeBackend::Simulated {
             launch_threads(&self.entry)
         } else {
@@ -230,13 +251,18 @@ impl AdapterState {
         self.coordinator_endpoint = record.coordinator.clone();
         if self.runtime_backend != RuntimeBackend::Simulated {
             if let Some(snapshots) = record.node_report.get("task_snapshots") {
-                self.threads = coordinator_threads_from_snapshots(
+                self.reconcile_runtime_threads(snapshots, record.node_report.get("process_status"));
-                    &self.entry,
+            } else if record.node_report.get("terminal_event").is_some() {
-                    snapshots,
+                self.threads.clear();
-                    record.node_report.get("process_status"),
-                );
             }
         }
+        if let Some(fingerprint) = record
+            .node_report
+            .get("snapshot_fingerprint")
+            .and_then(Value::as_str)
+        {
+            self.runtime_snapshot_fingerprint = fingerprint.to_owned();
+        }
         if !record.placed_task_launched {
             self.command_status = format!(
                 "virtual process started through {}; no runtime task event observed yet",
@@ -287,7 +313,11 @@ impl AdapterState {
             self.debug_all_threads_stopped = record.all_participants_frozen;
             self.epoch = record.debug_epoch.unwrap_or(self.epoch);
             self.coordinator_debug_epoch = record.debug_epoch;
-            self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new);
+            self.stopped_task = record
+                .stopped_task
+                .as_deref()
+                .and_then(|task| TaskInstanceId::try_new(task).ok());
+            self.stopped_probe_symbol = record.stopped_probe_symbol.clone();
             if let Some(acknowledgements) = record
                 .node_report
                 .pointer("/debug_epoch/acknowledgements")
@@ -303,6 +333,12 @@ impl AdapterState {
                     else {
                         continue;
                     };
+                    let Ok(task_id) = TaskInstanceId::try_new(task) else {
+                        continue;
+                    };
+                    let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+                        continue;
+                    };
                     let coordinator_main = acknowledgement.get("node").and_then(Value::as_str)
                         == Some("coordinator-main");
                     let thread_id = if coordinator_main {
@@ -312,14 +348,19 @@ impl AdapterState {
                             .values()
                             .find(|thread| thread.task.as_str() == task)
                             .map(|thread| thread.id)
-                            .unwrap_or_else(|| self.insert_runtime_thread(task, task_definition))
+                            .unwrap_or_else(|| {
+                                self.insert_runtime_thread(
+                                    task_id.clone(),
+                                    task_definition_id.clone(),
+                                )
+                            })
                     };
                     let thread = self
                         .threads
                         .get_mut(&thread_id)
                         .expect("runtime thread was found or inserted");
-                    thread.task = TaskInstanceId::new(task);
+                    thread.task = task_id;
-                    thread.task_definition = TaskDefinitionId::new(task_definition);
+                    thread.task_definition = task_definition_id;
                     thread.runtime_stack_frames = acknowledgement
                         .get("stack_frames")
                         .and_then(Value::as_array)
@@ -362,23 +403,12 @@ impl AdapterState {
                 .and_then(Value::as_str)
                 .unwrap_or(self.entry.as_str())
                 .to_owned();
-            let terminal_task_definition = terminal_event
-                .get("task_definition")
-                .and_then(Value::as_str)
-                .unwrap_or(self.entry.as_str())
-                .to_owned();
             let terminal_thread_id = self
                 .threads
                 .values()
-                .find(|thread| {
+                .find(|thread| thread.task.as_str() == terminal_task.as_str())
-                    thread.task.as_str() == terminal_task.as_str()
+                .map(|thread| thread.id);
-                        || thread.task_definition.as_str() == terminal_task_definition.as_str()
+            if let Some(thread) = terminal_thread_id.and_then(|id| self.threads.get_mut(&id)) {
-                })
-                .map(|thread| thread.id)
-                .unwrap_or_else(|| {
-                    self.insert_runtime_thread(&terminal_task, &terminal_task_definition)
-                });
-            if let Some(thread) = self.threads.get_mut(&terminal_thread_id) {
                 thread.stdout_bytes = record.stdout_bytes;
                 thread.stderr_bytes = record.stderr_bytes;
                 thread.stdout_tail = record.stdout_tail.clone();
@@ -412,24 +442,22 @@ impl AdapterState {
         let _ = crate::view_state::write_view_state(self, &record);
     }
 
-    fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 {
+    fn insert_runtime_thread(
-        let id = self
+        &mut self,
-            .threads
+        task: TaskInstanceId,
-            .keys()
+        task_definition: TaskDefinitionId,
-            .next_back()
+    ) -> i64 {
-            .copied()
+        let id = self.allocate_runtime_thread_id();
-            .unwrap_or(MAIN_THREAD)
-            + 1;
         let line = self
             .debug_probes
             .iter()
             .find(|probe| {
-                probe.task.as_str() == task_definition || probe.function == task_definition
+                probe.task == task_definition || probe.function == task_definition.as_str()
             })
             .map(|probe| i64::from(probe.line_start))
             .unwrap_or(1);
-        let definition_name = task_definition.replace(['_', '-'], " ");
+        let definition_name = task_definition.as_str().replace(['_', '-'], " ");
-        let name = if task == task_definition {
+        let name = if task.as_str() == task_definition.as_str() {
             definition_name
         } else {
             format!("{definition_name} ({task})")
@@ -447,9 +475,9 @@ impl AdapterState {
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::new(task),
+                task,
                 attempt_id: "unknown-attempt".to_owned(),
-                task_definition: TaskDefinitionId::new(task_definition),
+                task_definition,
                 name,
                 line,
                 state: DebugRuntimeState::Running,
@@ -475,6 +503,51 @@ impl AdapterState {
         id
     }
 
+    fn allocate_runtime_thread_id(&mut self) -> i64 {
+        while self.threads.contains_key(&self.next_thread_id) || self.next_thread_id == MAIN_THREAD
+        {
+            self.next_thread_id += 1;
+        }
+        let id = self.next_thread_id;
+        self.next_thread_id += 1;
+        id
+    }
+
+    fn reconcile_runtime_threads(
+        &mut self,
+        task_snapshots: &Value,
+        process_status: Option<&Value>,
+    ) {
+        let desired =
+            coordinator_threads_from_snapshots(&self.entry, task_snapshots, process_status);
+        let previous = std::mem::take(&mut self.threads);
+        let mut reconciled = BTreeMap::new();
+        for (_, mut thread) in desired {
+            let coordinator_main = process_status
+                .and_then(|status| status.get("main_task_instance"))
+                .and_then(Value::as_str)
+                .is_some_and(|task| task == thread.task.as_str());
+            let existing = if coordinator_main {
+                previous.get(&MAIN_THREAD)
+            } else {
+                previous
+                    .values()
+                    .find(|current| current.task == thread.task)
+            };
+            if let Some(existing) = existing {
+                preserve_thread_identity(&mut thread, existing);
+            } else if !coordinator_main {
+                thread.id = self.allocate_runtime_thread_id();
+                assign_thread_references(&mut thread);
+            } else {
+                thread.id = MAIN_THREAD;
+                assign_thread_references(&mut thread);
+            }
+            reconciled.insert(thread.id, thread);
+        }
+        self.threads = reconciled;
+    }
+
     pub(crate) fn apply_attach_record(&mut self, record: RuntimeLaunchRecord) {
         self.coordinator_endpoint = record.coordinator.clone();
         self.command_status = format!(
@@ -485,8 +558,7 @@ impl AdapterState {
         self.last_task_failed = false;
         self.runtime_artifact_path = None;
         self.runtime_event_count = record.event_count;
-        self.threads = coordinator_threads_from_snapshots(
+        self.reconcile_runtime_threads(
-            &self.entry,
             record
                 .node_report
                 .get("task_snapshots")
@@ -507,6 +579,35 @@ impl AdapterState {
     }
 }
 
+fn preserve_thread_identity(thread: &mut VirtualThread, existing: &VirtualThread) {
+    thread.id = existing.id;
+    thread.frame_id = existing.frame_id;
+    thread.locals_ref = existing.locals_ref;
+    thread.wasm_locals_ref = existing.wasm_locals_ref;
+    thread.args_ref = existing.args_ref;
+    thread.runtime_ref = existing.runtime_ref;
+    thread.output_ref = existing.output_ref;
+    thread.target_ref = existing.target_ref;
+    thread.vfs_ref = existing.vfs_ref;
+    thread.command_ref = existing.command_ref;
+    if thread.recent_output.is_empty() {
+        thread.recent_output = existing.recent_output.clone();
+    }
+}
+
+fn assign_thread_references(thread: &mut VirtualThread) {
+    let id = thread.id;
+    thread.frame_id = 1000 + id;
+    thread.locals_ref = 5000 + id;
+    thread.wasm_locals_ref = 9000 + id;
+    thread.args_ref = 2000 + id;
+    thread.runtime_ref = 3000 + id;
+    thread.output_ref = 4000 + id;
+    thread.target_ref = 6000 + id;
+    thread.vfs_ref = 7000 + id;
+    thread.command_ref = 8000 + id;
+}
+
 fn value_string_pairs(value: Option<&Value>) -> Vec<(String, String)> {
     value
         .and_then(Value::as_array)
@@ -642,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId {
     ProcessId::new(format!("vp-{suffix}"))
 }
 
-fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread {
+fn coordinator_main_thread(
+    entry: &str,
+    task: TaskInstanceId,
+    task_definition: TaskDefinitionId,
+) -> VirtualThread {
+    let name = format!("{entry} coordinator main ({task})");
     VirtualThread {
         id: MAIN_THREAD,
         frame_id: 1_001,
@@ -654,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi
         target_ref: 4_501,
         vfs_ref: 5_001,
         command_ref: 5_501,
-        task: TaskInstanceId::from(task),
+        task,
         attempt_id: "main:unknown".to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
+        task_definition,
-        name: format!("{entry} coordinator main ({task})"),
+        name,
         line: 0,
         state: DebugRuntimeState::Running,
         freeze_supported: true,
@@ -693,10 +799,14 @@ fn coordinator_threads_from_events(
         .expect("launch thread model should include main");
     if let Some(status) = process_status {
         if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) {
-            main.task = TaskInstanceId::from(task);
+            if let Ok(task) = TaskInstanceId::try_new(task) {
+                main.task = task;
+            }
         }
         if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) {
-            main.task_definition = TaskDefinitionId::from(definition);
+            if let Ok(definition) = TaskDefinitionId::try_new(definition) {
+                main.task_definition = definition;
+            }
         }
         main.name = format!("{entry} coordinator main ({})", main.task);
         main.state = if status
@@ -758,6 +868,12 @@ fn coordinator_threads_from_snapshots(
                 .get("main_task_definition")
                 .and_then(Value::as_str)
                 .unwrap_or(entry);
+            let (Ok(task), Ok(task_definition)) = (
+                TaskInstanceId::try_new(task),
+                TaskDefinitionId::try_new(task_definition),
+            ) else {
+                return threads;
+            };
             let mut main = coordinator_main_thread(entry, task, task_definition);
             main.attempt_id = format!(
                 "main:{}",
@@ -812,6 +928,12 @@ fn coordinator_threads_from_snapshots(
         let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else {
             continue;
         };
+        let Ok(task_id) = TaskInstanceId::try_new(task) else {
+            continue;
+        };
+        let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+            continue;
+        };
         let attempt_id = snapshot
             .get("attempt_id")
             .and_then(Value::as_str)
@@ -862,9 +984,9 @@ fn coordinator_threads_from_snapshots(
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::from(task),
+                task: task_id,
                 attempt_id,
-                task_definition: TaskDefinitionId::from(task_definition),
+                task_definition: task_definition_id,
                 name: format!("{display_name} ({task}, attempt {short_attempt})"),
                 line: snapshot
                     .get("source_line")
@@ -911,6 +1033,8 @@ fn coordinator_threads_from_snapshots(
 fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option<VirtualThread> {
     let task = event.get("task").and_then(Value::as_str)?;
     let task_definition = event.get("task_definition").and_then(Value::as_str)?;
+    let task_id = TaskInstanceId::try_new(task).ok()?;
+    let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?;
     let definition_name = task_definition.replace(['_', '-'], " ");
     let name = if task == task_definition {
         definition_name
@@ -956,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti
         target_ref: 6000 + id,
         vfs_ref: 7000 + id,
         command_ref: 8000 + id,
-        task: TaskInstanceId::from(task),
+        task: task_id,
         attempt_id: event
             .get("attempt_id")
             .and_then(Value::as_str)
             .unwrap_or("unknown-attempt")
             .to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
+        task_definition: task_definition_id,
         name,
         line: 0,
         state,

crates/clusterflux-node/src/assignment_runner.rs

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::io::Read;
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant};
@@ -46,6 +46,40 @@ use validation::{
     resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
 };
 
+#[derive(Debug)]
+struct AssignmentExecutionError {
+    message: String,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
+}
+
+impl std::fmt::Display for AssignmentExecutionError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for AssignmentExecutionError {}
+
+fn execution_error_with_log_bytes(
+    message: impl Into<String>,
+    stdout_source_bytes: &AtomicU64,
+    stderr_source_bytes: &AtomicU64,
+) -> Box<dyn std::error::Error> {
+    Box::new(AssignmentExecutionError {
+        message: message.into(),
+        stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed),
+        stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed),
+    })
+}
+
+pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) {
+    error
+        .downcast_ref::<AssignmentExecutionError>()
+        .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes))
+        .unwrap_or((0, 0))
+}
+
 pub(crate) fn run_verified_wasmtime_assignment(
     args: &Args,
     task: &RuntimeTask,
@@ -95,6 +129,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
             return Err("Wasm entrypoint assignment omitted its descriptor export".into());
         }
     };
+    let command_stdout_source_bytes = Arc::new(AtomicU64::new(0));
+    let command_stderr_source_bytes = Arc::new(AtomicU64::new(0));
     let (stdout, boundary_result) = match abi {
         WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
             let invocation = WasmTaskInvocation::new(
@@ -102,7 +138,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
                 task_spec.task_instance.clone(),
                 task_spec.args.clone(),
             );
-            let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
+            let result = WasmtimeTaskRuntime::new()?
+                .run_task_export_verified_with_task_host(
                     &module,
                     expected_bundle_digest,
                     export,
@@ -112,8 +149,17 @@ pub(crate) fn run_verified_wasmtime_assignment(
                         task,
                         node_private_key,
                         &module,
+                        Arc::clone(&command_stdout_source_bytes),
+                        Arc::clone(&command_stderr_source_bytes),
                     )?),
-            )?;
+                )
+                .map_err(|error| {
+                    execution_error_with_log_bytes(
+                        error.to_string(),
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    )
+                })?;
             if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
                 eprintln!(
                     "clusterflux debug control: Wasm assignment returned for task {}",
@@ -122,17 +168,35 @@ pub(crate) fn run_verified_wasmtime_assignment(
             }
             match result.outcome {
                 WasmTaskOutcome::Completed => {
-                    let boundary = result.result.ok_or("completed Wasm task omitted result")?;
+                    let boundary = result.result.ok_or_else(|| {
+                        execution_error_with_log_bytes(
+                            "completed Wasm task omitted result",
+                            &command_stdout_source_bytes,
+                            &command_stderr_source_bytes,
+                        )
+                    })?;
                     (
-                        format!("{}\n", serde_json::to_string(&boundary)?),
+                        format!(
+                            "{}\n",
+                            serde_json::to_string(&boundary).map_err(|error| {
+                                execution_error_with_log_bytes(
+                                    error.to_string(),
+                                    &command_stdout_source_bytes,
+                                    &command_stderr_source_bytes,
+                                )
+                            })?
+                        ),
                         Some(boundary),
                     )
                 }
                 WasmTaskOutcome::Failed => {
-                    return Err(result
+                    return Err(execution_error_with_log_bytes(
+                        result
                             .error
-                        .unwrap_or_else(|| "Wasm task failed without an error".to_owned())
+                            .unwrap_or_else(|| "Wasm task failed without an error".to_owned()),
-                        .into())
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    ))
                 }
             }
         }
@@ -140,12 +204,18 @@ pub(crate) fn run_verified_wasmtime_assignment(
     let task_id = TaskInstanceId::new(task.task.clone());
     let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
     let manifest = artifacts.flush();
+    let stdout_source_bytes = command_stdout_source_bytes
+        .load(Ordering::Relaxed)
+        .saturating_add(stdout.len() as u64);
+    let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed);
     Ok((
         CommandOutput {
             virtual_thread: task_id,
             status_code: Some(0),
             stdout,
             stderr: String::new(),
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: false,
             stderr_truncated: false,
             log_backpressured: false,
@@ -176,6 +246,8 @@ struct CoordinatorWasmTaskHost {
     next_handle_id: u64,
     handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
     command_status: Arc<Mutex<Option<String>>>,
+    command_stdout_source_bytes: Arc<AtomicU64>,
+    command_stderr_source_bytes: Arc<AtomicU64>,
     cancellation_requested: Arc<AtomicBool>,
     abort_requested: Arc<AtomicBool>,
     debug_control: Arc<WasmDebugControl>,
@@ -188,6 +260,8 @@ impl CoordinatorWasmTaskHost {
         parent: &RuntimeTask,
         node_private_key: &str,
         module: &[u8],
+        command_stdout_source_bytes: Arc<AtomicU64>,
+        command_stderr_source_bytes: Arc<AtomicU64>,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let task_spec = parent
             .task_spec
@@ -268,6 +342,8 @@ impl CoordinatorWasmTaskHost {
             next_handle_id: 1,
             handles,
             command_status,
+            command_stdout_source_bytes,
+            command_stderr_source_bytes,
             cancellation_requested,
             abort_requested,
             debug_control,
@@ -293,7 +369,16 @@ impl CoordinatorWasmTaskHost {
             runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
         let execution =
             run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
-        let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
+        let (
+            terminal_state,
+            status_code,
+            stdout,
+            stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
+            result,
+            retained,
+        ) = match execution {
             Ok((output, _manifest, result)) => {
                 let retained = retained_result_artifact(
                     self.args.project_root.as_deref(),
@@ -306,20 +391,40 @@ impl CoordinatorWasmTaskHost {
                         output.status_code,
                         output.stdout,
                         output.stderr,
+                        output.stdout_source_bytes,
+                        output.stderr_source_bytes,
                         result,
                         retained,
                     ),
-                    Err(error) => ("failed", Some(1), String::new(), error, None, None),
-                }
-            }
                     Err(error) => (
                         "failed",
                         Some(1),
                         String::new(),
-                error.to_string(),
+                        error.clone(),
+                        output.stdout_source_bytes,
+                        output
+                            .stderr_source_bytes
+                            .saturating_add(error.len() as u64),
                         None,
                         None,
                     ),
+                }
+            }
+            Err(error) => {
+                let (stdout_source_bytes, stderr_source_bytes) =
+                    assignment_error_log_bytes(error.as_ref());
+                let error = error.to_string();
+                (
+                    "failed",
+                    Some(1),
+                    String::new(),
+                    error.clone(),
+                    stdout_source_bytes,
+                    stderr_source_bytes.saturating_add(error.len() as u64),
+                    None,
+                    None,
+                )
+            }
         };
         let artifact_path = retained
             .as_ref()
@@ -339,8 +444,8 @@ impl CoordinatorWasmTaskHost {
                     "task": runtime_task.task,
                     "terminal_state": terminal_state,
                     "status_code": status_code,
-                    "stdout_bytes": stdout.len(),
+                    "stdout_bytes": stdout_source_bytes,
-                    "stderr_bytes": stderr.len(),
+                    "stderr_bytes": stderr_source_bytes,
                     "stdout_tail": stdout,
                     "stderr_tail": stderr,
                     "stdout_truncated": false,
@@ -676,6 +781,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost {
         let mut runner = CoordinatorControlledProcessRunner::new(
             self,
             Duration::from_millis(request.timeout_ms),
+            configured_secrets.clone(),
         );
         let output = LinuxRootlessPodmanBackend
             .execute_local_checkout_task(

crates/clusterflux-node/src/assignment_runner/process_runner.rs

@@ -1,4 +1,83 @@
 use super::*;
+use std::sync::mpsc::{self, Receiver, SyncSender};
+
+struct LiveLogChunk {
+    stream: &'static str,
+    offset: u64,
+    source_bytes: u64,
+    bytes: Vec<u8>,
+    truncated: bool,
+}
+
+fn append_bounded_tail(tail: &mut Vec<u8>, bytes: &[u8], maximum: usize) {
+    if maximum == 0 {
+        tail.clear();
+        return;
+    }
+    if bytes.len() >= maximum {
+        tail.clear();
+        tail.extend_from_slice(&bytes[bytes.len() - maximum..]);
+        return;
+    }
+    let overflow = tail
+        .len()
+        .saturating_add(bytes.len())
+        .saturating_sub(maximum);
+    if overflow > 0 {
+        tail.drain(..overflow);
+    }
+    tail.extend_from_slice(bytes);
+}
+
+fn redact_safe_live_log_prefix(
+    pending: &[u8],
+    configured_secrets: &[String],
+    final_chunk: bool,
+) -> Option<(usize, String)> {
+    if pending.is_empty() {
+        return None;
+    }
+    let secret_bytes = configured_secrets
+        .iter()
+        .filter(|secret| secret.len() >= 4)
+        .map(String::as_bytes)
+        .collect::<Vec<_>>();
+    let maximum_secret_bytes = secret_bytes
+        .iter()
+        .map(|secret| secret.len())
+        .max()
+        .unwrap_or(0);
+    if !final_chunk && pending.len() <= maximum_secret_bytes {
+        return None;
+    }
+    let mut consumed = if final_chunk {
+        pending.len()
+    } else {
+        pending.len() - maximum_secret_bytes
+    };
+    loop {
+        let previous = consumed;
+        for secret in &secret_bytes {
+            for start in 0..=pending.len().saturating_sub(secret.len()) {
+                let end = start + secret.len();
+                if start < consumed && end > consumed && &pending[start..end] == *secret {
+                    consumed = end;
+                }
+            }
+        }
+        if consumed == previous {
+            break;
+        }
+    }
+    if consumed == 0 {
+        return None;
+    }
+    let text = redact_configured_values(
+        String::from_utf8_lossy(&pending[..consumed]).into_owned(),
+        configured_secrets,
+    );
+    Some((consumed, text))
+}
 
 pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) args: Args,
@@ -7,13 +86,20 @@ pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) node_private_key: String,
     pub(super) debug_control: Arc<WasmDebugControl>,
     pub(super) command_status: Arc<Mutex<Option<String>>>,
+    pub(super) stdout_source_bytes: Arc<AtomicU64>,
+    pub(super) stderr_source_bytes: Arc<AtomicU64>,
     pub(super) timeout: Duration,
+    pub(super) configured_secrets: Vec<String>,
 }
 
 impl CoordinatorControlledProcessRunner {
     const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
 
-    pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
+    pub(super) fn new(
+        host: &CoordinatorWasmTaskHost,
+        timeout: Duration,
+        configured_secrets: Vec<String>,
+    ) -> Self {
         Self {
             args: host.args.clone(),
             process: host.process.clone(),
@@ -21,7 +107,10 @@ impl CoordinatorControlledProcessRunner {
             node_private_key: host.node_private_key.clone(),
             debug_control: Arc::clone(&host.debug_control),
             command_status: Arc::clone(&host.command_status),
+            stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes),
+            stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes),
             timeout,
+            configured_secrets,
         }
     }
 
@@ -201,10 +290,18 @@ impl CoordinatorControlledProcessRunner {
     fn drain_bounded(
         mut reader: impl Read + Send + 'static,
         maximum: usize,
+        stream: &'static str,
+        sender: SyncSender<LiveLogChunk>,
+        configured_secrets: Vec<String>,
+        source_bytes_total: Arc<AtomicU64>,
     ) -> thread::JoinHandle<Result<Vec<u8>, String>> {
         thread::spawn(move || {
             let mut captured = Vec::new();
             let mut buffer = [0_u8; 16 * 1024];
+            let stream_base = source_bytes_total.load(Ordering::Relaxed);
+            let mut source_bytes_read = 0_u64;
+            let mut pending_offset = stream_base;
+            let mut pending = Vec::new();
             loop {
                 let count = reader
                     .read(&mut buffer)
@@ -212,12 +309,128 @@ impl CoordinatorControlledProcessRunner {
                 if count == 0 {
                     break;
                 }
-                let remaining = maximum.saturating_sub(captured.len());
+                let _ = source_bytes_total.fetch_update(
-                captured.extend_from_slice(&buffer[..count.min(remaining)]);
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                    |current| Some(current.saturating_add(count as u64)),
+                );
+                source_bytes_read = source_bytes_read.saturating_add(count as u64);
+                append_bounded_tail(&mut captured, &buffer[..count], maximum);
+                pending.extend_from_slice(&buffer[..count]);
+                if let Some((consumed, text)) =
+                    redact_safe_live_log_prefix(&pending, &configured_secrets, false)
+                {
+                    let _ = sender.try_send(LiveLogChunk {
+                        stream,
+                        offset: pending_offset,
+                        source_bytes: consumed as u64,
+                        bytes: text.into_bytes(),
+                        truncated: false,
+                    });
+                    pending.drain(..consumed);
+                    pending_offset = pending_offset.saturating_add(consumed as u64);
+                }
+            }
+            if let Some((consumed, text)) =
+                redact_safe_live_log_prefix(&pending, &configured_secrets, true)
+            {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: pending_offset,
+                    source_bytes: consumed as u64,
+                    bytes: text.into_bytes(),
+                    truncated: false,
+                });
+            }
+            if source_bytes_read > maximum as u64 {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: stream_base.saturating_add(source_bytes_read),
+                    source_bytes: 0,
+                    bytes: b"[log output truncated at node capture limit]".to_vec(),
+                    truncated: true,
+                });
             }
             Ok(captured)
         })
     }
+
+    fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
+        let args = self.args.clone();
+        let process = self.process.clone();
+        let task = self.task.clone();
+        let node_private_key = self.node_private_key.clone();
+        let configured_secrets = self.configured_secrets.clone();
+        let command_status = Arc::clone(&self.command_status);
+        thread::spawn(move || {
+            let mut log_session = None;
+            let mut delivery_available = true;
+            while let Ok(chunk) = receiver.recv() {
+                if !delivery_available {
+                    continue;
+                }
+                let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
+                text = redact_configured_values(text, &configured_secrets);
+                let mut delivered = false;
+                for _ in 0..2 {
+                    if log_session.is_none() {
+                        log_session = CoordinatorSession::connect_with_timeouts(
+                            &args.coordinator,
+                            Duration::from_millis(500),
+                            Duration::from_millis(500),
+                        )
+                        .ok();
+                    }
+                    let Some(session) = log_session.as_mut() else {
+                        continue;
+                    };
+                    let request = signed_node_request_json(
+                        &args,
+                        &node_private_key,
+                        "report_task_log_chunk",
+                        serde_json::json!({
+                            "type": "report_task_log_chunk",
+                            "tenant": &args.tenant,
+                            "project": &args.project,
+                            "process": &process,
+                            "node": &args.node,
+                            "task": &task,
+                            "stream": chunk.stream,
+                            "offset": chunk.offset,
+                            "source_bytes": chunk.source_bytes,
+                            "text": &text,
+                            "truncated": chunk.truncated,
+                        }),
+                    );
+                    let result = match request {
+                        Ok(request) => session
+                            .request(request)
+                            .map(|_| ())
+                            .map_err(|error| error.to_string()),
+                        Err(error) => Err(error.to_string()),
+                    };
+                    match result {
+                        Ok(()) => {
+                            delivered = true;
+                            break;
+                        }
+                        Err(_) => {
+                            log_session = None;
+                        }
+                    }
+                }
+                if !delivered {
+                    delivery_available = false;
+                    if let Ok(mut current) = command_status.lock() {
+                        *current = Some(
+                            "live log delivery was interrupted; final bounded output remains available"
+                                .to_owned(),
+                        );
+                    }
+                }
+            }
+        })
+    }
 }
 
 impl ProcessRunner for CoordinatorControlledProcessRunner {
@@ -245,12 +458,17 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             command.program,
             command.args.join(" ")
         ));
+        let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
         let stdout = Self::drain_bounded(
             child
                 .stdout
                 .take()
                 .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stdout",
+            live_log_sender.clone(),
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stdout_source_bytes),
         );
         let stderr = Self::drain_bounded(
             child
@@ -258,11 +476,19 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 .take()
                 .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stderr",
+            live_log_sender,
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stderr_source_bytes),
         );
+        let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
         let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
             Ok(session) => session,
             Err(error) => {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
+                let _ = stdout.join();
+                let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "establish execution control channel: {error}"
                 )));
@@ -277,6 +503,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Ok(None) => {}
                 Err(error) => {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
+                    let _ = stdout.join();
+                    let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Command(error.to_string()));
                 }
             }
@@ -289,6 +518,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
                 let _ = stdout.join();
                 let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "native command exceeded wall-clock timeout of {} ms",
                     self.timeout.as_millis()
@@ -303,6 +533,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Cancelled(
                         "coordinator requested cancellation or abort".to_owned(),
                     ));
@@ -312,6 +543,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(error);
                 }
             }
@@ -360,6 +592,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             .join()
             .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
             .map_err(BackendError::Command)?;
+        live_log_reporter
+            .join()
+            .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
         self.set_command_status(format!(
             "native command exited with status {:?}",
             status.code()
@@ -371,3 +606,52 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner};
+    use std::io::Cursor;
+    use std::sync::atomic::{AtomicU64, Ordering};
+    use std::sync::{mpsc, Arc};
+
+    #[test]
+    fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
+        let secrets = vec!["correct-horse".to_owned()];
+        let mut pending = b"prefix correct-".to_vec();
+        let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
+        pending.drain(..consumed);
+        pending.extend_from_slice(b"horse suffix");
+        let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
+
+        let combined = format!("{first}{second}");
+        assert_eq!(combined, "prefix [REDACTED] suffix");
+        assert!(!combined.contains("correct-"));
+        assert!(!combined.contains("horse"));
+    }
+
+    #[test]
+    fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() {
+        let source_bytes = Arc::new(AtomicU64::new(5));
+        let (sender, receiver) = mpsc::sync_channel(8);
+        let reader = CoordinatorControlledProcessRunner::drain_bounded(
+            Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()),
+            8,
+            "stdout",
+            sender,
+            Vec::new(),
+            Arc::clone(&source_bytes),
+        );
+        let captured = reader.join().unwrap().unwrap();
+        let chunks = receiver.into_iter().collect::<Vec<_>>();
+
+        assert_eq!(captured, b"opqrstuv");
+        assert_eq!(source_bytes.load(Ordering::Relaxed), 37);
+        assert_eq!(
+            chunks.iter().map(|chunk| chunk.source_bytes).sum::<u64>(),
+            32
+        );
+        assert_eq!(chunks[0].offset, 5);
+        assert_eq!(chunks.last().unwrap().offset, 37);
+        assert!(chunks.iter().any(|chunk| chunk.truncated));
+    }
+}

crates/clusterflux-node/src/assignment_runner/tests.rs

@@ -48,7 +48,10 @@ fn test_controlled_runner(
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout,
+        configured_secrets: Vec::new(),
     }
 }
 
@@ -89,7 +92,10 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout: Duration::from_secs(30),
+        configured_secrets: Vec::new(),
     };
     let started = Instant::now();
     let error = runner

crates/clusterflux-node/src/command_runner.rs

@@ -1,6 +1,6 @@
 use clusterflux_core::{
-    CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
+    CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay,
-    VfsOverlay, VfsPath,
+    VfsPath,
 };
 use serde::{Deserialize, Serialize};
 
@@ -42,6 +42,8 @@ pub struct CommandOutput {
     pub status_code: Option<i32>,
     pub stdout: String,
     pub stderr: String,
+    pub stdout_source_bytes: u64,
+    pub stderr_source_bytes: u64,
     pub stdout_truncated: bool,
     pub stderr_truncated: bool,
     pub log_backpressured: bool,
@@ -83,6 +85,8 @@ impl LocalCommandExecutor {
             &output.stderr,
             max_log_bytes,
         );
+        let stdout_source_bytes = output.stdout.len() as u64;
+        let stderr_source_bytes = output.stderr.len() as u64;
         let staged_artifact = if let Some(path) = command.stage_stdout_as {
             Some(overlay.write(
                 path,
@@ -98,6 +102,8 @@ impl LocalCommandExecutor {
             status_code: output.status.code(),
             stdout: logs.stdout,
             stderr: logs.stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: logs.stdout_truncated,
             stderr_truncated: logs.stderr_truncated,
             log_backpressured: logs.backpressured,
@@ -107,24 +113,20 @@ impl LocalCommandExecutor {
 }
 
 pub(super) fn capture_command_logs(
-    task: &TaskInstanceId,
+    _task: &TaskInstanceId,
     stdout: &[u8],
     stderr: &[u8],
     max_log_bytes: usize,
 ) -> CapturedCommandLogs {
-    let mut logs = LogBuffer::new(max_log_bytes);
+    let stdout_truncated = stdout.len() > max_log_bytes;
-    logs.push(task.clone(), stdout);
+    let stderr_truncated = stderr.len() > max_log_bytes;
-    logs.push(task.clone(), stderr);
+    let stdout_start = stdout.len().saturating_sub(max_log_bytes);
-    let records = logs.records();
+    let stderr_start = stderr.len().saturating_sub(max_log_bytes);
-    let stdout_record = &records[0];
-    let stderr_record = &records[1];
-    debug_assert_eq!(&stdout_record.task, task);
-    debug_assert_eq!(&stderr_record.task, task);
     CapturedCommandLogs {
-        stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
+        stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(),
-        stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
+        stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(),
-        stdout_truncated: stdout_record.truncated,
+        stdout_truncated,
-        stderr_truncated: stderr_record.truncated,
+        stderr_truncated,
-        backpressured: logs.backpressured(),
+        backpressured: stdout_truncated || stderr_truncated,
     }
 }

crates/clusterflux-node/src/coordinator_session.rs

@@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity;
 use clusterflux_control::ControlSession;
 use clusterflux_core::coordinator_wire_request;
 use serde_json::Value;
+use std::time::Duration;
 
 pub(crate) struct CoordinatorSession {
     inner: ControlSession,
@@ -15,6 +16,16 @@ impl CoordinatorSession {
         })
     }
 
+    pub(crate) fn connect_with_timeouts(
+        addr: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        Ok(Self {
+            inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?,
+        })
+    }
+
     pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
         let request_id = format!("node-{}", self.inner.requests() + 1);
         let wire_request = coordinator_wire_request(request_id, value);

crates/clusterflux-node/src/daemon.rs

@@ -7,11 +7,11 @@ use std::time::{Duration, Instant};
 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
 use clusterflux_core::{
     sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
-    TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
+    ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
 };
 use serde_json::{json, Value};
 
-use crate::assignment_runner::run_verified_wasmtime_assignment;
+use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment};
 #[cfg(test)]
 use crate::coordinator_session::control_endpoint_identity;
 use crate::coordinator_session::CoordinatorSession;
@@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
     let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
     });
     let heartbeat_signature = sign_node_request(
@@ -222,7 +224,7 @@ fn worker_loop(
             let expiry = Instant::now()
                 .checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
                 .unwrap_or_else(Instant::now);
-            restart_pins.insert(ArtifactId::new(artifact), expiry);
+            restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry);
         }
         println!("{}", serde_json::to_string(&report)?);
         std::io::stdout().flush()?;
@@ -289,7 +291,7 @@ fn service_pending_artifact_transfer(
         return Ok(false);
     };
     let transfer_id = required_string(transfer, "transfer_id")?;
-    let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
+    let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?;
     let expected_digest: Digest = serde_json::from_value(
         transfer
             .get("expected_digest")
@@ -369,9 +371,11 @@ pub(crate) fn runtime_task_from_assignment(
             .cloned()
             .ok_or("task assignment missing task_spec")?,
     )?;
+    let process = ProcessId::try_new(required_string(value, "process")?)?;
+    let task = TaskInstanceId::try_new(required_string(value, "task")?)?;
     Ok(RuntimeTask {
-        process: required_string(value, "process")?,
+        process: process.to_string(),
-        task: required_string(value, "task")?,
+        task: task.to_string(),
         epoch: value.get("epoch").and_then(Value::as_u64),
         bundle_digest: task_spec.bundle_digest.clone(),
         task_spec: Some(task_spec),
@@ -418,6 +422,8 @@ fn run_runtime_task(
         "reconnect_node",
         json!({
         "type": "reconnect_node",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
         "process": &task.process,
         "epoch": epoch,
@@ -458,6 +464,8 @@ fn run_runtime_task(
             capability_report,
             debug_command,
             node_private_key,
+            0,
+            0,
         );
     }
 
@@ -492,9 +500,13 @@ fn run_runtime_task(
                 debug_command,
                 node_private_key,
                 &error,
+                output.stdout_source_bytes,
+                output.stderr_source_bytes,
             ),
         },
         Err(error) => {
+            let (stdout_source_bytes, stderr_source_bytes) =
+                assignment_error_log_bytes(error.as_ref());
             let error = error.to_string();
             if error.contains("task execution cancelled:") {
                 record_cancelled_task(
@@ -506,6 +518,8 @@ fn run_runtime_task(
                     capability_report,
                     debug_command,
                     node_private_key,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             } else {
                 record_failed_task(
@@ -518,6 +532,8 @@ fn run_runtime_task(
                     debug_command,
                     node_private_key,
                     &error,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             }
         }
@@ -570,6 +586,12 @@ fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
         }
     }
 
+    TenantId::try_new(tenant.clone())?;
+    ProjectId::try_new(project.clone())?;
+    NodeId::try_new(node.clone())?;
+    if let Some(grant) = enrollment_grant.as_ref() {
+        clusterflux_core::validate_opaque_token(grant, 512)?;
+    }
     Ok(Args {
         coordinator: coordinator.ok_or("--coordinator is required")?,
         tenant,
@@ -622,13 +644,12 @@ mod tests {
     #[test]
     fn hosted_url_remains_an_https_control_endpoint() {
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
+            control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(),
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
+            "https://clusterflux.lesstuff.com/api/v1/control"
         );
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
+            control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
-                .unwrap(),
+            "https://clusterflux.lesstuff.com/api/v1/control"
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
         );
         assert_eq!(
             control_endpoint_identity("127.0.0.1:7999").unwrap(),

crates/clusterflux-node/src/lib.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clusterflux_core::{
     Capability, CommandBackendKind, CommandInvocation, CommandPlan, Digest, EnvironmentKind,
@@ -9,6 +9,7 @@ use sha2::{Digest as ShaDigest, Sha256};
 use thiserror::Error;
 
 mod command_runner;
+mod source_snapshot;
 mod windows_dev;
 pub use clusterflux_wasm_runtime::{
     WasmDebugControl, WasmTaskError, WasmTaskHost, WasmtimeDebugProbe, WasmtimeTaskRuntime,
@@ -20,6 +21,10 @@ pub use command_runner::{
 };
 pub use windows_dev::{WindowsCommandDevBackend, WindowsSandboxStubBackend};
 
+pub fn snapshot_project_digest(project_root: &Path) -> Result<Digest, String> {
+    source_snapshot::snapshot_project(project_root).map(|snapshot| snapshot.digest)
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct MaterializedEnvironment {
     pub name: String,
@@ -736,7 +741,7 @@ mod tests {
     }
 
     #[test]
-    fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() {
+    fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() {
         let invocation = CommandInvocation {
             program: "cargo".to_owned(),
             args: vec!["build".to_owned()],
@@ -769,7 +774,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
+        assert_eq!(output.stdout, "cdef");
         assert!(output.stdout_truncated);
         assert!(output.log_backpressured);
         assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
@@ -987,7 +992,7 @@ mod tests {
 
     #[cfg(unix)]
     #[test]
-    fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() {
+    fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() {
         let executor = LocalCommandExecutor {
             node: clusterflux_core::NodeId::from("node"),
             hosted_control_plane: false,
@@ -1016,15 +1021,15 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
+        assert_eq!(output.stdout, "cdef");
-        assert_eq!(output.stderr, "");
+        assert_eq!(output.stderr, "err");
         assert!(output.stdout_truncated);
-        assert!(output.stderr_truncated);
+        assert!(!output.stderr_truncated);
         assert!(output.log_backpressured);
     }
 
     #[test]
-    fn public_node_crate_does_not_require_hosted_private_types() {
+    fn public_node_crate_does_not_require_hosted_service_types() {
         let _tenant = TenantId::from("tenant");
         let _project = ProjectId::from("project");
         let _backend = LinuxRootlessPodmanBackend;

crates/clusterflux-node/src/main.rs

@@ -8,5 +8,34 @@ mod task_artifacts;
 mod task_reports;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-node {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux node worker.\n\n\
+                 Usage: clusterflux-node --coordinator <URL> [OPTIONS]\n\n\
+                 Options:\n  \
+                   --coordinator <URL>\n  \
+                   --tenant <TENANT>                   [default: tenant]\n  \
+                   --project-id <PROJECT>              [default: project]\n  \
+                   --node <NODE>                       [default: node]\n  \
+                   --project-root <PATH>\n  \
+                   --enrollment-grant <GRANT>\n  \
+                   --public-key <KEY>\n  \
+                   --worker\n  \
+                   --emit-ready\n  \
+                   --control-poll-ms <MILLISECONDS>\n  \
+                   --assignment-poll-ms <MILLISECONDS> [default: 500]\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
     daemon::run()
 }

crates/clusterflux-node/src/task_reports.rs

@@ -47,8 +47,8 @@ pub(crate) fn record_completed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": output.stdout.len(),
+            "stdout_bytes": output.stdout_source_bytes,
-            "stderr_bytes": output.stderr.len(),
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -85,8 +85,8 @@ pub(crate) fn record_completed_task(
             "node": &args.node,
             "task": &task.task,
             "status_code": output.status_code,
-            "stdout_bytes": output.stdout.len(),
+            "stdout_bytes": output.stdout_source_bytes,
-            "stderr_bytes": output.stderr.len(),
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -123,8 +123,11 @@ pub(crate) fn record_failed_task(
     debug_command: Value,
     node_private_key: &str,
     error: &str,
+    stdout_source_bytes: u64,
+    command_stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let error = bounded_runtime_error(error);
+    let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64);
     let log_event = session.request(signed_node_request_json(
         args,
         node_private_key,
@@ -136,8 +139,8 @@ pub(crate) fn record_failed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": error.len(),
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -175,8 +178,8 @@ pub(crate) fn record_failed_task(
             "task": &task.task,
             "terminal_state": "failed",
             "status_code": -1,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": error.len(),
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -189,6 +192,8 @@ pub(crate) fn record_failed_task(
     Ok(failed_node_report(
         task,
         &error,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -210,6 +215,8 @@ pub(crate) fn record_cancelled_task(
     capability_report: Value,
     debug_command: Value,
     node_private_key: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let recorded = session.request(signed_node_request_json(
         args,
@@ -224,12 +231,12 @@ pub(crate) fn record_cancelled_task(
             "task": &task.task,
             "terminal_state": "cancelled",
             "status_code": null,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": 0,
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": "",
-            "stdout_truncated": false,
+            "stdout_truncated": stdout_source_bytes > 0,
-            "stderr_truncated": false,
+            "stderr_truncated": stderr_source_bytes > 0,
             "artifact_path": null,
             "artifact_digest": null,
             "artifact_size_bytes": null,
@@ -238,6 +245,8 @@ pub(crate) fn record_cancelled_task(
     )?)?;
     Ok(cancelled_node_report(
         task,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -281,8 +290,8 @@ pub(crate) fn completed_node_report(
         "virtual_thread": output.virtual_thread,
         "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
         "status_code": output.status_code,
-        "stdout_bytes": output.stdout.len(),
+        "stdout_bytes": output.stdout_source_bytes,
-        "stderr_bytes": output.stderr.len(),
+        "stderr_bytes": output.stderr_source_bytes,
         "stdout_tail": &output.stdout,
         "stderr_tail": &output.stderr,
         "stdout_truncated": output.stdout_truncated,
@@ -305,6 +314,8 @@ pub(crate) fn completed_node_report(
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn cancelled_node_report(
     task: &RuntimeTask,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -318,12 +329,12 @@ pub(crate) fn cancelled_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "cancelled",
         "status_code": null,
-        "stdout_bytes": 0,
+        "stdout_bytes": stdout_source_bytes,
-        "stderr_bytes": 0,
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": "",
-        "stdout_truncated": false,
+        "stdout_truncated": stdout_source_bytes > 0,
-        "stderr_truncated": false,
+        "stderr_truncated": stderr_source_bytes > 0,
         "log_backpressured": false,
         "staged_artifact": null,
         "large_bytes_uploaded": false,
@@ -343,6 +354,8 @@ pub(crate) fn cancelled_node_report(
 pub(crate) fn failed_node_report(
     task: &RuntimeTask,
     error: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -357,8 +370,8 @@ pub(crate) fn failed_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "failed",
         "status_code": -1,
-        "stdout_bytes": 0,
+        "stdout_bytes": stdout_source_bytes,
-        "stderr_bytes": error.len(),
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": error,
         "stdout_truncated": false,
@@ -377,3 +390,40 @@ pub(crate) fn failed_node_report(
         "coordinator_response": coordinator_response,
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use clusterflux_core::TaskInstanceId;
+
+    #[test]
+    fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() {
+        let report = completed_node_report(
+            CommandOutput {
+                virtual_thread: TaskInstanceId::from("task"),
+                status_code: Some(0),
+                stdout: "bounded tail".to_owned(),
+                stderr: String::new(),
+                stdout_source_bytes: 519,
+                stderr_source_bytes: 0,
+                stdout_truncated: true,
+                stderr_truncated: false,
+                log_backpressured: false,
+                staged_artifact: None,
+            },
+            false,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            1,
+        );
+
+        assert_eq!(report["stdout_bytes"], 519);
+        assert_eq!(report["stdout_tail"], "bounded tail");
+    }
+}

crates/clusterflux-sdk/src/lib.rs

@@ -1,3 +1,5 @@
+extern crate self as clusterflux;
+
 use serde::Serialize;
 
 pub use clusterflux_core as core;
@@ -10,6 +12,26 @@ pub use task_args::{
     TaskArgBudget, TaskArgError, TaskArgKind, TaskArgValidation, Vfs,
 };
 
+pub use clusterflux_core::TaskFailurePolicy;
+pub use sdk_runtime::TaskRuntimeError as Error;
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub mod prelude {
+    pub use crate::{
+        command::Command, fs, source, Artifact, Result, SourceSnapshot, TaskFailurePolicy,
+    };
+}
+
+#[macro_export]
+macro_rules! spawn {
+    ($task:ident()) => {
+        $crate::spawn::__product_async_task($task, stringify!($task))
+    };
+    ($task:ident($argument:expr)) => {
+        $crate::spawn::__product_async_task_with_arg($argument, $task, stringify!($task))
+    };
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
 pub struct EntrypointDescriptor {
     pub name: &'static str,
@@ -63,6 +85,8 @@ impl RegisteredProgram {
 pub mod __private;
 
 pub mod spawn {
+    use std::future::{Future, IntoFuture};
+    use std::pin::Pin;
     use std::sync::atomic::{AtomicU64, Ordering};
     use std::sync::{Mutex, OnceLock};
 
@@ -115,6 +139,273 @@ pub mod spawn {
         runtime_threads().lock().unwrap().clone()
     }
 
+    #[doc(hidden)]
+    pub fn __product_async_task<F, Fut, R>(
+        entry: F,
+        task_id: &'static str,
+    ) -> ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        ProductAsyncTaskBuilder {
+            entry: Some(entry),
+            env: None,
+            name: task_id,
+            task_id,
+            failure_policy: TaskFailurePolicy::FailFast,
+            marker: std::marker::PhantomData,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn __product_async_task_with_arg<A, F, Fut, R>(
+        arg: A,
+        entry: F,
+        task_id: &'static str,
+    ) -> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        ProductAsyncTaskWithArgBuilder {
+            arg: Some(arg),
+            entry: Some(entry),
+            env: None,
+            name: task_id,
+            task_id,
+            arg_budget: TaskArgBudget::default(),
+            failure_policy: TaskFailurePolicy::FailFast,
+            marker: std::marker::PhantomData,
+        }
+    }
+
+    pub struct ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+        entry: Option<F>,
+        env: Option<EnvRef>,
+        name: &'static str,
+        task_id: &'static str,
+        failure_policy: TaskFailurePolicy,
+        marker: std::marker::PhantomData<fn() -> (Fut, R)>,
+    }
+
+    impl<F, Fut, R> ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg + DeserializeOwned,
+    {
+        pub fn on(mut self, env: EnvRef) -> Self {
+            self.env = Some(env);
+            self
+        }
+
+        pub fn name(mut self, name: &'static str) -> Self {
+            self.name = name;
+            self
+        }
+
+        pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
+            self.failure_policy = failure_policy;
+            self
+        }
+
+        pub async fn start(self) -> crate::Result<TaskHandle<R>> {
+            let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
+            let runtime_event = register_runtime_thread(id, self.name, self.env);
+            #[cfg(target_arch = "wasm32")]
+            {
+                let remote = start_guest_host_task(
+                    clusterflux_core::TaskDefinitionId::from(self.task_id),
+                    self.env,
+                    Vec::new(),
+                    self.failure_policy,
+                )?;
+                return Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: None,
+                    remote: Some(remote),
+                });
+            }
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                if let Some(config) = ProductRuntimeConfig::from_env()? {
+                    let remote = start_remote_task(
+                        config,
+                        self.task_id.to_owned(),
+                        self.env,
+                        Vec::new(),
+                        self.failure_policy,
+                    )?;
+                    return Ok(TaskHandle {
+                        virtual_thread_id: id,
+                        name: self.name,
+                        env: self.env,
+                        debugger_visible: runtime_event.debugger_visible,
+                        result: None,
+                        remote: Some(remote),
+                    });
+                }
+                let result = self.entry.expect("task entry used once")().await?;
+                Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: Some(result),
+                    remote: None,
+                })
+            }
+        }
+    }
+
+    impl<F, Fut, R> IntoFuture for ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut + 'static,
+        Fut: Future<Output = crate::Result<R>> + 'static,
+        R: TaskArg + DeserializeOwned + 'static,
+    {
+        type Output = crate::Result<TaskHandle<R>>;
+        type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
+
+        fn into_future(self) -> Self::IntoFuture {
+            Box::pin(self.start())
+        }
+    }
+
+    pub struct ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        arg: Option<A>,
+        #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+        entry: Option<F>,
+        env: Option<EnvRef>,
+        name: &'static str,
+        task_id: &'static str,
+        arg_budget: TaskArgBudget,
+        failure_policy: TaskFailurePolicy,
+        marker: std::marker::PhantomData<fn() -> (Fut, R)>,
+    }
+
+    impl<A, F, Fut, R> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg + DeserializeOwned,
+    {
+        pub fn on(mut self, env: EnvRef) -> Self {
+            self.env = Some(env);
+            self
+        }
+
+        pub fn name(mut self, name: &'static str) -> Self {
+            self.name = name;
+            self
+        }
+
+        pub fn arg_budget(mut self, arg_budget: TaskArgBudget) -> Self {
+            self.arg_budget = arg_budget;
+            self
+        }
+
+        pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
+            self.failure_policy = failure_policy;
+            self
+        }
+
+        pub async fn start(self) -> crate::Result<TaskHandle<R>> {
+            let arg = self.arg.expect("task argument used once");
+            validate_task_arg(&arg, self.arg_budget)
+                .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+            let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
+            let runtime_event = register_runtime_thread(id, self.name, self.env);
+            #[cfg(target_arch = "wasm32")]
+            {
+                let boundary = arg
+                    .task_boundary_value()
+                    .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+                let remote = start_guest_host_task(
+                    clusterflux_core::TaskDefinitionId::from(self.task_id),
+                    self.env,
+                    vec![boundary],
+                    self.failure_policy,
+                )?;
+                return Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: None,
+                    remote: Some(remote),
+                });
+            }
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                if let Some(config) = ProductRuntimeConfig::from_env()? {
+                    let boundary = arg
+                        .task_boundary_value()
+                        .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+                    let remote = start_remote_task(
+                        config,
+                        self.task_id.to_owned(),
+                        self.env,
+                        vec![boundary],
+                        self.failure_policy,
+                    )?;
+                    return Ok(TaskHandle {
+                        virtual_thread_id: id,
+                        name: self.name,
+                        env: self.env,
+                        debugger_visible: runtime_event.debugger_visible,
+                        result: None,
+                        remote: Some(remote),
+                    });
+                }
+                let result = self.entry.expect("task entry used once")(arg).await?;
+                Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: Some(result),
+                    remote: None,
+                })
+            }
+        }
+    }
+
+    impl<A, F, Fut, R> IntoFuture for ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg + 'static,
+        F: FnOnce(A) -> Fut + 'static,
+        Fut: Future<Output = crate::Result<R>> + 'static,
+        R: TaskArg + DeserializeOwned + 'static,
+    {
+        type Output = crate::Result<TaskHandle<R>>;
+        type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
+
+        fn into_future(self) -> Self::IntoFuture {
+            Box::pin(self.start())
+        }
+    }
+
     pub fn task<F, R>(entry: F) -> TaskBuilder<F, R>
     where
         F: FnOnce() -> R,
@@ -711,6 +1002,10 @@ pub mod command {
             self
         }
 
+        pub fn cwd(self, directory: impl Into<String>) -> Self {
+            self.current_dir(directory)
+        }
+
         pub fn env(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
             self.environment_variables.insert(name.into(), value.into());
             self
@@ -761,6 +1056,34 @@ pub mod command {
                     .to_owned(),
             ))
         }
+
+        pub async fn run(self) -> Result<Output, TaskRuntimeError> {
+            let program = self.program.clone();
+            let output = self.output().await?;
+            if output.status_code == Some(0) {
+                return Ok(output);
+            }
+            Err(TaskRuntimeError::CommandFailed {
+                program,
+                status_code: output.status_code,
+                stdout: bounded_tail(output.stdout),
+                stderr: bounded_tail(output.stderr),
+                stdout_truncated: output.stdout_truncated,
+                stderr_truncated: output.stderr_truncated,
+            })
+        }
+    }
+
+    fn bounded_tail(value: String) -> String {
+        const LIMIT: usize = 4 * 1024;
+        if value.len() <= LIMIT {
+            return value;
+        }
+        let mut start = value.len() - LIMIT;
+        while !value.is_char_boundary(start) {
+            start += 1;
+        }
+        value[start..].to_owned()
     }
 }
 
@@ -782,6 +1105,34 @@ pub mod source {
             "source snapshots are produced only by a source-capable Clusterflux node".to_owned(),
         ))
     }
+
+    pub struct CurrentProject;
+
+    pub fn current_project() -> CurrentProject {
+        CurrentProject
+    }
+
+    impl CurrentProject {
+        pub async fn snapshot(self) -> Result<SourceSnapshot, TaskRuntimeError> {
+            crate::spawn::__product_async_task(snapshot_current_project, "snapshot_current_project")
+                .await?
+                .join()
+                .await
+        }
+    }
+
+    #[clusterflux::task(capabilities = "source_filesystem")]
+    async fn snapshot_current_project() -> crate::Result<SourceSnapshot> {
+        snapshot().await
+    }
+}
+
+impl SourceSnapshot {
+    pub fn mount(&self) -> Result<&'static str> {
+        crate::task_args::parse_handle_digest(&self.digest)
+            .map_err(|error| Error::Argument(error.to_string()))?;
+        Ok("/workspace")
+    }
 }
 
 pub mod fs {
@@ -814,6 +1165,10 @@ pub mod fs {
         })
     }
 
+    pub fn output(relative: impl Into<String>) -> Result<OutputPath, TaskRuntimeError> {
+        output_path(relative)
+    }
+
     pub async fn flush(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
         #[cfg(target_arch = "wasm32")]
         {
@@ -838,6 +1193,10 @@ pub mod fs {
         }
     }
 
+    pub async fn publish(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
+        flush(path).await
+    }
+
     pub async fn materialize(
         artifact: &Artifact,
         relative: impl Into<String>,

crates/clusterflux-sdk/src/sdk_runtime.rs

@@ -106,6 +106,14 @@ pub enum TaskRuntimeError {
         waited: Duration,
     },
     ResultDecode(String),
+    CommandFailed {
+        program: String,
+        status_code: Option<i32>,
+        stdout: String,
+        stderr: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+    },
 }
 
 impl std::fmt::Display for TaskRuntimeError {
@@ -124,6 +132,21 @@ impl std::fmt::Display for TaskRuntimeError {
                 );
             }
             Self::ResultDecode(message) => ("remote result", message),
+            Self::CommandFailed {
+                program,
+                status_code,
+                stdout,
+                stderr,
+                stdout_truncated,
+                stderr_truncated,
+            } => {
+                return write!(
+                    formatter,
+                    "command `{program}` failed with status {status_code:?}; stdout{}: {stdout:?}; stderr{}: {stderr:?}",
+                    if *stdout_truncated { " (truncated)" } else { "" },
+                    if *stderr_truncated { " (truncated)" } else { "" },
+                );
+            }
         };
         write!(formatter, "{kind} error: {message}")
     }

crates/clusterflux-sdk/tests/product_api.rs

@@ -0,0 +1,37 @@
+#[clusterflux::task]
+async fn plus_one(value: i32) -> clusterflux::Result<i32> {
+    Ok(value + 1)
+}
+
+#[test]
+fn unified_spawn_defaults_to_function_identity_and_unwraps_task_result() {
+    futures_executor::block_on(async {
+        let handle = clusterflux::spawn!(plus_one(41))
+            .on(clusterflux::env!("linux"))
+            .await
+            .unwrap();
+        assert_eq!(handle.name(), "plus_one");
+        assert_eq!(handle.env().unwrap().name, "linux");
+        assert_eq!(handle.join().await.unwrap(), 42);
+    });
+}
+
+#[test]
+fn source_mount_and_command_failure_are_public_typed_contracts() {
+    let source = clusterflux::SourceSnapshot {
+        digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+            .to_owned(),
+    };
+    assert_eq!(source.mount().unwrap(), "/workspace");
+
+    let error = clusterflux::Error::CommandFailed {
+        program: "cc".to_owned(),
+        status_code: Some(2),
+        stdout: "out".to_owned(),
+        stderr: "bad input".to_owned(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+    };
+    assert!(error.to_string().contains("command"));
+    assert!(error.to_string().contains("cc"));
+}

docs/artifacts.md

@@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context.
 The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
 transfers. The CLI verifies the final digest. Hosted policy may impose
 per-project, per-tenant, and per-account size, concurrency, and period limits.
-Operators may disable relay traffic as an emergency safety control; one tenant's
+There is no hosted global circuit breaker, and one tenant's period usage does
-period usage does not consume a shared customer byte quota.
+not consume a shared customer byte quota.

docs/debugging.md

@@ -1,14 +1,22 @@
 # Debugging
 
+The DAP runtime observer keeps one coordinator session open and reconnects it
+with a 100 ms to 5 second exponential backoff. Polling is responsive around
+debug actions (100 ms) and backs off to one observation cycle every 5 seconds
+while idle. A cycle makes at most four coordinator requests, so steady-state
+idle traffic is bounded at 0.8 requests per second per debug session.
+
 The VS Code extension uses the Debug Adapter Protocol and the coordinator's
 authoritative process, task, attempt, and Debug Epoch APIs.
 
 ## Launch
 
 Open your project and start "Clusterflux: Launch Virtual Process". The adapter
-builds the bundle, launches the selected entrypoint, and replaces its thread
+acknowledges launch configuration immediately, then builds and starts the
-view with coordinator task snapshots. Product mode does not infer threads from
+selected entrypoint in the background. You can keep using the debug UI while a
-completion events or demo fixtures.
+long-running process is attached. The adapter replaces its thread view with
+coordinator task snapshots as they arrive; product mode does not infer threads
+from completion events or demo fixtures.
 
 Each task view includes its logical task, attempt ID, definition, state, node,
 environment, arguments, typed handles, command status, VFS checkpoint, probe
@@ -19,6 +27,12 @@ source when known, and restart compatibility. Unknown source remains unknown.
 When a breakpoint is hit, the coordinator asks every active participant to
 freeze. A fully frozen epoch is a consistent all-participant view.
 
+You do not need to configure a breakpoint before launch. Breakpoint changes are
+sent to the running coordinator, and a stopped event is emitted only after an
+executing Wasm probe reports a real hit. Pause, continue, inspection, and
+disconnect requests remain responsive while the adapter observes the runtime in
+the background. Attach does not fabricate an initial breakpoint.
+
 If one or more participants cannot freeze within five seconds, the epoch becomes
 partially frozen when at least one participant did freeze. The adapter warns
 that running and frozen tasks do not form a consistent global snapshot. You may
@@ -31,7 +45,8 @@ runtime did not report it.
 
 ## Continue and restart
 
-Continue resumes only participants that acknowledged the freeze.
+Continue responds immediately, then resumes only participants that acknowledged
+the freeze while runtime observation continues in the background.
 
 Restarting a terminal task rebuilds the bundle and checks its clean entry
 checkpoint. A compatible restart creates a new attempt under the same logical

docs/environments.md

@@ -1,5 +1,9 @@
 # Environments
 
+Clusterflux disables network access while a task executes. Materializing an
+environment for the first time can still fetch declared inputs; subsequent task
+execution uses the materialized environment with networking disabled.
+
 Declare environments in the bundle under:
 
 ~~~text

docs/getting-started.md

@@ -11,6 +11,9 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Install rootless Podman on each Linux node that will execute container-backed
 environments.
 
@@ -52,7 +55,7 @@ same stored identity:
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <hosted-project-id> \
   --node workstation \
@@ -81,6 +84,21 @@ clusterflux bundle inspect --project .
 clusterflux run --project . build
 ~~~
 
+From this checkout, the complete source-to-task-to-artifact path is:
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The workflow snapshots `examples/hello-build`, starts its `compile` task in the
+network-disabled Linux execution environment, and retains the real static executable returned by
+that task.
+
 Choose another entrypoint by replacing `build`. Clusterflux rejects an oversized
 or invalid bundle before it creates the virtual process.
 
@@ -101,12 +119,23 @@ Restart it as a new attempt under the same logical task identity:
 clusterflux task restart <task-id> --process <process-id> --yes
 ~~~
 
+`examples/recovery-build` demonstrates this without a synthetic trap. It starts
+two instances of `build_lane`; one completes while the other runs a command that
+exits with status 23. Edit that command to produce the recovering output, then
+restart the failed task. Its original join resolves from the replacement
+attempt.
+
 ## 6. Debug in VS Code
 
 Open the project in VS Code and start `Clusterflux: Launch Virtual Process`.
 Set a breakpoint on a generated probe location and use Threads, Stack,
 Variables, Continue, Pause, and Restart.
 
+Breakpoints remain unverified until the coordinator installs them. Thread IDs
+are stable for an exact logical task instance across snapshot updates and retry
+attempts. Observer reconnect diagnostics do not manufacture stopped events, and
+Continue succeeds only after the coordinator acknowledges resume.
+
 A fully frozen Debug Epoch gives a consistent all-participant view. If a
 participant cannot freeze within five seconds, the adapter reports a partial
 epoch. You may inspect frozen participants, but values across running and frozen

docs/nodes.md

@@ -23,7 +23,7 @@ key pair.
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <project-id> \
   --node workstation \

docs/task-abi.md

@@ -9,10 +9,13 @@ capability-bearing values use typed handles. A structured boundary value carries
 both its JSON structure and the complete typed handle table required to
 materialize it.
 
-Supported handles include VFS and artifact identities with full tenant, project,
+Portable handles carry the minimum stable value needed to cross the task ABI,
-process, producer, digest, size, and path provenance. The SDK encodes handles as
+such as an artifact ID, digest, and size. Tenant, project, process, producer,
-explicit placeholders and the runtime replaces only placeholders that match the
+path, retaining location, and authorization state are coordinator-side
-declared type and metadata.
+metadata; they are not portable authority fields supplied by task code. The SDK
+encodes handles as explicit placeholders and the runtime replaces only
+placeholders that match the declared type and authoritative coordinator
+metadata.
 
 A malformed, missing, extra, cross-scope, or type-mismatched handle fails closed.
 The runtime does not reconstruct authority from a string path or caller-supplied

examples/hello-build/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "hello-build"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+
+[lib]
+crate-type = ["rlib", "cdylib"]
+
+[dependencies]
+clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" }

examples/hello-build/README.md

@@ -0,0 +1,18 @@
+# Hello build
+
+The primary Clusterflux example snapshots this project, compiles a real static C
+executable in the declared network-disabled Linux execution environment, and publishes the
+executable as a retained artifact.
+
+Run it with:
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The final command prints hello from a real Clusterflux build.

examples/hello-build/src/lib.rs

@@ -0,0 +1,30 @@
+use clusterflux::prelude::*;
+
+#[clusterflux::task(capabilities = "command")]
+pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
+    let executable = fs::output("hello-clusterflux")?;
+    Command::new("cc")
+        .args([
+            "-Os",
+            "-static",
+            "-s",
+            "fixture/hello-clusterflux.c",
+            "-o",
+            executable.as_str(),
+        ])
+        .cwd(source.mount()?)
+        .env("SOURCE_DATE_EPOCH", "0")
+        .network_disabled()
+        .run()
+        .await?;
+    fs::publish(&executable).await
+}
+
+#[clusterflux::main]
+pub async fn build() -> Result<Artifact> {
+    let source = source::current_project().snapshot().await?;
+    let compile = clusterflux::spawn!(compile(source))
+        .on(clusterflux::env!("linux"))
+        .await?;
+    compile.join().await
+}

examples/launch-build-demo/README.md

@@ -1,14 +0,0 @@
-# Launch Build Demo
-
-This build workflow is expressed as Rust source code. It uses `env!("linux")`,
-recognizes `env!("windows")` when a Windows development node is attached,
-spawns debugger-visible virtual tasks, and returns portable artifact and source
-handles instead of moving large bytes through task arguments.
-
-The Linux environment is defined by `envs/linux/Containerfile`. The Windows environment is a user-attached development contract and does not claim secure managed Windows sandboxing.
-
-Inspect the bundle metadata:
-
-```bash
-clusterflux bundle inspect --project examples/launch-build-demo
-```

examples/launch-build-demo/envs/windows/Dockerfile

@@ -1,2 +0,0 @@
-# User-attached Windows development execution contract.
-# This is not a managed untrusted Windows sandbox.

examples/launch-build-demo/src/bin/sdk-product-runtime.rs

@@ -1,33 +0,0 @@
-use std::error::Error;
-
-use futures_executor::block_on;
-
-fn main() -> Result<(), Box<dyn Error>> {
-    let handle = block_on(
-        clusterflux::spawn::task_with_arg(41_i32, |_| -> i32 {
-            panic!("product-mode spawn invoked its local Rust closure")
-        })
-        .task_id("task_add_one")
-        .name("SDK product Wasmtime task")
-        .env(clusterflux::env!("linux"))
-        .start(),
-    )?;
-    let task_spec = handle
-        .task_spec()
-        .cloned()
-        .ok_or("CLUSTERFLUX_SDK_COORDINATOR did not enable product runtime")?;
-    let virtual_thread_id = handle.virtual_thread_id();
-    let result = block_on(handle.join())?;
-
-    println!(
-        "{}",
-        serde_json::to_string(&serde_json::json!({
-            "kind": "clusterflux-sdk-product-runtime",
-            "virtual_thread_id": virtual_thread_id,
-            "task_spec": task_spec,
-            "remote_result": result,
-            "local_function_invoked_by_spawn": false,
-        }))?
-    );
-    Ok(())
-}