.gitignore

@@ -1,8 +1,5 @@
 /target/
+/web/target/
 /.clusterflux/
 **/.clusterflux/
 /vscode-extension/node_modules/
-/private/*/Cargo.lock
-!/private/hosted-policy/Cargo.lock
-/private/*/target/
-/scripts/containers-home/

CLUSTERFLUX_PUBLIC_TREE.json

@@ -1,22 +0,0 @@
-{
-  "kind": "clusterflux-filtered-public-tree",
-  "source_commit": "f6c01dca61eefca1a7431c48878a04dbb7a3f710",
-  "release_name": "release-f6c01dca61ee",
-  "filtered_out": [
-    "private/**",
-    "internal/**",
-    "experiments/**",
-    ".git",
-    "target",
-    "git-ignored source paths",
-    "root/*.md except README.md",
-    "**/.clusterflux/**",
-    ".forgejo/**"
-  ],
-  "public_export": {
-    "host_neutral": true,
-    "include_forgejo_workflows": false
-  },
-  "forgejo_host": "git.michelpaulissen.com",
-  "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
-}

Cargo.lock

@@ -290,6 +290,20 @@ dependencies = [
  "wasmparser 0.245.1",
 ]
 
+[[package]]
+name = "clusterflux-client"
+version = "0.1.0"
+dependencies = [
+ "base64",
+ "clusterflux-control",
+ "clusterflux-coordinator",
+ "clusterflux-core",
+ "serde",
+ "serde_json",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
 [[package]]
 name = "clusterflux-control"
 version = "0.1.0"
@@ -967,6 +981,13 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hello-build"
+version = "0.1.0"
+dependencies = [
+ "clusterflux-sdk",
+]
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -1144,16 +1165,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "launch-build-demo"
-version = "0.1.0"
-dependencies = [
- "clusterflux-sdk",
- "futures-executor",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -1670,6 +1681,14 @@ dependencies = [
  "yasna",
 ]
 
+[[package]]
+name = "recovery-build"
+version = "0.1.0"
+dependencies = [
+ "clusterflux-sdk",
+ "serde",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 resolver = "2"
+exclude = ["web"]
 members = [
   "crates/clusterflux-cli",
+  "crates/clusterflux-client",
   "crates/clusterflux-control",
   "crates/clusterflux-coordinator",
   "crates/clusterflux-core",
@@ -10,13 +12,14 @@ members = [
   "crates/clusterflux-node",
   "crates/clusterflux-sdk",
   "crates/clusterflux-wasm-runtime",
-  "examples/launch-build-demo",
+    "examples/hello-build",
+    "examples/recovery-build",
 ]
 
 [workspace.package]
 edition = "2021"
 license = "Apache-2.0 OR MIT"
-repository = "https://git.michelpaulissen.com/michel/clusterflux-public"
+repository = "https://clusterflux.lesstuff.com"
 
 [workspace.dependencies]
 anyhow = "1.0"

README.md

@@ -1,14 +1,82 @@
 # Clusterflux
 
-Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
+Clusterflux runs a Rust-defined workflow as one distributed virtual process. The
-coordinator hosts the async main, while attached nodes execute Wasm tasks,
+async main runs serverless, provisioning nodes to run tasks through rootless
-rootless containers, and native commands. Tasks exchange canonical values and
+containers. Tasks on nodes exchange data simply and efficiently.
-portable typed handles instead of sharing host memory.
+
+The user experience is built to be as much as possible like building a regular
+program. The processes are debuggable as normal through a debugger adapter.
+
+The primary use case is to consolidate build processes into a single streamlined
+developer experience. An example program can be seen below:
+
+~~~rust
+use clusterflux::prelude::*;
+
+#[clusterflux::task(capabilities = "command")]
+pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
+    let executable = fs::output("hello-clusterflux")?;
+    Command::new("cc")
+        .args([
+            "-Os",
+            "-static",
+            "-s",
+            "fixture/hello-clusterflux.c",
+            "-o",
+            executable.as_str(),
+        ])
+        .cwd(source.mount()?)
+        .env("SOURCE_DATE_EPOCH", "0")
+        .network_disabled()
+        .run()
+        .await?;
+    fs::publish(&executable).await
+}
+
+#[clusterflux::main]
+pub async fn build() -> Result<Artifact> {
+    let source = source::current_project().snapshot().await?;
+    let compile = clusterflux::spawn!(compile(source))
+        .on(clusterflux::env!("linux"))
+        .await?;
+    compile.join().await
+}
+~~~
+
+After setup, this build pipeline could be deployed as easily as launching it
+through your IDE. This repository includes a VS Code extension to make
+development as straightforward as possible. A full collection of CLI tools is
+included for advanced usage.
+
+Clusterflux is explicitly local-first. It is trivial to provision existing
+hardware as resources. Bulk data will typically not leave the local network,
+allowing maximum throughput. The same capability, however, also makes it
+possible to leverage cloud resources easily.
 
 Start with [Getting started](docs/getting-started.md). It takes you through
 authentication, project setup, node enrollment, a run, debugging, task restart,
 and artifact download.
 
+## Build a real executable
+
+The primary example is intentionally small: [hello-build](examples/hello-build)
+snapshots its source project, spawns one container-backed compile task, and
+publishes the resulting executable as a retained artifact.
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The final command prints `hello from a real Clusterflux build`. The source uses
+only the public SDK path: current-project snapshot, `spawn!`, `Command::run`, and
+artifact publication. See [recovery-build](examples/recovery-build) for two
+same-definition task instances, a real command failure, and operator restart.
+
 ## What you get
 
 - One virtual process with distinct task instances and restart attempts.
@@ -29,6 +97,9 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Rootless Podman is required on Linux nodes that build or run a declared
 Containerfile environment. Install VS Code when you want the graphical debug
 workflow.
@@ -54,8 +125,8 @@ explicit-key option.
 Choose an entrypoint and run it:
 
 ~~~bash
-clusterflux bundle inspect --project examples/launch-build-demo
+clusterflux bundle inspect --project examples/hello-build
-clusterflux run --project examples/launch-build-demo build
+clusterflux run --project examples/hello-build build
 ~~~
 
 Inspect the result:
@@ -66,7 +137,7 @@ clusterflux task list
 clusterflux logs
 clusterflux artifact list
 # Use the `artifact` value returned by the list command, for example:
-clusterflux artifact download release.tar-4f61c2... --to ./release.tar
+clusterflux artifact download hello-clusterflux-4f61c2... --to ./hello-clusterflux
 ~~~
 
 To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md).

SECURITY.md

@@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained.
 
 ## Report a vulnerability
 
-Email security@michelpaulissen.com with a concise description, affected version
+Email security@clusterflux.lesstuff.com with a concise description, affected version
 or source revision, reproduction steps, and impact. Do not open a public issue
 for an unpatched vulnerability or include credentials, session tokens, private
 keys, provider tokens, customer data, or operator secrets in a public report.

crates/clusterflux-cli/src/admin.rs

@@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
                 .unwrap_or(json!(false)),
             "response": response,
             "safe_default": "read_only",
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
         "command": "admin status",
         "mode": "self_hosted_local",
         "safe_default": "read_only",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
         "project": project.clone(),
         "user": user,
         "coordinator": scope.coordinator,
-        "private_website_required": false,
+        "external_website_required": false,
         "self_hosted_cli_only": true,
         "project_config_written": project_init
             .get("project_config_written")
@@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
             {
                 "step": "start_self_hosted_coordinator",
                 "command": "clusterflux-coordinator --listen 127.0.0.1:0",
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_or_link_project",
                 "command": "clusterflux project init --yes",
                 "completed": true,
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_node_enrollment_grant",
@@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "attach_worker_node",
@@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "run_process",
@@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "inspect_status_logs_artifacts",
@@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux artifact list",
                     "clusterflux quota status",
                 ],
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "revoke_access",
                 "command": "clusterflux admin revoke-node --node <node-id> --yes",
-                "private_website_required": false,
+                "external_website_required": false,
             }
         ],
     }))
@@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
             "actor_tenant": actor_tenant,
             "actor_user": actor_user,
             "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
         "status": "requires_coordinator",
         "requires_confirmation": !args.yes,
         "tenant": tenant,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 

crates/clusterflux-cli/src/artifact.rs

@@ -12,6 +12,7 @@ use crate::client::{
 };
 use crate::config::StoredCliSession;
 use crate::errors::cli_error_summary_for_category;
+use crate::process::hydrate_process_scope;
 use crate::process_events::{
     artifact_download_grant_disclosures, artifact_download_session_summary,
     artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
@@ -27,9 +28,10 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
 }
 
 pub(crate) fn artifact_list_report_with_session(
-    args: ArtifactListArgs,
+    mut args: ArtifactListArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,
@@ -53,9 +55,10 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Val
 }
 
 pub(crate) fn artifact_download_report_with_session(
-    args: ArtifactDownloadArgs,
+    mut args: ArtifactDownloadArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(
@@ -143,9 +146,10 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value>
 }
 
 pub(crate) fn artifact_export_report_with_session(
-    args: ArtifactExportArgs,
+    mut args: ArtifactExportArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(

crates/clusterflux-cli/src/auth.rs

@@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<V
                 "suspension_known": false,
                 "account_state_known": false,
                 "account_status": "unknown",
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
             })
         });
@@ -263,7 +263,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -296,7 +296,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -316,7 +316,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -338,7 +338,7 @@ fn coordinator_auth_status_summary(
             "account_status": "unknown",
             "suspension_known": false,
             "account_state_known": false,
-            "private_moderation_details_exposed": false,
+            "sensitive_moderation_details_exposed": false,
             "signup_failure_details_exposed": false,
             "machine_error": cli_error_summary(message),
             "coordinator_response_type": "error",
@@ -403,7 +403,7 @@ fn coordinator_auth_status_summary(
         "manual_review": manual_review,
         "sanitized_reason": sanitized_reason,
         "next_actions": next_actions,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
         "coordinator_session_requests": coordinator_session_requests,

crates/clusterflux-cli/src/auth_scope.rs

@@ -60,7 +60,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
         "account_status": "unknown",
         "suspension_known": false,
         "account_state_known": false,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "next_actions": ["clusterflux login --browser"],
     })

crates/clusterflux-cli/src/config.rs

@@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::CliScopeArgs;
 
-pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
+pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com";
-    "https://clusterflux.michelpaulissen.com";
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub(crate) struct ProjectConfig {

crates/clusterflux-cli/src/debug.rs

@@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
         "command": "dap",
         "adapter": dap_binary_path()?.display().to_string(),
         "args": args.args,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
                 .cloned()
                 .unwrap_or_else(|| json!(0)),
             "debug_reads_quota_limited": true,
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
         "dap": dap,
         "authorized": "unknown_without_coordinator",
         "debug_reads_quota_limited": "unknown_without_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }

crates/clusterflux-cli/src/errors.rs

@@ -38,7 +38,10 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s
             );
             object.insert("community_tier_language".to_owned(), json!(true));
             object.insert("community_tier_label".to_owned(), json!("community tier"));
-            object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
+            object.insert(
+                "sensitive_abuse_heuristics_exposed".to_owned(),
+                json!(false),
+            );
         }
     }
     summary

crates/clusterflux-cli/src/logs.rs

@@ -3,6 +3,7 @@ use serde_json::{json, Value};
 
 use crate::client::list_task_events_if_available_with_session;
 use crate::config::StoredCliSession;
+use crate::process::hydrate_process_scope;
 use crate::process_events::log_entries;
 use crate::LogsArgs;
 
@@ -12,9 +13,10 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
 }
 
 pub(crate) fn logs_report_with_session(
-    args: LogsArgs,
+    mut args: LogsArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,

crates/clusterflux-cli/src/node.rs

@@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
             "tenant": tenant,
             "project": project,
             "user": user,
-            "private_website_required": false,
+            "external_website_required": false,
             "enrollment_grant": enrollment_grant,
             "response": response,
             "coordinator_session_requests": session.requests(),
@@ -161,7 +161,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
     Ok(json!({
         "command": "node enroll",
         "status": "requires_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
         "requested_ttl_seconds": args.ttl_seconds,
         "enrollment_grant": null,
         "reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
@@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
     };
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &tenant,
+        "project": &project,
         "node": &plan.node,
     });
     let heartbeat_signature = sign_node_request(

crates/clusterflux-cli/src/output.rs

@@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         }
         push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
         if let Some(exposed) = account
-            .get("private_moderation_details_exposed")
+            .get("sensitive_moderation_details_exposed")
             .and_then(Value::as_bool)
         {
-            lines.push(format!("private moderation details exposed: {exposed}"));
+            lines.push(format!("sensitive moderation details exposed: {exposed}"));
         }
     }
     if let Some(coordinator_selection) = value.get("coordinator") {
@@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         ));
     }
     if let Some(flag) = value
-        .get("private_website_required")
+        .get("external_website_required")
         .and_then(Value::as_bool)
     {
-        lines.push(format!("private website required: {flag}"));
+        lines.push(format!("external website required: {flag}"));
     }
     if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
         let actions = next_actions

crates/clusterflux-cli/src/process.rs

@@ -15,7 +15,10 @@ use crate::{
     ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
 };
 
-fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
+pub(crate) fn hydrate_process_scope(
+    scope: &mut CliScopeArgs,
+    stored_session: Option<&StoredCliSession>,
+) {
     if scope.coordinator.is_none() {
         scope.coordinator = stored_session
             .filter(|session| session.session_secret.is_some())

crates/clusterflux-cli/src/process_events.rs

@@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
         "cross_tenant_reuse_allowed": false,
         "unauthorized_project_reuse_allowed": false,
         "default_durable_store_assumed": false,
-        "private_website_required": false,
+        "external_website_required": false,
     }])
 }
 
@@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<
         "current_usage": current_usage,
         "limits": quota_limits_value(),
         "next_blocked_action": next_blocked_action,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
     })
 }
 

crates/clusterflux-cli/src/project.rs

@@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
     Ok(json!({
         "command": "project init",
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config_written": true,
         "project_config_write_after_coordinator_acceptance": coordinator.is_some(),
         "coordinator_create_before_local_write": coordinator.is_some(),
@@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "config_format": "clusterflux_project_config_v1",
             "links_current_directory": true,
             "writes_current_directory_only": true,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "safe_defaults": {
             "tenant": config.tenant.clone(),
@@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "default_project_id_used": args.new_project == "project",
             "default_project_name_used": args.name == "Clusterflux Project",
             "browser_interaction_required": false,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "project_config": config,
         "config_file": project_config_file(&cwd),
@@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
             "user": user,
             "projects": projects,
             "project_count": project_count,
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
         "source": "local_project_config",
         "projects": projects,
         "project_count": project_count,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
         "selected_project": selected_project,
         "project_config_written": true,
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config": config,
         "coordinator_response": coordinator_response,
     }))

crates/clusterflux-cli/src/quota.rs

@@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "user": effective_scope.user,
         "coordinator": coordinator,
         "project_config": config,
-        "policy_surface": "generic public quota categories; hosted tuning remains private policy",
+        "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined",
         "limits": limits,
         "window_seconds": window_seconds,
         "current_usage": current_usage,
@@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "next_blocked_action": quota_next_blocked_action(&current_usage),
         "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
         "quota_tier": quota_tier,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
         "quota_response": quota_status,
     }))
 }

crates/clusterflux-cli/src/run.rs

@@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
         "safe_failure": true,
         "message": message,
         "next_actions": next_actions,
-        "private_website_required": false,
+        "external_website_required": false,
         "machine_error": machine_error,
     })
 }
@@ -195,12 +195,14 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         );
     }
     let process = "vp-current".to_owned();
+    let launch_attempt = new_launch_attempt_id();
     let mut session = JsonLineSession::connect(&coordinator)?;
     let mut request = json!({
         "type": "start_process",
         "tenant": tenant.clone(),
         "project": project.clone(),
         "process": process.clone(),
+        "launch_attempt": launch_attempt.clone(),
         "restart": false,
     });
     request = authenticated_human_or_local_trusted_workflow(
@@ -219,6 +221,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         &tenant,
         &project,
         &process,
+        &launch_attempt,
     )?;
     let run_start = run_start_summary(&response);
     let status = run_start
@@ -286,6 +289,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
                     tenant: &tenant,
                     project: &project,
                     process: &process,
+                    launch_attempt: &launch_attempt,
                 };
                 let cleanup = rollback_failed_process_launch_reconnecting(
                     &coordinator,
@@ -325,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         "task_launch": launch_task_response,
         "coordinator_response": response,
         "coordinator_session_requests": session.requests(),
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -340,6 +344,7 @@ fn request_process_start_with_rollback(
     tenant: &str,
     project: &str,
     process: &str,
+    launch_attempt: &str,
 ) -> Result<Value> {
     let rollback = ProcessLaunchRollback {
         cli_session,
@@ -348,9 +353,25 @@ fn request_process_start_with_rollback(
         tenant,
         project,
         process,
+        launch_attempt,
     };
     match session.request_allow_error(request) {
-        Ok(response) => Ok(response),
+        Ok(response) => {
+            if response.get("type").and_then(Value::as_str) == Some("process_started")
+                && response.get("launch_attempt").and_then(Value::as_str) != Some(launch_attempt)
+            {
+                let ownership_error = anyhow::anyhow!(
+                    "coordinator returned a process-start acknowledgement owned by a different launch attempt"
+                );
+                rollback_failed_process_launch_reconnecting(
+                    coordinator,
+                    &rollback,
+                    &json!({ "error": ownership_error.to_string(), "phase": "start_process_ownership" }),
+                )?;
+                return Err(ownership_error);
+            }
+            Ok(response)
+        }
         Err(start_error) => {
             let cleanup = rollback_failed_process_launch_reconnecting(
                 coordinator,
@@ -370,6 +391,17 @@ fn request_process_start_with_rollback(
     }
 }
 
+static NEXT_LAUNCH_ATTEMPT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+
+fn new_launch_attempt_id() -> String {
+    let sequence = NEXT_LAUNCH_ATTEMPT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_nanos();
+    format!("launch-{}-{nanos}-{sequence}", std::process::id())
+}
+
 struct ProcessLaunchRollback<'a> {
     cli_session: &'a CliSession,
     fallback_user: &'a str,
@@ -377,6 +409,7 @@ struct ProcessLaunchRollback<'a> {
     tenant: &'a str,
     project: &'a str,
     process: &'a str,
+    launch_attempt: &'a str,
 }
 
 fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> {
@@ -402,6 +435,7 @@ fn rollback_failed_process_launch(
             "tenant": context.tenant,
             "project": context.project,
             "process": context.process,
+            "launch_attempt": context.launch_attempt,
         }),
         context.cli_session,
         context.fallback_user,
@@ -983,6 +1017,10 @@ mod transactional_launch_tests {
                 payload.get("process").and_then(Value::as_str),
                 Some("vp-current")
             );
+            assert_eq!(
+                payload.get("launch_attempt").and_then(Value::as_str),
+                Some("launch-test")
+            );
             writeln!(
                 stream,
                 "{}",
@@ -1003,6 +1041,7 @@ mod transactional_launch_tests {
             tenant: "tenant-a",
             project: "project-a",
             process: "vp-current",
+            launch_attempt: "launch-test",
         };
         rollback_failed_process_launch(
             &mut session,
@@ -1024,6 +1063,7 @@ mod transactional_launch_tests {
                 .read_line(&mut start_line)
                 .unwrap();
             assert!(start_line.contains("\"type\":\"start_process\""));
+            assert!(start_line.contains("\"launch_attempt\":\"launch-test\""));
             drop(stream);
 
             let (mut cleanup_stream, _) = listener.accept().unwrap();
@@ -1032,6 +1072,7 @@ mod transactional_launch_tests {
                 .read_line(&mut cleanup_line)
                 .unwrap();
             assert!(cleanup_line.contains("\"type\":\"abort_process\""));
+            assert!(cleanup_line.contains("\"launch_attempt\":\"launch-test\""));
             writeln!(
                 cleanup_stream,
                 "{}",
@@ -1043,6 +1084,12 @@ mod transactional_launch_tests {
                 })
             )
             .unwrap();
+            listener.set_nonblocking(true).unwrap();
+            std::thread::sleep(std::time::Duration::from_millis(25));
+            assert!(matches!(
+                listener.accept().unwrap_err().kind(),
+                std::io::ErrorKind::WouldBlock
+            ));
         });
         let mut session = JsonLineSession::connect(&address).unwrap();
         let error = request_process_start_with_rollback(
@@ -1053,6 +1100,7 @@ mod transactional_launch_tests {
                 "project": "project-a",
                 "actor_user": "user-a",
                 "process": "vp-current",
+                "launch_attempt": "launch-test",
                 "restart": false
             }),
             &address,
@@ -1062,10 +1110,63 @@ mod transactional_launch_tests {
             "tenant-a",
             "project-a",
             "vp-current",
+            "launch-test",
         )
         .unwrap_err()
         .to_string();
         assert!(error.contains("closed") || error.contains("response"));
         server.join().unwrap();
     }
+
+    #[test]
+    fn definitive_start_rejection_does_not_abort_existing_process() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap().to_string();
+        let server = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().unwrap();
+            let mut line = String::new();
+            std::io::BufReader::new(stream.try_clone().unwrap())
+                .read_line(&mut line)
+                .unwrap();
+            writeln!(
+                stream,
+                "{}",
+                json!({
+                    "type": "error",
+                    "message": "project already has active virtual process vp-existing"
+                })
+            )
+            .unwrap();
+            listener.set_nonblocking(true).unwrap();
+            std::thread::sleep(std::time::Duration::from_millis(25));
+            assert_eq!(
+                listener.accept().unwrap_err().kind(),
+                std::io::ErrorKind::WouldBlock
+            );
+        });
+        let mut session = JsonLineSession::connect(&address).unwrap();
+        let response = request_process_start_with_rollback(
+            &mut session,
+            json!({
+                "type": "start_process",
+                "tenant": "tenant-a",
+                "project": "project-a",
+                "actor_user": "user-a",
+                "process": "vp-current",
+                "launch_attempt": "launch-rejected",
+                "restart": false
+            }),
+            &address,
+            &CliSession::Anonymous,
+            "user-a",
+            None,
+            "tenant-a",
+            "project-a",
+            "vp-current",
+            "launch-rejected",
+        )
+        .unwrap();
+        assert_eq!(response.get("type").and_then(Value::as_str), Some("error"));
+        server.join().unwrap();
+    }
 }

crates/clusterflux-cli/src/tests.rs

@@ -10,6 +10,15 @@ fn parse(args: &[&str]) -> Cli {
     Cli::parse_from(args)
 }
 
+fn launch_attempt_from_wire(line: &str) -> String {
+    let wire: Value = serde_json::from_str(line).unwrap();
+    wire.pointer("/payload/request/launch_attempt")
+        .or_else(|| wire.pointer("/payload/launch_attempt"))
+        .and_then(Value::as_str)
+        .expect("start request must carry a launch attempt")
+        .to_owned()
+}
+
 fn write_runnable_wasm_project(project: &Path) {
     let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
     let sdk = cli_manifest_dir.parent().unwrap().join("clusterflux-sdk");
@@ -115,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
                 assert_eq!(summary["resource_category"], "api_calls");
                 assert_eq!(summary["community_tier_language"], true);
                 assert_eq!(summary["community_tier_label"], "community tier");
-                assert_eq!(summary["private_abuse_heuristics_exposed"], false);
+                assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false);
                 let rendered = human_report(&json!({
                     "command": "run",
                     "machine_error": summary,
@@ -198,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
     assert_eq!(report["status"], "authentication_required");
     assert_eq!(report["non_interactive"], true);
     assert_eq!(report["browser_opened"], false);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["machine_error"]["category"], "authentication");
     assert_eq!(report["machine_error"]["stable_exit_code"], 20);
     assert_eq!(report["machine_error"]["browser_opened"], false);
@@ -438,10 +447,11 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
         assert!(line.contains(r#""nonce":"agent-signature-"#));
         assert!(line.contains(r#""signature":"ed25519:"#));
         assert!(!line.contains(r#""actor_user""#));
+        let launch_attempt = launch_attempt_from_wire(&line);
         stream
                 .write_all(
                     format!(
-                        r#"{{"type":"process_started","process":"vp-current","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
+                        r#"{{"type":"process_started","process":"vp-current","launch_attempt":"{launch_attempt}","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
                     )
                     .as_bytes(),
                 )
@@ -588,7 +598,32 @@ fn run_with_human_session_derives_workflow_scope_from_authenticated_envelope() {
             assert!(wire["payload"]["request"].get("tenant").is_none());
             assert!(wire["payload"]["request"].get("project").is_none());
             assert!(wire["payload"]["request"].get("actor_user").is_none());
-            stream.write_all(response).unwrap();
+            if expected_operation == "start_process" {
+                let launch_attempt = launch_attempt_from_wire(&line);
+                stream
+                    .write_all(
+                        serde_json::to_string(&json!({
+                            "type": "process_started",
+                            "process": "vp-current",
+                            "launch_attempt": launch_attempt,
+                            "epoch": 7,
+                            "actor": {
+                                "kind": "user",
+                                "user": "user-session",
+                                "agent": null,
+                                "credential_kind": "cli_device_session",
+                                "public_key_fingerprint": null,
+                                "authenticated_without_browser": false,
+                                "scopes": ["project:read", "project:run"]
+                            }
+                        }))
+                        .unwrap()
+                        .as_bytes(),
+                    )
+                    .unwrap();
+            } else {
+                stream.write_all(response).unwrap();
+            }
             stream.write_all(b"\n").unwrap();
         }
     });
@@ -661,6 +696,18 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
                 assert!(line.contains(r#""project":"project-live""#));
                 assert!(line.contains(r#""process":"vp-current""#));
                 assert!(line.contains(r#""restart":false"#));
+                let response = if index == 0 {
+                    let launch_attempt = launch_attempt_from_wire(&line);
+                    serde_json::to_string(&json!({
+                        "type": "process_started",
+                        "process": "vp-current",
+                        "launch_attempt": launch_attempt,
+                        "epoch": 7,
+                    }))
+                    .unwrap()
+                } else {
+                    response.to_owned()
+                };
                 stream.write_all(response.as_bytes()).unwrap();
                 stream.write_all(b"\n").unwrap();
                 if index == 0 {
@@ -780,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() {
     assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
     assert_eq!(rejected["machine_error"]["community_tier_language"], true);
     assert_eq!(
-        rejected["machine_error"]["private_abuse_heuristics_exposed"],
+        rejected["machine_error"]["sensitive_abuse_heuristics_exposed"],
         false
     );
     assert!(rejected["machine_error"]["next_actions"]
@@ -1618,12 +1665,11 @@ fn node_attach_refuses_a_symlink_credential_target() {
 fn hosted_coordinator_remains_a_real_https_control_endpoint() {
     assert_eq!(
         control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
+        "https://clusterflux.lesstuff.com/api/v1/control"
     );
     assert_eq!(
-        control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
+        control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
-            .unwrap(),
+        "https://clusterflux.lesstuff.com/api/v1/control"
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
     );
     assert!(control_endpoint_identity("http://operator.example.test").is_err());
     assert_eq!(
@@ -1741,7 +1787,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         assert!(!line.contains(r#""actor_user":"user-session""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1804,7 +1850,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         "active"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
 }
@@ -1886,7 +1932,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
 }
 
 #[test]
-fn auth_status_queries_coordinator_account_state_without_private_moderation_details() {
+fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() {
     let listener = TcpListener::bind("127.0.0.1:0").unwrap();
     let addr = listener.local_addr().unwrap().to_string();
     let server = std::thread::spawn(move || {
@@ -1900,7 +1946,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         assert!(line.contains(r#""actor_user":"user-live""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1940,7 +1986,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         "account or tenant is suspended by hosted policy"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
     assert_eq!(
@@ -1958,13 +2004,13 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
     let serialized = serde_json::to_string(&report).unwrap();
     assert!(!serialized.contains("abuse_score"));
     assert!(!serialized.contains("moderation_notes"));
-    assert!(!serialized.contains("private moderation note"));
+    assert!(!serialized.contains("sensitive moderation note"));
 
     let rendered = human_report(&report);
     assert!(rendered.contains("account status: suspended"));
     assert!(rendered.contains("account suspended: true"));
-    assert!(rendered.contains("private moderation details exposed: false"));
+    assert!(rendered.contains("sensitive moderation details exposed: false"));
-    assert!(!rendered.contains("private moderation note"));
+    assert!(!rendered.contains("sensitive moderation note"));
 }
 
 #[test]
@@ -2015,11 +2061,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
                     "manual_review": manual_review,
                     "sanitized_reason": reason,
                     "next_actions": ["contact the hosted operator"],
-                    "private_moderation_details_exposed": false,
+                    "sensitive_moderation_details_exposed": false,
                     "signup_failure_details_exposed": false,
                     "abuse_score": 99,
-                    "moderation_notes": "private moderation note",
+                    "moderation_notes": "sensitive moderation note",
-                    "signup_policy_trace": "private signup trace",
+                    "signup_policy_trace": "sensitive signup trace",
                 })
             )
             .unwrap();
@@ -2058,7 +2104,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
             true
         );
         assert_eq!(
-            report["coordinator_account_status"]["private_moderation_details_exposed"],
+            report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
             false
         );
         assert_eq!(
@@ -2069,11 +2115,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
         assert!(!serialized.contains("abuse_score"));
         assert!(!serialized.contains("moderation_notes"));
         assert!(!serialized.contains("signup_policy_trace"));
-        assert!(!serialized.contains("private moderation note"));
+        assert!(!serialized.contains("sensitive moderation note"));
         let rendered = human_report(&report);
         assert!(rendered.contains(&format!("account status: {status}")));
         assert!(rendered.contains(rendered_marker));
-        assert!(!rendered.contains("private moderation note"));
+        assert!(!rendered.contains("sensitive moderation note"));
     }
     server.join().unwrap();
 }
@@ -2188,11 +2234,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
 
     assert_eq!(report["command"], "admin bootstrap");
     assert_eq!(report["mode"], "self_hosted_local");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["self_hosted_cli_only"], true);
     assert_eq!(report["project_config_written"], true);
     assert_eq!(report["project_init"]["command"], "project init");
-    assert_eq!(report["project_init"]["private_website_required"], false);
+    assert_eq!(report["project_init"]["external_website_required"], false);
     assert_eq!(
         report["project_init"]["project_config"]["project"],
         "self-hosted"
@@ -2218,7 +2264,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
     }
     assert!(steps.iter().all(|step| {
         !step
-            .get("private_website_required")
+            .get("external_website_required")
             .and_then(Value::as_bool)
             .unwrap_or(false)
     }));
@@ -2846,13 +2892,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
 
     assert_eq!(status["command"], "admin status");
     assert_eq!(status["safe_default"], "read_only");
-    assert_eq!(status["private_website_required"], false);
+    assert_eq!(status["external_website_required"], false);
     assert_eq!(status["suspended"], false);
     assert_eq!(suspended["command"], "admin suspend-tenant");
     assert_eq!(suspended["tenant"], "tenant");
     assert_eq!(suspended["actor_tenant"], "admin-tenant");
     assert_eq!(suspended["suspended"], true);
-    assert_eq!(suspended["private_website_required"], false);
+    assert_eq!(suspended["external_website_required"], false);
 }
 
 #[test]
@@ -2921,7 +2967,7 @@ fn debug_attach_reports_public_authorization() {
     assert_eq!(report["charged_debug_read_bytes"], 1024);
     assert_eq!(report["used_debug_read_bytes"], 1024);
     assert_eq!(report["debug_reads_quota_limited"], true);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
 }
 
 #[test]
@@ -2958,6 +3004,14 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
                 "restart_task",
                 br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
             ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(),
+            ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(),
+            ),
             (
                 "create_artifact_download_link",
                 br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
@@ -3062,9 +3116,26 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
-    artifact_download_report_with_session(
+    let logs = logs_report_with_session(
+        LogsArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+            task: Some("task-a".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let artifacts = artifact_list_report_with_session(
+        ArtifactListArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let download = artifact_download_report_with_session(
         ArtifactDownloadArgs {
-            scope: coordinator_scope.clone(),
+            scope,
             artifact: "app.txt".to_owned(),
             to: None,
             max_bytes: 2048,
@@ -3072,6 +3143,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
+    assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n");
+    assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt");
+    assert_eq!(download["coordinator"], session.coordinator);
     debug_attach_report_with_dap_and_session(
         DebugAttachArgs {
             scope: coordinator_scope,
@@ -3264,7 +3338,7 @@ fn project_init_uses_public_create_before_writing_local_config() {
         created["safe_defaults"]["browser_interaction_required"],
         false
     );
-    assert_eq!(created["private_website_required"], false);
+    assert_eq!(created["external_website_required"], false);
     assert_eq!(
         read_project_config(temp_success.path())
             .unwrap()
@@ -3364,7 +3438,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(list["source"], "public_coordinator_api");
     assert_eq!(list["project_count"], 1);
     assert_eq!(list["projects"][0]["id"], "project-a");
-    assert_eq!(list["private_website_required"], false);
+    assert_eq!(list["external_website_required"], false);
     assert_eq!(list["coordinator_session_requests"], 1);
 
     let selected = project_select_report(
@@ -3379,7 +3453,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(selected["source"], "public_coordinator_api");
     assert_eq!(selected["selected_project"]["id"], "project-a");
     assert_eq!(selected["project_config_written"], true);
-    assert_eq!(selected["private_website_required"], false);
+    assert_eq!(selected["external_website_required"], false);
     assert_eq!(
         read_project_config(temp.path()).unwrap().unwrap().project,
         "project-a"
@@ -4408,7 +4482,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     let temp = tempfile::tempdir().unwrap();
     let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .join("../..")
-        .join("examples/launch-build-demo");
+        .join("examples/hello-build");
     let output = temp.path().join("bundle");
 
     let report = build_report(
@@ -4426,8 +4500,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     assert_eq!(report["command"], "build");
     assert_eq!(report["content_addressed"], true);
     assert_eq!(report["contains_full_repository_upload"], false);
-    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 11);
+    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2);
-    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 6);
+    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1);
     assert!(output.join("module.wasm").is_file());
     assert!(output.join("manifest.json").is_file());
     assert!(output.join("task-descriptors.json").is_file());
@@ -4438,9 +4512,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
         .unwrap()
         .iter()
         .any(|descriptor| {
-            descriptor["name"] == "task_add_one"
+            descriptor["name"] == "compile"
-                && descriptor["argument_schema"] == "input : i32"
+                && descriptor["argument_schema"] == "source : SourceSnapshot"
-                && descriptor["result_schema"] == "i32"
+                && descriptor["result_schema"] == "Result < Artifact >"
                 && descriptor["restart_compatibility_hash"]
                     .as_str()
                     .unwrap()
@@ -4601,7 +4675,7 @@ fn node_enroll_reports_short_lived_public_api_grant() {
 
     assert_eq!(report["command"], "node enroll");
     assert_eq!(report["status"], "created");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["tenant"], "tenant");
     assert_eq!(report["project"], "project");
     assert_eq!(report["user"], "user");
@@ -4744,7 +4818,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() {
     )
     .unwrap();
     assert_eq!(enroll["status"], "requires_coordinator");
-    assert_eq!(enroll["private_website_required"], false);
+    assert_eq!(enroll["external_website_required"], false);
     assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
     assert_eq!(enroll["requested_ttl_seconds"], 60);
 

crates/clusterflux-client/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "clusterflux-client"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+clusterflux-control = { path = "../clusterflux-control" }
+clusterflux-core = { path = "../clusterflux-core" }
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["sync", "time"] }
+
+[dev-dependencies]
+clusterflux-coordinator = { path = "../clusterflux-coordinator" }

crates/clusterflux-client/README.md

@@ -0,0 +1,68 @@
+# clusterflux-client
+
+`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web
+backend. It talks to the same versioned control and login endpoints as the CLI.
+Callers do not construct JSON envelopes or place session secrets into request
+objects.
+
+The crate intentionally contains no coordinator implementation, persistence
+types, scheduler policy, or website-specific business rules. A web backend only
+needs this crate to authenticate and work with account status, projects, Agent
+keys, node enrollment and liveness, current/recent processes, task attempts,
+recent logs, artifacts, quota state, process control, task recovery, and Debug
+Epoch state. It does not expose workflow launch or whole-process replay.
+
+```rust,no_run
+use clusterflux_client::{ClusterfluxClient, SessionCredential};
+
+# async fn example() -> Result<(), clusterflux_client::ClientError> {
+let credential = SessionCredential::from_secret(
+    std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"),
+);
+let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")?
+    .with_session_credential(&credential);
+
+let account = client.account_status().await?;
+let nodes = client.list_nodes().await?;
+let processes = client.list_processes(None, 20).await?;
+# let _ = (account, nodes, processes);
+# Ok(())
+# }
+```
+
+## Login boundary
+
+`begin_browser_login` starts the hosted Authentik flow. The identity callback
+returns only to the fixed configured website URL with a short-lived handoff.
+The website backend calls `exchange_browser_login_handoff` once and stores the
+returned `SessionCredential` only in encrypted server-side session storage.
+The credential has redacted `Debug` output and deliberately does not implement
+Serde serialization. Logout calls the existing session-revocation operation.
+
+The browser must not receive or persist the Clusterflux session credential,
+provider tokens, authorization code, PKCE verifier, or CLI polling secret.
+
+## Errors, bounds, and transport
+
+API failures are returned as `ClientError::Api(ApiError)`. Decisions should use
+the stable `code`, `category`, `retryable`, and `request_id` fields, while
+`message` is for people. The client rejects an error whose request ID does not
+match the originating request.
+
+Paginated list methods require an explicit bounded page size. Process pages are
+limited to 100 entries; node, artifact, and recent-log pages to 200.
+`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes
+its cursor. Recent logs are ephemeral and can report truncation or sequence
+loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates
+offsets and decodes bounded chunks without materializing the complete artifact.
+
+`ControlTransport` has bounded connect and I/O timeouts and performs blocking
+network work outside the async executor. Dropping a request future cancels the
+caller’s wait; any already-started blocking I/O remains bounded by its timeout.
+`MockTransport` supplies deterministic responses and records exact envelopes
+for application tests.
+
+`CLIENT_API_VERSION` is the one supported control protocol version. The
+checked-in `web_operations.json` contract fixture records every website
+operation and its stable error shape, and the crate’s contract suite checks the
+fixture.

crates/clusterflux-client/src/lib.rs

@@ -0,0 +1,924 @@
+mod protocol;
+mod transport;
+mod types;
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION};
+use protocol::{AuthenticatedRequest, LoginRequest, WireResponse};
+use serde_json::json;
+use thiserror::Error;
+use transport::{ClientTransport, TransportRequest};
+
+pub use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind,
+    Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId,
+    ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId,
+    UserId, VfsPath,
+};
+pub use transport::{
+    ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse,
+};
+pub use types::*;
+
+pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION;
+
+#[derive(Debug, Error)]
+pub enum ClientError {
+    #[error("Clusterflux API error: {0}")]
+    Api(ApiError),
+    #[error(transparent)]
+    Transport(#[from] ClientTransportError),
+    #[error("Clusterflux client protocol error: {0}")]
+    Protocol(String),
+}
+
+#[derive(Clone)]
+pub struct ClusterfluxClient {
+    transport: Arc<dyn ClientTransport>,
+    session_secret: Arc<Mutex<Option<String>>>,
+    next_request: Arc<AtomicU64>,
+}
+
+impl ClusterfluxClient {
+    pub fn connect(endpoint: impl Into<String>) -> Result<Self, ClientError> {
+        Ok(Self::with_transport(ControlTransport::new(endpoint)?))
+    }
+
+    pub fn with_transport(transport: impl ClientTransport) -> Self {
+        Self {
+            transport: Arc::new(transport),
+            session_secret: Arc::new(Mutex::new(None)),
+            next_request: Arc::new(AtomicU64::new(1)),
+        }
+    }
+
+    pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self {
+        self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone())));
+        self
+    }
+
+    pub fn is_session_configured(&self) -> bool {
+        self.session_secret
+            .lock()
+            .map(|secret| secret.is_some())
+            .unwrap_or(false)
+    }
+
+    pub async fn begin_browser_login(&self) -> Result<BrowserLoginStart, ClientError> {
+        match self
+            .send_login(LoginRequest::BeginWebBrowserLogin {})
+            .await?
+        {
+            WireResponse::WebBrowserLoginStarted {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            } => Ok(BrowserLoginStart {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_login_started")),
+        }
+    }
+
+    pub async fn exchange_browser_login_handoff(
+        &self,
+        transaction_id: impl Into<String>,
+        handoff_code: impl Into<String>,
+    ) -> Result<BrowserSession, ClientError> {
+        match self
+            .send_login(LoginRequest::ExchangeWebLoginHandoff {
+                transaction_id: transaction_id.into(),
+                handoff_code: handoff_code.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserSession { session } => Ok(BrowserSession {
+                tenant: session.tenant,
+                project: session.project,
+                user: session.user,
+                credential: SessionCredential(session.session_secret),
+                expires_at_epoch_seconds: session.expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_session")),
+        }
+    }
+
+    pub async fn cancel_browser_login(
+        &self,
+        transaction_id: impl Into<String>,
+    ) -> Result<(), ClientError> {
+        match self
+            .send_login(LoginRequest::CancelWebBrowserLogin {
+                transaction_id: transaction_id.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserLoginCancelled {} => Ok(()),
+            _ => Err(unexpected_response("web_browser_login_cancelled")),
+        }
+    }
+
+    pub async fn account_status(&self) -> Result<AccountStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AuthStatus)
+            .await?
+        {
+            WireResponse::AuthStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            } => Ok(AccountStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            }),
+            _ => Err(unexpected_response("auth_status")),
+        }
+    }
+
+    pub async fn logout(&self) -> Result<(), ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeCliSession)
+            .await?
+        {
+            WireResponse::CliSessionRevoked {} => {
+                *self.session_secret.lock().map_err(|_| {
+                    ClientError::Protocol("session credential lock was poisoned".to_owned())
+                })? = None;
+                Ok(())
+            }
+            _ => Err(unexpected_response("cli_session_revoked")),
+        }
+    }
+
+    pub async fn list_projects(&self) -> Result<Vec<Project>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProjects)
+            .await?
+        {
+            WireResponse::Projects { projects } => Ok(projects),
+            _ => Err(unexpected_response("projects")),
+        }
+    }
+
+    pub async fn create_project(
+        &self,
+        project: ProjectId,
+        name: impl Into<String>,
+    ) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateProject {
+                project,
+                name: name.into(),
+            })
+            .await?
+        {
+            WireResponse::ProjectCreated { project } => Ok(project),
+            _ => Err(unexpected_response("project_created")),
+        }
+    }
+
+    pub async fn select_project(&self, project: ProjectId) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::SelectProject { project })
+            .await?
+        {
+            WireResponse::ProjectSelected { project } => Ok(project),
+            _ => Err(unexpected_response("project_selected")),
+        }
+    }
+
+    pub async fn register_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn list_agent_public_keys(&self) -> Result<Vec<AgentPublicKey>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys)
+            .await?
+        {
+            WireResponse::AgentPublicKeys { records } => Ok(records),
+            _ => Err(unexpected_response("agent_public_keys")),
+        }
+    }
+
+    pub async fn rotate_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn revoke_agent_public_key(
+        &self,
+        agent: AgentId,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent })
+            .await
+    }
+
+    async fn agent_key_mutation(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<AgentPublicKey, ClientError> {
+        match self.send_authenticated(request).await? {
+            WireResponse::AgentPublicKey { record } => Ok(record),
+            _ => Err(unexpected_response("agent_public_key")),
+        }
+    }
+
+    pub async fn create_node_enrollment_grant(
+        &self,
+        ttl_seconds: u64,
+    ) -> Result<NodeEnrollmentGrant, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds })
+            .await?
+        {
+            WireResponse::NodeEnrollmentGrantCreated {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            } => Ok(NodeEnrollmentGrant {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("node_enrollment_grant_created")),
+        }
+    }
+
+    pub async fn list_nodes(&self) -> Result<Vec<NodeSummary>, ClientError> {
+        Ok(self.list_nodes_page(None, 200).await?.nodes)
+    }
+
+    pub async fn list_nodes_page(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<NodePage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::NodeSummaries { nodes, next_cursor } => {
+                Ok(NodePage { nodes, next_cursor })
+            }
+            _ => Err(unexpected_response("node_summaries")),
+        }
+    }
+
+    pub async fn revoke_node(&self, node: NodeId) -> Result<NodeRevocation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node })
+            .await?
+        {
+            WireResponse::NodeCredentialRevoked {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            } => Ok(NodeRevocation {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            }),
+            _ => Err(unexpected_response("node_credential_revoked")),
+        }
+    }
+
+    pub async fn list_processes(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ProcessPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::ProcessSummaries {
+                processes,
+                next_cursor,
+            } => Ok(ProcessPage {
+                processes,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("process_summaries")),
+        }
+    }
+
+    pub async fn cancel_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CancelProcess { process })
+            .await?
+        {
+            WireResponse::ProcessCancellationRequested {
+                process,
+                cancelled_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: cancelled_tasks,
+                affected_nodes,
+                aborted: false,
+            }),
+            _ => Err(unexpected_response("process_cancellation_requested")),
+        }
+    }
+
+    pub async fn abort_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AbortProcess {
+                process,
+                launch_attempt: None,
+            })
+            .await?
+        {
+            WireResponse::ProcessAborted {
+                process,
+                aborted_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: aborted_tasks,
+                affected_nodes,
+                aborted: true,
+            }),
+            _ => Err(unexpected_response("process_aborted")),
+        }
+    }
+
+    pub async fn quota_status(&self) -> Result<QuotaStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::QuotaStatus)
+            .await?
+        {
+            WireResponse::QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            } => Ok(QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("quota_status")),
+        }
+    }
+
+    pub async fn list_task_events(
+        &self,
+        process: Option<ProcessId>,
+    ) -> Result<Vec<TaskCompletionEvent>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskEvents { process })
+            .await?
+        {
+            WireResponse::TaskEvents { events } => Ok(events),
+            _ => Err(unexpected_response("task_events")),
+        }
+    }
+
+    pub async fn list_task_snapshots(
+        &self,
+        process: ProcessId,
+    ) -> Result<Vec<TaskAttemptSnapshot>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process })
+            .await?
+        {
+            WireResponse::TaskSnapshots { snapshots } => Ok(snapshots),
+            _ => Err(unexpected_response("task_snapshots")),
+        }
+    }
+
+    pub async fn list_recent_logs(
+        &self,
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<RecentLogPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::RecentLogs {
+                entries,
+                next_sequence,
+                history_truncated,
+            } => Ok(RecentLogPage {
+                entries,
+                next_sequence,
+                history_truncated,
+            }),
+            _ => Err(unexpected_response("recent_logs")),
+        }
+    }
+
+    pub async fn restart_task(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    ) -> Result<TaskRestart, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RestartTask {
+                process,
+                task,
+                replacement_bundle,
+            })
+            .await?
+        {
+            WireResponse::TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            } => Ok(TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("task_restart")),
+        }
+    }
+
+    pub async fn resolve_task_failure(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    ) -> Result<TaskFailureResolutionResult, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResolveTaskFailure {
+                process,
+                task,
+                resolution,
+            })
+            .await?
+        {
+            WireResponse::TaskFailureResolved {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            } => Ok(TaskFailureResolutionResult {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            }),
+            _ => Err(unexpected_response("task_failure_resolved")),
+        }
+    }
+
+    pub async fn debug_attach(&self, process: ProcessId) -> Result<DebugAttach, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::DebugAttach { process })
+            .await?
+        {
+            WireResponse::DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            } => Ok(DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_attach")),
+        }
+    }
+
+    pub async fn create_debug_epoch(
+        &self,
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: impl Into<String>,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateDebugEpoch {
+                process,
+                stopped_task,
+                reason: reason.into(),
+            })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn resume_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn inspect_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            } => Ok(DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch_status")),
+        }
+    }
+
+    pub async fn list_artifacts(
+        &self,
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ArtifactPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::Artifacts {
+                artifacts,
+                next_cursor,
+            } => Ok(ArtifactPage {
+                artifacts,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("artifacts")),
+        }
+    }
+
+    pub async fn get_artifact(&self, artifact: ArtifactId) -> Result<ArtifactSummary, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::GetArtifact { artifact })
+            .await?
+        {
+            WireResponse::Artifact { artifact } => Ok(artifact),
+            _ => Err(unexpected_response("artifact")),
+        }
+    }
+
+    pub async fn begin_artifact_download(
+        &self,
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+        chunk_bytes: u64,
+    ) -> Result<ArtifactDownload, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink {
+                artifact: artifact.clone(),
+                max_bytes,
+                ttl_seconds,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload {
+                client: self.clone(),
+                artifact,
+                max_bytes,
+                chunk_bytes,
+                token_digest: link.scoped_token_digest.clone(),
+                link,
+                expected_offset: 0,
+                finished: false,
+            }),
+            _ => Err(unexpected_response("artifact_download_link")),
+        }
+    }
+
+    async fn send_authenticated(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<WireResponse, ClientError> {
+        let session_secret = self
+            .session_secret
+            .lock()
+            .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))?
+            .clone()
+            .ok_or_else(|| {
+                ClientError::Api(ApiError::from_message(
+                    "client",
+                    "no authenticated Clusterflux session is configured",
+                ))
+            })?;
+        self.send(
+            CONTROL_API_PATH,
+            json!({
+                "type": "authenticated",
+                "session_secret": session_secret,
+                "request": request,
+            }),
+        )
+        .await
+    }
+
+    async fn send_login(&self, request: LoginRequest) -> Result<WireResponse, ClientError> {
+        let payload = serde_json::to_value(request)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        self.send(LOGIN_API_PATH, payload).await
+    }
+
+    async fn send(
+        &self,
+        api_path: &str,
+        payload: serde_json::Value,
+    ) -> Result<WireResponse, ClientError> {
+        let request_number = self.next_request.fetch_add(1, Ordering::Relaxed);
+        let request_id = format!("client-{request_number}");
+        let envelope = coordinator_wire_request(&request_id, payload);
+        let body = serde_json::to_vec(&envelope)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        let response = self
+            .transport
+            .send(TransportRequest {
+                api_path: api_path.to_owned(),
+                body,
+            })
+            .await?;
+        let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| {
+            ClientError::Protocol(format!("decode typed API response: {error}"))
+        })?;
+        match response {
+            WireResponse::Error {
+                code,
+                category,
+                message,
+                retryable,
+                request_id: response_request_id,
+            } => {
+                if response_request_id != request_id {
+                    return Err(ClientError::Protocol(format!(
+                        "error response request_id {response_request_id} does not match {request_id}"
+                    )));
+                }
+                Err(ClientError::Api(ApiError::new(
+                    code,
+                    category,
+                    message,
+                    retryable,
+                    response_request_id,
+                )))
+            }
+            response => Ok(response),
+        }
+    }
+}
+
+pub struct ArtifactDownload {
+    client: ClusterfluxClient,
+    artifact: ArtifactId,
+    max_bytes: u64,
+    chunk_bytes: u64,
+    token_digest: Digest,
+    link: clusterflux_core::DownloadLink,
+    expected_offset: u64,
+    finished: bool,
+}
+
+impl ArtifactDownload {
+    pub fn link(&self) -> &clusterflux_core::DownloadLink {
+        &self.link
+    }
+
+    pub async fn next_chunk(&mut self) -> Result<ArtifactDownloadPoll, ClientError> {
+        if self.finished {
+            return Ok(ArtifactDownloadPoll::Chunk {
+                offset: self.expected_offset,
+                bytes: Vec::new(),
+                eof: true,
+            });
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream {
+                artifact: self.artifact.clone(),
+                max_bytes: self.max_bytes,
+                token_digest: self.token_digest.clone(),
+                chunk_bytes: self.chunk_bytes,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadStream {
+                content_bytes_available,
+                content_offset,
+                content_eof,
+                content_base64,
+                ..
+            } => {
+                if !content_bytes_available {
+                    return Ok(ArtifactDownloadPoll::Pending);
+                }
+                let offset = content_offset.ok_or_else(|| {
+                    ClientError::Protocol(
+                        "artifact response contained bytes without an offset".to_owned(),
+                    )
+                })?;
+                if offset != self.expected_offset {
+                    return Err(ClientError::Protocol(format!(
+                        "artifact chunk offset {offset} does not match expected {}",
+                        self.expected_offset
+                    )));
+                }
+                let bytes = BASE64_STANDARD
+                    .decode(content_base64.unwrap_or_default())
+                    .map_err(|error| {
+                        ClientError::Protocol(format!(
+                            "artifact response contains invalid base64: {error}"
+                        ))
+                    })?;
+                self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64);
+                self.finished = content_eof;
+                Ok(ArtifactDownloadPoll::Chunk {
+                    offset,
+                    bytes,
+                    eof: content_eof,
+                })
+            }
+            _ => Err(unexpected_response("artifact_download_stream")),
+        }
+    }
+
+    pub async fn cancel(mut self) -> Result<(), ClientError> {
+        if self.finished {
+            return Ok(());
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink {
+                artifact: self.artifact.clone(),
+                token_digest: self.token_digest.clone(),
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLinkRevoked {} => {
+                self.finished = true;
+                Ok(())
+            }
+            _ => Err(unexpected_response("artifact_download_link_revoked")),
+        }
+    }
+}
+
+fn unexpected_response(expected: &str) -> ClientError {
+    ClientError::Protocol(format!(
+        "expected typed response {expected}, received another response variant"
+    ))
+}

crates/clusterflux-client/src/protocol.rs

@@ -0,0 +1,403 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink,
+    LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId,
+};
+use serde::{Deserialize, Serialize};
+
+use crate::types::*;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum AuthenticatedRequest {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject {
+        project: ProjectId,
+        name: String,
+    },
+    SelectProject {
+        project: ProjectId,
+    },
+    ListProjects,
+    RegisterAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    ListAgentPublicKeys,
+    RotateAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        agent: AgentId,
+    },
+    CreateNodeEnrollmentGrant {
+        ttl_seconds: u64,
+    },
+    ListNodeSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    RevokeNodeCredential {
+        node: NodeId,
+    },
+    ListProcessSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    CancelProcess {
+        process: ProcessId,
+    },
+    AbortProcess {
+        process: ProcessId,
+        launch_attempt: Option<String>,
+    },
+    QuotaStatus,
+    ListTaskEvents {
+        process: Option<ProcessId>,
+    },
+    ListTaskSnapshots {
+        process: ProcessId,
+    },
+    ListRecentLogs {
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    },
+    RestartTask {
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+    },
+    CreateDebugEpoch {
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    ListArtifacts {
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    },
+    GetArtifact {
+        artifact: ArtifactId,
+    },
+    CreateArtifactDownloadLink {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        artifact: ArtifactId,
+        token_digest: Digest,
+    },
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum LoginRequest {
+    BeginWebBrowserLogin {},
+    CancelWebBrowserLogin {
+        transaction_id: String,
+    },
+    ExchangeWebLoginHandoff {
+        transaction_id: String,
+        handoff_code: String,
+    },
+}
+
+#[derive(Clone, Deserialize)]
+pub(crate) struct WireBrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub session_secret: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum WireResponse {
+    AuthStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        authenticated: bool,
+        account_status: String,
+        suspended: bool,
+        disabled: bool,
+        deleted: bool,
+        manual_review: bool,
+        sanitized_reason: Option<String>,
+        next_actions: Vec<String>,
+    },
+    CliSessionRevoked {},
+    ProjectCreated {
+        project: Project,
+    },
+    ProjectSelected {
+        project: Project,
+    },
+    Projects {
+        projects: Vec<Project>,
+    },
+    AgentPublicKey {
+        record: AgentPublicKey,
+    },
+    AgentPublicKeys {
+        records: Vec<AgentPublicKey>,
+    },
+    NodeEnrollmentGrantCreated {
+        tenant: TenantId,
+        project: ProjectId,
+        grant: String,
+        scope: String,
+        expires_at_epoch_seconds: u64,
+    },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+    },
+    NodeCredentialRevoked {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        descriptor_removed: bool,
+        queued_assignments_removed: usize,
+    },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+    },
+    ProcessCancellationRequested {
+        process: ProcessId,
+        cancelled_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessAborted {
+        process: ProcessId,
+        aborted_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    QuotaStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        policy_label: Option<String>,
+        limits: ResourceLimits,
+        window_seconds: BTreeMap<LimitKind, u64>,
+        usage: BTreeMap<LimitKind, u64>,
+        window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+    },
+    TaskEvents {
+        events: Vec<TaskCompletionEvent>,
+    },
+    TaskSnapshots {
+        snapshots: Vec<TaskAttemptSnapshot>,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
+    TaskRestart {
+        process: ProcessId,
+        task: TaskInstanceId,
+        restarted_task_instance: Option<TaskInstanceId>,
+        restarted_attempt_id: Option<String>,
+        actor: UserId,
+        accepted: bool,
+        clean_boundary_available: bool,
+        active_task: bool,
+        completed_event_observed: bool,
+        requires_whole_process_restart: bool,
+        message: String,
+        audit_event: DebugAuditEvent,
+    },
+    TaskFailureResolved {
+        process: ProcessId,
+        task: TaskInstanceId,
+        attempt_id: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+        actor: UserId,
+        authorization: Authorization,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpoch {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        affected_tasks: Vec<TaskCancellationTarget>,
+        all_stop_requested: bool,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpochStatus {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        expected_tasks: Vec<TaskCancellationTarget>,
+        acknowledgements: Vec<DebugParticipantAcknowledgement>,
+        fully_frozen: bool,
+        partially_frozen: bool,
+        fully_resumed: bool,
+        failed: bool,
+        failure_messages: Vec<String>,
+        audit_event: DebugAuditEvent,
+    },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
+    ArtifactDownloadLink {
+        link: DownloadLink,
+    },
+    ArtifactDownloadLinkRevoked {},
+    ArtifactDownloadStream {
+        #[serde(rename = "link")]
+        _link: DownloadLink,
+        content_bytes_available: bool,
+        content_offset: Option<u64>,
+        content_eof: bool,
+        content_base64: Option<String>,
+    },
+    WebBrowserLoginStarted {
+        transaction_id: String,
+        authorization_url: String,
+        expires_at_epoch_seconds: u64,
+    },
+    WebBrowserLoginCancelled {},
+    WebBrowserSession {
+        session: WireBrowserSession,
+    },
+    Error {
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: String,
+        retryable: bool,
+        request_id: String,
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use serde::Deserialize;
+    use serde_json::Value;
+
+    use super::*;
+
+    #[derive(Deserialize)]
+    struct OperationFixture {
+        operation: String,
+        boundary: String,
+        request: Value,
+        response: Value,
+    }
+
+    #[test]
+    fn website_operation_contract_fixtures_cover_every_typed_request() {
+        let fixtures: Vec<OperationFixture> =
+            serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap();
+        let expected = BTreeSet::from([
+            "abort_process",
+            "auth_status",
+            "begin_web_browser_login",
+            "cancel_web_browser_login",
+            "cancel_process",
+            "create_artifact_download_link",
+            "create_debug_epoch",
+            "create_node_enrollment_grant",
+            "create_project",
+            "debug_attach",
+            "exchange_web_login_handoff",
+            "get_artifact",
+            "inspect_debug_epoch",
+            "list_agent_public_keys",
+            "list_artifacts",
+            "list_node_summaries",
+            "list_process_summaries",
+            "list_projects",
+            "list_recent_logs",
+            "list_task_events",
+            "list_task_snapshots",
+            "open_artifact_download_stream",
+            "quota_status",
+            "register_agent_public_key",
+            "resolve_task_failure",
+            "restart_task",
+            "resume_debug_epoch",
+            "revoke_agent_public_key",
+            "revoke_artifact_download_link",
+            "revoke_cli_session",
+            "revoke_node_credential",
+            "rotate_agent_public_key",
+            "select_project",
+        ])
+        .into_iter()
+        .map(str::to_owned)
+        .collect();
+        let mut observed = BTreeSet::new();
+
+        for fixture in fixtures {
+            assert_eq!(fixture.request["type"], fixture.operation);
+            let round_trip = match fixture.boundary.as_str() {
+                "control" => {
+                    let typed: AuthenticatedRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                "login" => {
+                    let typed: LoginRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                boundary => panic!("unknown fixture boundary {boundary}"),
+            };
+            assert_eq!(round_trip, fixture.request);
+            let response: WireResponse = serde_json::from_value(fixture.response).unwrap();
+            assert!(
+                !matches!(response, WireResponse::Error { .. }),
+                "{} must carry its operation-specific success response fixture",
+                fixture.operation
+            );
+            assert!(observed.insert(fixture.operation));
+        }
+        assert_eq!(observed, expected);
+    }
+}

crates/clusterflux-client/src/transport.rs

@@ -0,0 +1,250 @@
+use std::collections::{BTreeMap, VecDeque};
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use clusterflux_control::{endpoint_identity, ControlSession};
+use thiserror::Error;
+
+pub type TransportFuture =
+    Pin<Box<dyn Future<Output = Result<TransportResponse, ClientTransportError>> + Send + 'static>>;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportRequest {
+    pub api_path: String,
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportResponse {
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ClientTransportError {
+    #[error("client transport failed: {0}")]
+    Failed(String),
+    #[error("client transport task failed: {0}")]
+    Task(String),
+}
+
+pub trait ClientTransport: Send + Sync + 'static {
+    fn send(&self, request: TransportRequest) -> TransportFuture;
+}
+
+type ControlSessionPool = Arc<Vec<Mutex<Option<ControlSession>>>>;
+
+pub struct ControlTransport {
+    endpoint: String,
+    connect_timeout: Duration,
+    io_timeout: Duration,
+    sessions: Arc<Mutex<BTreeMap<String, ControlSessionPool>>>,
+    next_session: Arc<AtomicU64>,
+}
+
+// A small pool allows independent HTMX reads to progress concurrently while
+// keeping connection and blocking-worker use strictly bounded.
+const SESSIONS_PER_API_PATH: usize = 4;
+
+impl ControlTransport {
+    pub fn new(endpoint: impl Into<String>) -> Result<Self, ClientTransportError> {
+        Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
+    }
+
+    pub fn with_timeouts(
+        endpoint: impl Into<String>,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ClientTransportError> {
+        let endpoint = endpoint.into();
+        endpoint_identity(&endpoint)
+            .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+        Ok(Self {
+            endpoint,
+            connect_timeout,
+            io_timeout,
+            sessions: Arc::new(Mutex::new(BTreeMap::new())),
+            next_session: Arc::new(AtomicU64::new(0)),
+        })
+    }
+}
+
+impl ClientTransport for ControlTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let endpoint = self.endpoint.clone();
+        let connect_timeout = self.connect_timeout;
+        let io_timeout = self.io_timeout;
+        let sessions = Arc::clone(&self.sessions);
+        let next_session = Arc::clone(&self.next_session);
+        Box::pin(async move {
+            let pool = {
+                let mut sessions = sessions.lock().map_err(|_| {
+                    ClientTransportError::Failed(
+                        "client transport pool lock was poisoned".to_owned(),
+                    )
+                })?;
+                Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| {
+                    Arc::new(
+                        (0..SESSIONS_PER_API_PATH)
+                            .map(|_| Mutex::new(None))
+                            .collect(),
+                    )
+                }))
+            };
+            let slot_index =
+                next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH;
+            tokio::task::spawn_blocking(move || {
+                let value = serde_json::from_slice(&request.body)
+                    .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+                let mut selected = None;
+                for offset in 0..SESSIONS_PER_API_PATH {
+                    let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                    match pool[index].try_lock() {
+                        Ok(session) if session.is_some() => {
+                            selected = Some(session);
+                            break;
+                        }
+                        Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {}
+                        Err(std::sync::TryLockError::Poisoned(_)) => {
+                            return Err(ClientTransportError::Failed(
+                                "client transport session lock was poisoned".to_owned(),
+                            ));
+                        }
+                    }
+                }
+                if selected.is_none() {
+                    for offset in 0..SESSIONS_PER_API_PATH {
+                        let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                        match pool[index].try_lock() {
+                            Ok(session) => {
+                                selected = Some(session);
+                                break;
+                            }
+                            Err(std::sync::TryLockError::WouldBlock) => {}
+                            Err(std::sync::TryLockError::Poisoned(_)) => {
+                                return Err(ClientTransportError::Failed(
+                                    "client transport session lock was poisoned".to_owned(),
+                                ));
+                            }
+                        }
+                    }
+                }
+                let mut session = match selected {
+                    Some(session) => session,
+                    None => pool[slot_index].lock().map_err(|_| {
+                        ClientTransportError::Failed(
+                            "client transport session lock was poisoned".to_owned(),
+                        )
+                    })?,
+                };
+                if session.is_none() {
+                    *session = Some(
+                        ControlSession::connect_to_api_path_with_timeouts(
+                            &endpoint,
+                            &request.api_path,
+                            connect_timeout,
+                            io_timeout,
+                        )
+                        .map_err(|error| ClientTransportError::Failed(error.to_string()))?,
+                    );
+                }
+                let response = session
+                    .as_mut()
+                    .expect("session was initialized for the requested API path")
+                    .request(&value);
+                match response {
+                    Ok(response) => serde_json::to_vec(&response)
+                        .map(|body| TransportResponse { body })
+                        .map_err(|error| ClientTransportError::Failed(error.to_string())),
+                    Err(error) => {
+                        *session = None;
+                        Err(ClientTransportError::Failed(error.to_string()))
+                    }
+                }
+            })
+            .await
+            .map_err(|error| ClientTransportError::Task(error.to_string()))?
+        })
+    }
+}
+
+#[derive(Clone, Default)]
+pub struct MockTransport {
+    state: Arc<Mutex<MockTransportState>>,
+}
+
+#[derive(Default)]
+struct MockTransportState {
+    responses: VecDeque<Result<Vec<u8>, ClientTransportError>>,
+    requests: Vec<TransportRequest>,
+}
+
+impl MockTransport {
+    pub fn from_json_responses(responses: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        let responses = responses
+            .into_iter()
+            .map(|response| Ok(response.into().into_bytes()))
+            .collect();
+        Self {
+            state: Arc::new(Mutex::new(MockTransportState {
+                responses,
+                requests: Vec::new(),
+            })),
+        }
+    }
+
+    pub fn push_json_response(&self, response: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Ok(response.into().into_bytes()));
+    }
+
+    pub fn push_error(&self, message: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Err(ClientTransportError::Failed(message.into())));
+    }
+
+    pub fn request_bodies(&self) -> Vec<String> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .iter()
+            .map(|request| String::from_utf8_lossy(&request.body).into_owned())
+            .collect()
+    }
+
+    pub fn requests(&self) -> Vec<TransportRequest> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .clone()
+    }
+}
+
+impl ClientTransport for MockTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let state = Arc::clone(&self.state);
+        Box::pin(async move {
+            let mut state = state.lock().map_err(|_| {
+                ClientTransportError::Failed("mock transport lock was poisoned".to_owned())
+            })?;
+            state.requests.push(request);
+            state
+                .responses
+                .pop_front()
+                .ok_or_else(|| {
+                    ClientTransportError::Failed("mock transport has no queued response".to_owned())
+                })?
+                .map(|body| TransportResponse { body })
+        })
+    }
+}

crates/clusterflux-client/src/types.rs

@@ -0,0 +1,468 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement,
+    ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy,
+    TaskInstanceId, TenantId, UserId, VfsPath,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AccountStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub authenticated: bool,
+    pub account_status: String,
+    pub suspended: bool,
+    pub disabled: bool,
+    pub deleted: bool,
+    pub manual_review: bool,
+    pub sanitized_reason: Option<String>,
+    pub next_actions: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Project {
+    pub id: ProjectId,
+    pub tenant: TenantId,
+    pub name: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AgentPublicKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub agent: AgentId,
+    pub public_key: String,
+    pub public_key_fingerprint: Digest,
+    pub version: u64,
+    pub revoked: bool,
+    pub scopes: Vec<String>,
+    pub human_account_creation_privilege: bool,
+    pub browser_interaction_required_each_run: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeEnrollmentGrant {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub grant: String,
+    pub scope: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodePage {
+    pub nodes: Vec<NodeSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeRevocation {
+    pub node: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub descriptor_removed: bool,
+    pub queued_assignments_removed: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessPage {
+    pub processes: Vec<ProcessSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskTerminalState {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskExecutor {
+    CoordinatorMain,
+    Node,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCompletionEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub node: NodeId,
+    pub executor: TaskExecutor,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub attempt_id: Option<String>,
+    pub placement: Option<Placement>,
+    pub terminal_state: TaskTerminalState,
+    pub status_code: Option<i32>,
+    pub stdout_bytes: u64,
+    pub stderr_bytes: u64,
+    pub stdout_tail: String,
+    pub stderr_tail: String,
+    pub stdout_truncated: bool,
+    pub stderr_truncated: bool,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub result: Option<TaskBoundaryValue>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskAttemptState {
+    Queued,
+    Running,
+    FailedAwaitingAction,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAttemptSnapshot {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub attempt_number: u32,
+    pub task_definition: TaskDefinitionId,
+    pub display_name: String,
+    pub state: TaskAttemptState,
+    pub current: bool,
+    pub node: Option<NodeId>,
+    pub environment_id: Option<String>,
+    pub environment_digest: Option<Digest>,
+    pub argument_summary: Vec<String>,
+    pub handle_summary: Vec<String>,
+    pub command_state: Option<String>,
+    pub vfs_checkpoint: String,
+    pub probe_symbol: Option<String>,
+    pub source_path: Option<String>,
+    pub source_line: Option<u32>,
+    pub restart_compatible: bool,
+    pub failure_policy: TaskFailurePolicy,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub status_code: Option<i32>,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogPage {
+    pub entries: Vec<RecentLogEntry>,
+    pub next_sequence: Option<u64>,
+    pub history_truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactPage {
+    pub artifacts: Vec<ArtifactSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct QuotaStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub policy_label: Option<String>,
+    pub limits: ResourceLimits,
+    pub window_seconds: BTreeMap<LimitKind, u64>,
+    pub usage: BTreeMap<LimitKind, u64>,
+    pub window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCancellationTarget {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessCancellation {
+    pub process: ProcessId,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub affected_nodes: Vec<NodeId>,
+    pub aborted: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAuditEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+    pub actor: UserId,
+    pub operation: String,
+    pub allowed: bool,
+    pub reason: String,
+    pub charged_debug_read_bytes: u64,
+    pub used_debug_read_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAttach {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub authorization: Authorization,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochControl {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub all_stop_requested: bool,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DebugAcknowledgementState {
+    Frozen,
+    Running,
+    Failed,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipantAcknowledgement {
+    pub node: NodeId,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub epoch: u64,
+    pub state: DebugAcknowledgementState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochStatus {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub expected_tasks: Vec<TaskCancellationTarget>,
+    pub acknowledgements: Vec<DebugParticipantAcknowledgement>,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+    pub failure_messages: Vec<String>,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskRestart {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub restarted_task_instance: Option<TaskInstanceId>,
+    pub restarted_attempt_id: Option<String>,
+    pub actor: UserId,
+    pub accepted: bool,
+    pub clean_boundary_available: bool,
+    pub active_task: bool,
+    pub completed_event_observed: bool,
+    pub requires_whole_process_restart: bool,
+    pub message: String,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskFailureResolutionResult {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub resolution: TaskFailureResolution,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskFailureResolution {
+    AcceptFailure,
+    Cancel,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskReplacementBundle {
+    pub bundle_digest: Digest,
+    pub wasm_module_base64: String,
+    pub source_snapshot: Option<Digest>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BrowserLoginStart {
+    pub transaction_id: String,
+    pub authorization_url: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct SessionCredential(pub(crate) String);
+
+impl SessionCredential {
+    pub fn from_secret(secret: impl Into<String>) -> Self {
+        Self(secret.into())
+    }
+
+    pub fn expose_secret(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Debug for SessionCredential {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("SessionCredential([REDACTED])")
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct BrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub credential: SessionCredential,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ArtifactDownloadPoll {
+    Pending,
+    Chunk {
+        offset: u64,
+        bytes: Vec<u8>,
+        eof: bool,
+    },
+}

crates/clusterflux-client/tests/client_contract.rs

@@ -0,0 +1,254 @@
+use std::net::TcpListener;
+use std::time::Duration;
+
+use clusterflux_client::{
+    ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError,
+    ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId,
+    UserId, CLIENT_API_VERSION,
+};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_coordinator::service::CoordinatorService;
+use serde_json::{json, Value};
+
+#[tokio::test]
+async fn mock_transport_exercises_typed_envelope_and_session_plumbing() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "projects",
+        "projects": [{
+            "id": "project-one",
+            "tenant": "tenant-one",
+            "name": "Project one"
+        }],
+        "actor": "user-one"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport.clone())
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, CONTROL_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION);
+    assert_eq!(envelope["request_id"], "client-1");
+    assert_eq!(envelope["operation"], "authenticated");
+    assert_eq!(envelope["payload"]["type"], "authenticated");
+    assert_eq!(envelope["payload"]["request"]["type"], "list_projects");
+    assert_eq!(envelope["payload"]["session_secret"], "test-session-secret");
+}
+
+#[tokio::test]
+async fn structured_errors_retain_machine_fields_and_originating_request_id() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "account_suspended",
+        "category": "authorization",
+        "message": "account access is suspended",
+        "retryable": false,
+        "request_id": "client-1"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Api(error) = client.account_status().await.unwrap_err() else {
+        panic!("expected typed API error");
+    };
+    assert_eq!(error.code, ApiErrorCode::AccountSuspended);
+    assert_eq!(error.category, ApiErrorCategory::Authorization);
+    assert_eq!(error.request_id, "client-1");
+    assert!(!error.retryable);
+}
+
+#[tokio::test]
+async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() {
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "web_browser_login_cancelled" }).to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport.clone());
+
+    client
+        .cancel_browser_login("login-transaction")
+        .await
+        .unwrap();
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, LOGIN_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["operation"], "cancel_web_browser_login");
+    assert_eq!(
+        envelope["payload"],
+        json!({
+            "type": "cancel_web_browser_login",
+            "transaction_id": "login-transaction"
+        })
+    );
+}
+
+#[tokio::test]
+async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "validation_error",
+        "category": "validation",
+        "message": "bad request",
+        "retryable": false,
+        "request_id": "another-request"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else {
+        panic!("expected protocol error");
+    };
+    assert!(message.contains("does not match client-1"));
+}
+
+#[test]
+fn session_credentials_are_redacted_from_debug_output() {
+    let credential = SessionCredential::from_secret("must-not-appear");
+    assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])");
+}
+
+#[tokio::test]
+async fn artifact_download_is_a_typed_bounded_chunk_stream() {
+    let link = json!({
+        "artifact": "artifact-one",
+        "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        "artifact_size_bytes": 5,
+        "source": { "RetainedNode": "node-one" },
+        "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one",
+        "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+        "expires_at_epoch_seconds": 1000,
+        "tenant": "tenant-one",
+        "project": "project-one",
+        "process": "process-one",
+        "actor": { "User": "user-one" },
+        "max_bytes": 1024,
+        "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+    });
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link.clone(),
+            "streamed_bytes": 0,
+            "charged_download_bytes": 0,
+            "content_bytes_available": false,
+            "content_eof": false
+        })
+        .to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link,
+            "streamed_bytes": 5,
+            "charged_download_bytes": 5,
+            "content_bytes_available": true,
+            "content_offset": 0,
+            "content_eof": true,
+            "content_base64": "aGVsbG8="
+        })
+        .to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+    let mut download = client
+        .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64)
+        .await
+        .unwrap();
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Pending
+    );
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Chunk {
+            offset: 0,
+            bytes: b"hello".to_vec(),
+            eof: true,
+        }
+    );
+}
+
+#[tokio::test]
+async fn typed_client_runs_against_the_real_strict_control_endpoint() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let mut service = CoordinatorService::new(7);
+        service
+            .issue_cli_session(
+                TenantId::from("tenant-one"),
+                ProjectId::from("project-one"),
+                UserId::from("user-one"),
+                "real-endpoint-session",
+                None,
+            )
+            .unwrap();
+        let (stream, _) = listener.accept().unwrap();
+        service.handle_stream(stream).unwrap();
+    });
+
+    let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}"))
+        .unwrap()
+        .with_session_credential(&SessionCredential::from_secret("real-endpoint-session"));
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+    let status = client.account_status().await.unwrap();
+    assert!(status.authenticated);
+    assert_eq!(status.actor, UserId::from("user-one"));
+
+    drop(client);
+    server.join().unwrap();
+}
+
+#[tokio::test]
+async fn concurrent_requests_expand_the_bounded_pool_without_serializing() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let handlers = (0..2)
+            .map(|_| {
+                let (stream, _) = listener.accept().unwrap();
+                std::thread::spawn(move || {
+                    let mut service = CoordinatorService::new(7);
+                    service
+                        .issue_cli_session(
+                            TenantId::from("tenant-one"),
+                            ProjectId::from("project-one"),
+                            UserId::from("user-one"),
+                            "concurrent-session",
+                            None,
+                        )
+                        .unwrap();
+                    service.handle_stream(stream).unwrap();
+                })
+            })
+            .collect::<Vec<_>>();
+        for handler in handlers {
+            handler.join().unwrap();
+        }
+    });
+
+    let transport = ControlTransport::with_timeouts(
+        format!("clusterflux+tcp://{address}"),
+        Duration::from_secs(2),
+        Duration::from_secs(5),
+    )
+    .unwrap();
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("concurrent-session"));
+    let (first, second) = tokio::join!(client.account_status(), client.account_status());
+    assert!(first.unwrap().authenticated);
+    assert!(second.unwrap().authenticated);
+
+    drop(client);
+    server.join().unwrap();
+}

crates/clusterflux-client/tests/fixtures/web_operations.json

@@ -0,0 +1,200 @@
+[
+  {
+    "operation": "begin_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "begin_web_browser_login" },
+    "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "cancel_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" },
+    "response": { "type": "web_browser_login_cancelled" }
+  },
+  {
+    "operation": "exchange_web_login_handoff",
+    "boundary": "login",
+    "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" },
+    "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } }
+  },
+  {
+    "operation": "auth_status",
+    "boundary": "control",
+    "request": { "type": "auth_status" },
+    "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] }
+  },
+  {
+    "operation": "revoke_cli_session",
+    "boundary": "control",
+    "request": { "type": "revoke_cli_session" },
+    "response": { "type": "cli_session_revoked" }
+  },
+  {
+    "operation": "create_project",
+    "boundary": "control",
+    "request": { "type": "create_project", "project": "project-new", "name": "New project" },
+    "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } }
+  },
+  {
+    "operation": "select_project",
+    "boundary": "control",
+    "request": { "type": "select_project", "project": "project-selected" },
+    "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } }
+  },
+  {
+    "operation": "list_projects",
+    "boundary": "control",
+    "request": { "type": "list_projects" },
+    "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] }
+  },
+  {
+    "operation": "register_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "list_agent_public_keys",
+    "boundary": "control",
+    "request": { "type": "list_agent_public_keys" },
+    "response": { "type": "agent_public_keys", "records": [] }
+  },
+  {
+    "operation": "rotate_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "revoke_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "create_node_enrollment_grant",
+    "boundary": "control",
+    "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 },
+    "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "list_node_summaries",
+    "boundary": "control",
+    "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 },
+    "response": { "type": "node_summaries", "nodes": [], "next_cursor": null }
+  },
+  {
+    "operation": "revoke_node_credential",
+    "boundary": "control",
+    "request": { "type": "revoke_node_credential", "node": "node-one" },
+    "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 }
+  },
+  {
+    "operation": "list_process_summaries",
+    "boundary": "control",
+    "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 },
+    "response": { "type": "process_summaries", "processes": [], "next_cursor": null }
+  },
+  {
+    "operation": "cancel_process",
+    "boundary": "control",
+    "request": { "type": "cancel_process", "process": "process-one" },
+    "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "abort_process",
+    "boundary": "control",
+    "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null },
+    "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "quota_status",
+    "boundary": "control",
+    "request": { "type": "quota_status" },
+    "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } }
+  },
+  {
+    "operation": "list_task_events",
+    "boundary": "control",
+    "request": { "type": "list_task_events", "process": "process-one" },
+    "response": { "type": "task_events", "events": [] }
+  },
+  {
+    "operation": "list_task_snapshots",
+    "boundary": "control",
+    "request": { "type": "list_task_snapshots", "process": "process-one" },
+    "response": { "type": "task_snapshots", "snapshots": [] }
+  },
+  {
+    "operation": "list_recent_logs",
+    "boundary": "control",
+    "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 },
+    "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false }
+  },
+  {
+    "operation": "restart_task",
+    "boundary": "control",
+    "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null },
+    "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resolve_task_failure",
+    "boundary": "control",
+    "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" },
+    "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" }
+  },
+  {
+    "operation": "debug_attach",
+    "boundary": "control",
+    "request": { "type": "debug_attach", "process": "process-one" },
+    "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "create_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resume_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "inspect_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "list_artifacts",
+    "boundary": "control",
+    "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 },
+    "response": { "type": "artifacts", "artifacts": [], "next_cursor": null }
+  },
+  {
+    "operation": "get_artifact",
+    "boundary": "control",
+    "request": { "type": "get_artifact", "artifact": "artifact-one" },
+    "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } }
+  },
+  {
+    "operation": "create_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 },
+    "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } }
+  },
+  {
+    "operation": "open_artifact_download_stream",
+    "boundary": "control",
+    "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 },
+    "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" }
+  },
+  {
+    "operation": "revoke_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
+    "response": { "type": "artifact_download_link_revoked" }
+  }
+]

crates/clusterflux-control/src/lib.rs

@@ -61,12 +61,34 @@ impl ControlSession {
         endpoint: &str,
         api_path: &str,
     ) -> Result<Self, ControlTransportError> {
-        let mut session = Self::connect(endpoint)?;
+        Self::connect_to_api_path_with_timeouts(
+            endpoint,
+            api_path,
+            Duration::from_secs(10),
+            Duration::from_secs(30),
+        )
+    }
+
+    pub fn connect_to_api_path_with_timeouts(
+        endpoint: &str,
+        api_path: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ControlTransportError> {
+        let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?;
         #[cfg(not(target_arch = "wasm32"))]
-        if let ControlTransport::Https { url, .. } = &mut session.transport {
+        {
-            *url = endpoint_api_url(endpoint, api_path)?;
+            let mut session = session;
+            if let ControlTransport::Https { url, .. } = &mut session.transport {
+                *url = endpoint_api_url(endpoint, api_path)?;
+            }
+            Ok(session)
+        }
+        #[cfg(target_arch = "wasm32")]
+        {
+            let _ = api_path;
+            Ok(session)
         }
-        Ok(session)
     }
 
     pub fn connect_with_timeouts(

crates/clusterflux-coordinator/src/durable.rs

@@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
     pub enrollment_scope: String,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct NodeScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub node: NodeId,
+}
+
+impl NodeScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
+        Self {
+            tenant,
+            project,
+            node,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
+        Self::new(tenant.clone(), project.clone(), node.clone())
+    }
+
+    pub fn credential_subject(&self) -> String {
+        format!(
+            "node:{}:{}:{}:{}:{}:{}",
+            self.tenant.as_str().len(),
+            self.tenant,
+            self.project.as_str().len(),
+            self.project,
+            self.node.as_str().len(),
+            self.node
+        )
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct CredentialRecord {
     pub subject: String,
@@ -107,7 +140,7 @@ pub struct DurableState {
     pub tenants: BTreeMap<TenantId, TenantRecord>,
     pub users: BTreeMap<UserId, UserRecord>,
     pub projects: BTreeMap<ProjectId, ProjectRecord>,
-    pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
+    pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
     pub credentials: BTreeMap<String, CredentialRecord>,
     pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
     pub source_provider_configs:

crates/clusterflux-coordinator/src/lib.rs

@@ -13,7 +13,7 @@ pub mod service;
 mod sessions;
 pub use durable::{
     AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
-    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
+    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
     ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
     TenantRecord, UserRecord,
 };
@@ -33,6 +33,7 @@ pub use service::{
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ActiveProcess {
     pub id: ProcessId,
+    pub launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
     pub tenant: TenantId,
     pub project: ProjectId,
     pub connected_nodes: BTreeSet<NodeId>,
@@ -127,8 +128,9 @@ impl Coordinator {
         public_key: impl Into<String>,
         enrollment_scope: impl Into<String>,
     ) {
+        let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
         self.durable.node_identities.insert(
-            node.clone(),
+            key,
             NodeIdentityRecord {
                 id: node,
                 tenant,
@@ -180,10 +182,13 @@ impl Coordinator {
             public_key,
             credential.scope.clone(),
         );
+        let subject =
+            NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
+                .credential_subject();
         self.durable.credentials.insert(
-            format!("node:{node}"),
+            subject.clone(),
             CredentialRecord {
-                subject: format!("node:{node}"),
+                subject,
                 tenant: credential.tenant.clone(),
                 project: Some(credential.project.clone()),
                 kind: credential.credential_kind.clone(),
@@ -322,9 +327,20 @@ impl Coordinator {
         tenant: TenantId,
         project: ProjectId,
         id: ProcessId,
+    ) -> ActiveProcess {
+        self.start_process_for_launch_attempt(tenant, project, id, None)
+    }
+
+    pub fn start_process_for_launch_attempt(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        id: ProcessId,
+        launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
     ) -> ActiveProcess {
         let process = ActiveProcess {
             id: id.clone(),
+            launch_attempt,
             tenant,
             project,
             connected_nodes: BTreeSet::new(),
@@ -344,15 +360,12 @@ impl Coordinator {
         project: &ProjectId,
         process: &ProcessId,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
-            .ok_or(CoordinatorError::UnknownNode)?;
+        {
-        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::UnknownNode);
-            return Err(CoordinatorError::Unauthorized(
-                "node identity is outside the requested tenant/project scope".to_owned(),
-            ));
         }
         if !self
             .active_processes
@@ -367,14 +380,18 @@ impl Coordinator {
 
     pub fn reconnect_node(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         process: Option<(&ProcessId, u64)>,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
-            .ok_or(CoordinatorError::UnknownNode)?;
+        {
+            return Err(CoordinatorError::UnknownNode);
+        }
 
         if let Some((process_id, stale_epoch)) = process {
             if stale_epoch != self.coordinator_epoch {
@@ -383,11 +400,7 @@ impl Coordinator {
                     current_epoch: self.coordinator_epoch,
                 });
             }
-            let key = (
+            let key = (tenant.clone(), project.clone(), process_id.clone());
-                identity.tenant.clone(),
-                identity.project.clone(),
-                process_id.clone(),
-            );
             if let Some(active) = self.active_processes.get_mut(&key) {
                 active.connected_nodes.insert(node.clone());
             }
@@ -404,23 +417,27 @@ impl Coordinator {
         let identity = self
             .durable
             .node_identities
-            .get(node)
+            .get(&NodeScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                node,
+            ))
             .ok_or(CoordinatorError::UnknownNode)?
             .clone();
-        if identity.tenant != context.tenant || identity.project != context.project {
-            return Err(CoordinatorError::Unauthorized(
-                "node credential is outside the signed-in tenant/project scope".to_owned(),
-            ));
-        }
         if !matches!(context.actor, Actor::User(_)) {
             return Err(CoordinatorError::Unauthorized(
                 "node credential revocation requires a user identity".to_owned(),
             ));
         }
-        self.durable.node_identities.remove(node);
+        let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
-        self.durable.credentials.remove(&format!("node:{node}"));
+        self.durable.node_identities.remove(&key);
-        for active in self.active_processes.values_mut() {
+        self.durable.credentials.remove(&key.credential_subject());
-            active.connected_nodes.remove(node);
+        for active in self
+            .active_processes
+            .values_mut()
+            .filter(|active| active.tenant == context.tenant && active.project == context.project)
+        {
+            active.connected_nodes.remove(&key.node);
         }
         Ok(identity)
     }
@@ -531,6 +548,32 @@ impl Coordinator {
             .expect("active process was checked immediately before removal"))
     }
 
+    pub fn abort_process_for_launch_attempt(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        launch_attempt: &clusterflux_core::LaunchAttemptId,
+    ) -> Result<ActiveProcess, CoordinatorError> {
+        let key = (tenant.clone(), project.clone(), process.clone());
+        let active = self.active_processes.get(&key).ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "launch rollback requires an active virtual process".to_owned(),
+            )
+        })?;
+        if active.launch_attempt.as_ref() != Some(launch_attempt) {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "launch rollback denied: attempt {} does not own process {}",
+                launch_attempt.as_str(),
+                process.as_str()
+            )));
+        }
+        Ok(self
+            .active_processes
+            .remove(&key)
+            .expect("active process was checked immediately before removal"))
+    }
+
     pub fn active_process_count(&self) -> usize {
         self.active_processes.len()
     }
@@ -542,8 +585,31 @@ impl Coordinator {
             .count()
     }
 
-    pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
+    pub fn tenant_count(&self) -> usize {
-        self.durable.node_identities.get(id)
+        self.durable.tenants.len()
+    }
+
+    pub fn user_count(&self) -> usize {
+        self.durable.users.len()
+    }
+
+    pub fn project_count(&self) -> usize {
+        self.durable.projects.len()
+    }
+
+    pub fn node_identity_count(&self) -> usize {
+        self.durable.node_identities.len()
+    }
+
+    pub fn node_identity(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        id: &NodeId,
+    ) -> Option<&NodeIdentityRecord> {
+        self.durable
+            .node_identities
+            .get(&NodeScopeKey::from_refs(tenant, project, id))
     }
 
     pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@@ -645,12 +711,24 @@ mod tests {
             .contains_key(&TenantId::from("tenant")));
         assert!(restarted.durable.users.contains_key(&UserId::from("user")));
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         assert_eq!(
             restarted
                 .durable
                 .credentials
-                .get("node:node")
+                .get(&node_subject)
                 .map(|credential| &credential.kind),
             Some(&CredentialKind::NodeCredential)
         );
@@ -674,7 +752,12 @@ mod tests {
         );
         assert_eq!(rerun.coordinator_epoch, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), Some((&process, 2)))
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                Some((&process, 2)),
+            )
             .unwrap();
         assert!(restarted
             .active_process(
@@ -687,6 +770,68 @@ mod tests {
             .contains(&NodeId::from("node")));
     }
 
+    #[test]
+    fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
+        let store = InMemoryDurableStore::default();
+        let mut coordinator = Coordinator::boot(&store, 1);
+        let node = NodeId::from("shared-node");
+        let scopes = [
+            (TenantId::from("tenant-a"), ProjectId::from("project-a")),
+            (TenantId::from("tenant-b"), ProjectId::from("project-b")),
+            (TenantId::from("tenant-a"), ProjectId::from("project-c")),
+        ];
+        for (index, (tenant, project)) in scopes.iter().enumerate() {
+            let mut grant = coordinator.create_node_enrollment_grant(
+                tenant.clone(),
+                project.clone(),
+                format!("grant-{index}"),
+                "node:attach",
+                100,
+            );
+            coordinator
+                .exchange_node_enrollment_grant(
+                    &mut grant,
+                    node.clone(),
+                    &format!("public-key-{index}"),
+                    "node:attach",
+                    99,
+                )
+                .unwrap();
+        }
+
+        let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
+        let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
+        let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
+        assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
+        assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
+        assert_eq!(
+            coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
+            coordinator.durable.node_identities.get(&scope_a)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
+            coordinator.durable.node_identities.get(&scope_b)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
+            coordinator.durable.node_identities.get(&scope_c)
+        );
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_a.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_b.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_c.credential_subject()));
+        assert_eq!(coordinator.durable.node_identities.len(), 3);
+        assert_eq!(coordinator.durable.credentials.len(), 3);
+    }
+
     #[test]
     fn identical_process_ids_are_isolated_by_tenant_and_project() {
         let store = InMemoryDurableStore::default();
@@ -735,11 +880,18 @@ mod tests {
 
         let mut restarted = Coordinator::boot(&store, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), None)
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                None,
+            )
             .unwrap();
 
         let error = restarted
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -771,7 +923,13 @@ mod tests {
             .unwrap();
 
         assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
+        assert!(coordinator
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
     }
 
     #[test]
@@ -785,10 +943,16 @@ mod tests {
             "public-key",
             "node:attach",
         );
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         coordinator.durable.credentials.insert(
-            "node:node".to_owned(),
+            node_subject.clone(),
             CredentialRecord {
-                subject: "node:node".to_owned(),
+                subject: node_subject.clone(),
                 tenant: TenantId::from("tenant"),
                 project: Some(ProjectId::from("project")),
                 kind: CredentialKind::NodeCredential,
@@ -802,6 +966,8 @@ mod tests {
         );
         coordinator
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -817,7 +983,7 @@ mod tests {
                 &NodeId::from("node"),
             )
             .unwrap_err();
-        assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(foreign, CoordinatorError::UnknownNode));
 
         let revoked = coordinator
             .revoke_node_credential(
@@ -830,8 +996,14 @@ mod tests {
             )
             .unwrap();
         assert_eq!(revoked.id, NodeId::from("node"));
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
+        assert!(coordinator
-        assert!(!coordinator.durable.credentials.contains_key("node:node"));
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_none());
+        assert!(!coordinator.durable.credentials.contains_key(&node_subject));
         assert!(!coordinator
             .active_process(
                 &TenantId::from("tenant"),
@@ -890,7 +1062,7 @@ mod tests {
     }
 
     #[test]
-    fn account_policy_state_summarizes_private_admin_records_safely() {
+    fn account_policy_state_summarizes_sensitive_admin_records_safely() {
         let tenant = TenantId::from("tenant");
         let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
 
@@ -1046,7 +1218,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(error, CoordinatorError::UnknownNode));
     }
 
     #[test]

crates/clusterflux-coordinator/src/main.rs

@@ -5,17 +5,40 @@ use clusterflux_core::{ProjectId, TenantId, UserId};
 use serde_json::json;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux coordinator.\n\n\
+                 Usage: clusterflux-coordinator [OPTIONS]\n\n\
+                 Options:\n  \
+                   --listen <ADDRESS>                 [default: 127.0.0.1:0]\n  \
+                   --allow-local-trusted-loopback\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
+
     let mut listen = "127.0.0.1:0".to_owned();
     let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
         .ok()
         .as_deref()
         == Some("1");
-    let mut args = std::env::args().skip(1);
+    let mut args = raw_args.into_iter();
     while let Some(arg) = args.next() {
         if arg == "--listen" {
             listen = args.next().ok_or("--listen requires an address")?;
         } else if arg == "--allow-local-trusted-loopback" {
             allow_local_trusted = true;
+        } else {
+            return Err(format!("unknown argument: {arg}").into());
         }
     }
 

crates/clusterflux-coordinator/src/postgres_store.rs

@@ -4,8 +4,8 @@ use thiserror::Error;
 
 use crate::{
     AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
-    DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
+    DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
-    ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
+    ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
 };
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
             state.projects.insert(record.id.clone(), record);
         }
         for record in self.query_records::<NodeIdentityRecord>(
-            "SELECT record FROM clusterflux_node_identities ORDER BY node_id",
+            "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
         )? {
-            state.node_identities.insert(record.id.clone(), record);
+            state.node_identities.insert(
+                NodeScopeKey::new(
+                    record.tenant.clone(),
+                    record.project.clone(),
+                    record.id.clone(),
+                ),
+                record,
+            );
         }
         for record in self.query_records::<CredentialRecord>(
             "SELECT record FROM clusterflux_credentials ORDER BY subject",
@@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
 );
 
 CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
-    node_id TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
     project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
-    record JSONB NOT NULL
+    node_id TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, node_id)
 );
 
+DO $clusterflux_node_scope_migration$
+DECLARE
+    current_primary_key TEXT;
+    current_primary_key_columns TEXT[];
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_node_identities
+        WHERE record->>'id' IS DISTINCT FROM node_id
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration refused an internally inconsistent legacy row';
+    END IF;
+
+    SELECT
+        constraint_record.conname,
+        array_agg(attribute.attname ORDER BY key_column.ordinality)
+    INTO current_primary_key, current_primary_key_columns
+    FROM pg_constraint AS constraint_record
+    CROSS JOIN LATERAL unnest(constraint_record.conkey)
+        WITH ORDINALITY AS key_column(attribute_number, ordinality)
+    JOIN pg_attribute AS attribute
+      ON attribute.attrelid = constraint_record.conrelid
+     AND attribute.attnum = key_column.attribute_number
+    WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
+      AND constraint_record.contype = 'p'
+    GROUP BY constraint_record.conname;
+
+    IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
+        EXECUTE format(
+            'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
+            current_primary_key
+        );
+        ALTER TABLE clusterflux_node_identities
+            ADD PRIMARY KEY (tenant_id, project_id, node_id);
+    ELSIF current_primary_key_columns
+        IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
+    THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration found an unexpected primary key shape';
+    END IF;
+END
+$clusterflux_node_scope_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     subject TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     record JSONB NOT NULL
 );
 
+DO $clusterflux_node_credential_migration$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials
+        WHERE record->>'subject' IS DISTINCT FROM subject
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration refused an internally inconsistent legacy row';
+    END IF;
+
+    UPDATE clusterflux_credentials AS credential
+    SET subject = format(
+            'node:%s:%s:%s:%s:%s:%s',
+            octet_length(identity.tenant_id),
+            identity.tenant_id,
+            octet_length(identity.project_id),
+            identity.project_id,
+            octet_length(identity.node_id),
+            identity.node_id
+        ),
+        record = jsonb_set(
+            credential.record,
+            '{subject}',
+            to_jsonb(format(
+                'node:%s:%s:%s:%s:%s:%s',
+                octet_length(identity.tenant_id),
+                identity.tenant_id,
+                octet_length(identity.project_id),
+                identity.project_id,
+                octet_length(identity.node_id),
+                identity.node_id
+            )),
+            false
+        )
+    FROM clusterflux_node_identities AS identity
+    WHERE credential.subject = 'node:' || identity.node_id
+      AND credential.tenant_id = identity.tenant_id
+      AND credential.project_id = identity.project_id;
+
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials AS credential
+        WHERE credential.subject LIKE 'node:%'
+          AND NOT EXISTS (
+              SELECT 1
+              FROM clusterflux_node_identities AS identity
+              WHERE credential.tenant_id = identity.tenant_id
+                AND credential.project_id = identity.project_id
+                AND credential.subject = format(
+                    'node:%s:%s:%s:%s:%s:%s',
+                    octet_length(identity.tenant_id),
+                    identity.tenant_id,
+                    octet_length(identity.project_id),
+                    identity.project_id,
+                    octet_length(identity.node_id),
+                    identity.node_id
+                )
+          )
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration found an unscoped or orphaned node subject';
+    END IF;
+END
+$clusterflux_node_credential_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
     session_digest TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -529,7 +651,13 @@ mod tests {
         let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
 
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
         assert_eq!(restarted.active_process_count(), 0);
     }
 

crates/clusterflux-coordinator/src/service.rs

@@ -7,13 +7,14 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use clusterflux_core::{
-    Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
+    Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
-    NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
+    CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
-    RateLimit, TenantId, TransportError, UserId,
+    NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
+    TenantId, TransportError, UserId,
 };
 use thiserror::Error;
 
-use crate::{Coordinator, CoordinatorError};
+use crate::{Coordinator, CoordinatorError, NodeScopeKey};
 
 mod admin;
 mod artifacts;
@@ -34,6 +35,7 @@ mod quota;
 mod relay;
 mod routing;
 mod signed_nodes;
+mod summaries;
 mod tcp;
 mod wire_protocol;
 use authorization::authorize_authenticated_user_operation;
@@ -43,12 +45,14 @@ use keys::{
     ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
 };
 pub use protocol::{
-    ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
-    CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
-    DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
+    DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
-    TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
-    TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
+    RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
-    TaskTerminalState, VirtualProcessStatus, WorkflowActor,
+    TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
+    TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
+    VirtualProcessStatus, WorkflowActor,
 };
 pub use quota::CoordinatorQuotaConfiguration;
 pub use relay::{
@@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
 const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
 const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
 const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
-const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
+const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
 const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
 const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
+const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
+const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
+const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
+const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
+const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
+const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
 const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
 fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
     requested.clamp(1, maximum)
@@ -111,6 +121,24 @@ pub struct CoordinatorAdmission {
     pub max_artifact_download_ttl_seconds: u64,
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CoordinatorOperationalMetrics {
+    pub tenants: usize,
+    pub users: usize,
+    pub projects: usize,
+    pub enrolled_nodes: usize,
+    pub reported_nodes: usize,
+    pub live_nodes: usize,
+    pub active_processes: usize,
+    pub active_coordinator_mains: usize,
+    pub max_active_coordinator_mains: usize,
+    pub active_tasks: usize,
+    pub queued_tasks: usize,
+    pub artifacts: usize,
+    pub retained_download_links: usize,
+    pub relay: ArtifactRelayUsage,
+}
+
 impl Default for CoordinatorAdmission {
     fn default() -> Self {
         Self {
@@ -151,16 +179,139 @@ pub enum CoordinatorServiceError {
     Durable(String),
 }
 
+impl CoordinatorServiceError {
+    pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
+        let request_id = request_id.into();
+        let message = self.to_string();
+        let (code, category, retryable) = match self {
+            Self::Io(_) => (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Json(_)
+            | Self::CapabilityReport(_)
+            | Self::InvalidArtifactPath(_)
+            | Self::InvalidTaskLogTail(_) => (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            ),
+            Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
+                return ApiError::from_message(request_id, message);
+            }
+            Self::Coordinator(CoordinatorError::UnknownNode) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Coordinator(CoordinatorError::Enrollment(_)) => (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            ),
+            Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
+                (ApiErrorCode::Conflict, ApiErrorCategory::State, true)
+            }
+            Self::Download(DownloadError::NotFound) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Download(DownloadError::Unavailable)
+            | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Download(DownloadError::LimitExceeded { .. }) => (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            ),
+            Self::Download(DownloadError::Unauthorized(_))
+            | Self::Download(DownloadError::InvalidToken)
+            | Self::Download(DownloadError::Expired)
+            | Self::Download(DownloadError::Revoked) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Scheduler(_) => (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Transport(_) => (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Panel(PanelError::RateLimited) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Panel(PanelError::UnknownWidget(_)) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Panel(_) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Durable(_) => (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                true,
+            ),
+        };
+        ApiError::new(code, category, message, retryable, request_id)
+    }
+}
+
 pub struct CoordinatorService {
     coordinator: Coordinator,
     store: RuntimeDurableStore,
-    node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
+    node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
-    node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
+    node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
     node_stale_after_seconds: u64,
     debug_freeze_timeout: std::time::Duration,
     enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
     task_events: VecDeque<TaskCompletionEvent>,
     process_scope_history: VecDeque<ProcessControlKey>,
+    process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
+    process_summary_order: VecDeque<ProcessControlKey>,
+    next_process_summary_order: u64,
+    task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
+    recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
+    recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
+    recent_log_accounted_bytes: BTreeMap<
+        (
+            TenantId,
+            ProjectId,
+            ProcessId,
+            clusterflux_core::TaskInstanceId,
+            String,
+        ),
+        u64,
+    >,
+    recent_log_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    recent_log_quota_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    next_recent_log_sequence: u64,
     debug_audit_events: VecDeque<DebugAuditEvent>,
     debug_epochs: BTreeMap<ProcessControlKey, u64>,
     debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@@ -180,7 +331,7 @@ pub struct CoordinatorService {
     process_cancellations: BTreeSet<ProcessControlKey>,
     process_aborts: BTreeSet<ProcessControlKey>,
     agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
-    node_replay_nonces: BTreeMap<(NodeId, String), u64>,
+    node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
     panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
     stopped_panels: BTreeSet<PanelStopKey>,
     panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@@ -215,6 +366,29 @@ impl CoordinatorService {
         self.artifact_relay.usage()
     }
 
+    pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
+        CoordinatorOperationalMetrics {
+            tenants: self.coordinator.tenant_count(),
+            users: self.coordinator.user_count(),
+            projects: self.coordinator.project_count(),
+            enrolled_nodes: self.coordinator.node_identity_count(),
+            reported_nodes: self.node_descriptors.len(),
+            live_nodes: self
+                .node_descriptors
+                .keys()
+                .filter(|scope| self.node_is_live(scope))
+                .count(),
+            active_processes: self.coordinator.active_process_count(),
+            active_coordinator_mains: self.main_runtime.active_main_count(),
+            max_active_coordinator_mains: self.main_runtime.max_active_mains(),
+            active_tasks: self.active_tasks.len(),
+            queued_tasks: self.pending_task_launches.len(),
+            artifacts: self.artifact_registry.artifact_count(),
+            retained_download_links: self.artifact_registry.retained_download_link_count(),
+            relay: self.artifact_relay.usage(),
+        }
+    }
+
     fn commit_artifact_relay(
         &mut self,
         candidate: relay::ArtifactRelayLedger,
@@ -284,15 +458,10 @@ impl CoordinatorService {
         {
             let identity = self
                 .coordinator
-                .node_identity(node)
+                .node_identity(tenant, project, node)
                 .ok_or(CoordinatorError::UnknownNode)?;
-            if &identity.tenant != tenant || &identity.project != project {
+            debug_assert_eq!(&identity.tenant, tenant);
-                return Err(CoordinatorError::Unauthorized(
+            debug_assert_eq!(&identity.project, project);
-                    "node process-control request is outside its enrolled tenant/project scope"
-                        .to_owned(),
-                )
-                .into());
-            }
             return Ok(());
         }
         self.coordinator
@@ -409,6 +578,16 @@ impl CoordinatorService {
             enrollment_grants: BTreeMap::new(),
             task_events: VecDeque::new(),
             process_scope_history: VecDeque::new(),
+            process_summaries: BTreeMap::new(),
+            process_summary_order: VecDeque::new(),
+            next_process_summary_order: 1,
+            task_terminal_states: BTreeMap::new(),
+            recent_logs: BTreeMap::new(),
+            recent_log_dropped_through: BTreeMap::new(),
+            recent_log_accounted_bytes: BTreeMap::new(),
+            recent_log_truncated_streams: BTreeSet::new(),
+            recent_log_quota_truncated_streams: BTreeSet::new(),
+            next_recent_log_sequence: 1,
             debug_audit_events: VecDeque::new(),
             debug_epochs: BTreeMap::new(),
             debug_epoch_runtime: BTreeMap::new(),

crates/clusterflux-coordinator/src/service/artifacts.rs

@@ -8,7 +8,7 @@ use clusterflux_core::{
 };
 use sha2::{Digest as _, Sha256};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::relay::RelayFinishReason;
 use super::{
@@ -52,7 +52,11 @@ impl CoordinatorService {
         let action = self
             .artifact_registry
             .download_action(&context, &artifact, &policy)?;
-        self.ensure_download_source_connectivity(&action.source)?;
+        self.ensure_download_source_connectivity(
+            &context.tenant,
+            &context.project,
+            &action.source,
+        )?;
         let downloadable_size = self
             .artifact_registry
             .downloadable_size(&context, &artifact, &policy)?;
@@ -163,7 +167,11 @@ impl CoordinatorService {
             },
             &mut validation_meter,
         )?;
-        self.ensure_download_source_connectivity(&stream.link.source)?;
+        self.ensure_download_source_connectivity(
+            &stream.link.tenant,
+            &stream.link.project,
+            &stream.link.source,
+        )?;
         self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
 
         if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@@ -302,7 +310,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let transfer_id = generate_opaque_token("artifact_transfer")
             .map_err(CoordinatorServiceError::Protocol)?;
@@ -410,7 +418,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
         let destination =
@@ -682,15 +690,10 @@ impl CoordinatorService {
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
+        debug_assert_eq!(&identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(&identity.project, project);
-                "artifact reverse transfer node is outside its enrolled tenant/project scope"
-                    .to_owned(),
-            )
-            .into());
-        }
         Ok(())
     }
 
@@ -725,15 +728,12 @@ impl CoordinatorService {
     ) -> Result<NodeEndpoint, CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
+        debug_assert_eq!(&identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(&identity.project, project);
-                "artifact export node is outside the tenant/project scope".to_owned(),
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
-            )
+        let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
-            .into());
-        }
-        let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "node {node} has not reported export connectivity"
             ))
@@ -744,7 +744,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "node {node} is offline for artifact export"
@@ -769,17 +769,20 @@ impl CoordinatorService {
 
     fn ensure_download_source_connectivity(
         &self,
+        tenant: &TenantId,
+        project: &ProjectId,
         source: &StorageLocation,
     ) -> Result<(), clusterflux_core::DownloadError> {
         let StorageLocation::RetainedNode(node) = source else {
             return Ok(());
         };
-        let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
+        let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "retaining node {node} has not reported online status for artifact download"
             ))
         })?;
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "retaining node {node} is offline for artifact download"

crates/clusterflux-coordinator/src/service/authorization.rs

@@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation {
     RevokeAgentPublicKey,
     CreateNodeEnrollmentGrant,
     ListNodeDescriptors,
+    ListNodeSummaries,
     RevokeNodeCredential,
     StartProcess,
     ScheduleTask,
@@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation {
     CancelProcess,
     AbortProcess,
     ListProcesses,
+    ListProcessSummaries,
     QuotaStatus,
     RestartTask,
     ResolveTaskFailure,
@@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation {
     InspectDebugEpoch,
     ListTaskEvents,
     ListTaskSnapshots,
+    ListRecentLogs,
     JoinTask,
+    ListArtifacts,
+    GetArtifact,
     CreateArtifactDownloadLink,
     OpenArtifactDownloadStream,
     RevokeArtifactDownloadLink,
@@ -56,6 +61,7 @@ impl PublicUserOperation {
             Self::RevokeAgentPublicKey => "revoke_agent_public_key",
             Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
             Self::ListNodeDescriptors => "list_node_descriptors",
+            Self::ListNodeSummaries => "list_node_summaries",
             Self::RevokeNodeCredential => "revoke_node_credential",
             Self::StartProcess => "start_process",
             Self::ScheduleTask => "schedule_task",
@@ -63,6 +69,7 @@ impl PublicUserOperation {
             Self::CancelProcess => "cancel_process",
             Self::AbortProcess => "abort_process",
             Self::ListProcesses => "list_processes",
+            Self::ListProcessSummaries => "list_process_summaries",
             Self::QuotaStatus => "quota_status",
             Self::RestartTask => "restart_task",
             Self::ResolveTaskFailure => "resolve_task_failure",
@@ -74,7 +81,10 @@ impl PublicUserOperation {
             Self::InspectDebugEpoch => "inspect_debug_epoch",
             Self::ListTaskEvents => "list_task_events",
             Self::ListTaskSnapshots => "list_task_snapshots",
+            Self::ListRecentLogs => "list_recent_logs",
             Self::JoinTask => "join_task",
+            Self::ListArtifacts => "list_artifacts",
+            Self::GetArtifact => "get_artifact",
             Self::CreateArtifactDownloadLink => "create_artifact_download_link",
             Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
             Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
@@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
                 Self::CreateNodeEnrollmentGrant
             }
             AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries,
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
                 Self::RevokeNodeCredential
             }
@@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
             AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
             AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => {
+                Self::ListProcessSummaries
+            }
             AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
             AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
             AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
@@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
             AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
             AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
+            AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs,
             AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
+            AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts,
+            AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact,
             AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
                 Self::CreateArtifactDownloadLink
             }

crates/clusterflux-coordinator/src/service/debug.rs

@@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct DebugBreakpointPlan {
     pub(super) actor: UserId,
+    pub(super) revision: u64,
     pub(super) probe_symbols: BTreeSet<String>,
     pub(super) hit_epoch: Option<u64>,
     pub(super) hit_task: Option<TaskInstanceId>,
@@ -85,6 +86,7 @@ impl CoordinatorService {
         project: String,
         actor_user: String,
         process: String,
+        revision: u64,
         probe_symbols: Vec<String>,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let probe_symbols = validate_probe_symbols(probe_symbols)?;
@@ -115,10 +117,28 @@ impl CoordinatorService {
             ))
             .into());
         }
+        let key = process_control_key(&tenant, &project, &process);
+        if let Some(current) = self.debug_breakpoints.get(&key) {
+            if current.actor == actor && revision < current.revision {
+                return Ok(CoordinatorResponse::DebugBreakpoints {
+                    process,
+                    actor,
+                    revision: current.revision,
+                    probe_symbols: current.probe_symbols.iter().cloned().collect(),
+                    hit_epoch: current.hit_epoch,
+                    hit_task: current.hit_task.clone(),
+                    hit_probe_symbol: current.hit_probe_symbol.clone(),
+                    charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
+                    used_debug_read_bytes: audit_event.used_debug_read_bytes,
+                    audit_event,
+                });
+            }
+        }
         self.debug_breakpoints.insert(
-            process_control_key(&tenant, &project, &process),
+            key,
             DebugBreakpointPlan {
                 actor: actor.clone(),
+                revision,
                 probe_symbols: probe_symbols.iter().cloned().collect(),
                 hit_epoch: None,
                 hit_task: None,
@@ -128,6 +148,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision,
             probe_symbols,
             hit_epoch: None,
             hit_task: None,
@@ -190,6 +211,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision: plan.revision,
             probe_symbols: plan.probe_symbols.into_iter().collect(),
             hit_epoch: plan.hit_epoch,
             hit_task: plan.hit_task,

crates/clusterflux-coordinator/src/service/debug_requests.rs

@@ -33,12 +33,14 @@ impl CoordinatorService {
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant,
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             ),
             CoordinatorRequest::InspectDebugBreakpoints {
@@ -96,12 +98,14 @@ impl CoordinatorService {
             ),
             AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant.as_str().to_owned(),
                 project.as_str().to_owned(),
                 actor.as_str().to_owned(),
                 process,
+                revision,
                 probe_symbols,
             ),
             AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
@@ -222,24 +226,25 @@ impl CoordinatorService {
         } else if !completed_event_observed {
             "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
         } else if let Some(checkpoint) = checkpoint {
-            let vfs_available =
+            let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
-                checkpoint
+                true,
-                    .checkpoint
+                |available, (path, object)| {
-                    .vfs_manifest
+                    let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
-                    .objects
+                        CoordinatorServiceError::InvalidArtifactPath(error.to_string())
-                    .iter()
+                    })?;
-                    .all(|(path, object)| {
+                    Ok::<_, CoordinatorServiceError>(
-                        let artifact = super::keys::artifact_id_from_path(path);
+                        available
-                        self.artifact_registry
+                            && self
-                            .metadata(&artifact)
+                                .artifact_registry
-                            .is_some_and(|metadata| {
+                                .metadata(&tenant, &project, &artifact)
-                                metadata.tenant == tenant
+                                .is_some_and(|metadata| {
-                                    && metadata.project == project
+                                    metadata.digest == object.digest
-                                    && metadata.digest == object.digest
+                                        && metadata.size == object.size
-                                    && metadata.size == object.size
+                                        && !metadata.retaining_nodes.is_empty()
-                                    && !metadata.retaining_nodes.is_empty()
+                                }),
-                            })
+                    )
-                    });
+                },
+            )?;
             if !vfs_available {
                 "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
             } else {
@@ -285,6 +290,25 @@ impl CoordinatorService {
                             task.as_str()
                         );
                         if let Some(replacement) = replacement {
+                            if assignment.task_spec.source_snapshot.is_some() {
+                                let replacement_source = replacement_bundle
+                                    .as_ref()
+                                    .and_then(|bundle| bundle.source_snapshot.clone())
+                                    .ok_or_else(|| {
+                                        CoordinatorServiceError::Protocol(
+                                            "replacement task omitted the current SourceSnapshot for source-bound arguments"
+                                                .to_owned(),
+                                        )
+                                    })?;
+                                assignment
+                                    .task_spec
+                                    .rebind_source_snapshot(replacement_source)
+                                    .map_err(|error| {
+                                        CoordinatorServiceError::Protocol(format!(
+                                            "replacement task SourceSnapshot is invalid: {error}"
+                                        ))
+                                    })?;
+                            }
                             assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
                                 export: Some(replacement.export),
                                 abi: WasmExportAbi::TaskV1,
@@ -456,6 +480,7 @@ impl CoordinatorService {
         }
         self.record_task_completion_event(event.clone());
         self.notify_coordinator_main_waiters(&event);
+        self.maybe_retire_terminal_process(&tenant, &project, &process)?;
         Ok(CoordinatorResponse::TaskFailureResolved {
             process,
             task,

crates/clusterflux-coordinator/src/service/keys.rs

@@ -1,6 +1,7 @@
 use clusterflux_core::{
     ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
 };
+use thiserror::Error;
 
 pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
 pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
     (tenant.clone(), project.clone(), grant.to_owned())
 }
 
-pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub(super) enum ArtifactPathError {
+    #[error("path must start with the exact /vfs/artifacts/ prefix")]
+    WrongPrefix,
+    #[error("path must name an artifact after /vfs/artifacts/")]
+    EmptyArtifact,
+    #[error("mapped artifact identifier is invalid: {0}")]
+    InvalidArtifactId(#[from] clusterflux_core::IdParseError),
+}
+
+pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
     let value = path
         .as_str()
         .strip_prefix("/vfs/artifacts/")
-        .unwrap_or(path.as_str())
+        .ok_or(ArtifactPathError::WrongPrefix)?;
-        .replace('/', ":");
+    if value.is_empty() {
-    ArtifactId::new(value)
+        return Err(ArtifactPathError::EmptyArtifact);
+    }
+    ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn artifact_id_conversion_is_fallible_and_structural() {
+        assert_eq!(
+            artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
+            ArtifactId::from("build:output")
+        );
+        for path in [
+            "/vfs/other/output",
+            "/vfs/artifacts",
+            "/vfs/artifacts/bad artifact!",
+        ] {
+            let path = VfsPath::new(path).unwrap();
+            assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
+        }
+
+        let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
+        assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
+    }
 }

crates/clusterflux-coordinator/src/service/logs.rs

@@ -1,5 +1,5 @@
 use clusterflux_core::{
-    ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
+    ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
     TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
 };
 
@@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key};
 use super::protocol::TaskAttemptState;
 use super::{
     artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
-    TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
+    RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState,
+    MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES,
+    MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT,
+    MAX_TASK_LOG_TAIL_BYTES,
 };
 
 impl CoordinatorService {
@@ -36,23 +39,44 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         validate_task_log_tail("stdout_tail", &stdout_tail)?;
         validate_task_log_tail("stderr_tail", &stderr_tail)?;
-        let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota
+        let stdout_retained = self.accept_final_log_stream(
-            .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+            &tenant,
-        self.quota
+            &project,
-            .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+            &process,
+            &task,
+            TaskLogStream::Stdout,
+            stdout_bytes,
+            &stdout_tail,
+            stdout_truncated,
+            now_epoch_seconds,
+        )?;
+        let stderr_retained = self.accept_final_log_stream(
+            &tenant,
+            &project,
+            &process,
+            &task,
+            TaskLogStream::Stderr,
+            stderr_bytes,
+            &stderr_tail,
+            stderr_truncated,
+            now_epoch_seconds,
+        )?;
         Ok(CoordinatorResponse::TaskLogRecorded {
             process,
             task,
             stdout_bytes,
             stderr_bytes,
-            stdout_tail: if stdout_truncated {
+            stdout_tail: if !stdout_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stdout_truncated {
                 format!("{stdout_tail}\n... truncated")
             } else {
                 stdout_tail
             },
-            stderr_tail: if stderr_truncated {
+            stderr_tail: if !stderr_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stderr_truncated {
                 format!("{stderr_tail}\n... truncated")
             } else {
                 stderr_tail
@@ -61,6 +85,158 @@ impl CoordinatorService {
         })
     }
 
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_report_task_log_chunk(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stream: TaskLogStream,
+        offset: u64,
+        source_bytes: u64,
+        text: String,
+        truncated: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
+                "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}",
+                text.len()
+            )));
+        }
+        if source_bytes == 0 && !text.is_empty() && !truncated {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes must describe non-empty text".to_owned(),
+            ));
+        }
+        if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(),
+            ));
+        }
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
+        let expected = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let end = offset.checked_add(source_bytes).ok_or_else(|| {
+            CoordinatorServiceError::Protocol(
+                "live log chunk offset exceeds the supported range".to_owned(),
+            )
+        })?;
+        let state_marker = source_bytes == 0 && truncated;
+        if end < expected || (end == expected && !state_marker) {
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: expected,
+            });
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key, end);
+            }
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: end.max(expected),
+            });
+        }
+        let newly_accounted = end.saturating_sub(expected);
+        if self
+            .quota
+            .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)
+            .is_err()
+        {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key.clone(), end);
+            }
+            let sequence = self.mark_log_quota_truncated(
+                &tenant,
+                &project,
+                &process,
+                &task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence,
+                next_offset: end.max(expected),
+            });
+        }
+        if offset > expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!("[log output lost: {} bytes]", offset - expected),
+                true,
+                now_epoch_seconds,
+            );
+        } else if offset < expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!(
+                    "[log output overlap omitted: {} new source bytes]",
+                    end - expected
+                ),
+                true,
+                now_epoch_seconds,
+            );
+        }
+        let marker_is_new = if truncated {
+            self.recent_log_truncated_streams.insert(key.clone())
+        } else {
+            false
+        };
+        let text = if state_marker && text.is_empty() {
+            "[log output truncated at source]".to_owned()
+        } else {
+            text
+        };
+        let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
+            .then(|| {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream,
+                    text,
+                    truncated,
+                    now_epoch_seconds,
+                )
+            });
+        if end > expected {
+            self.recent_log_accounted_bytes.insert(key, end);
+        }
+        Ok(CoordinatorResponse::TaskLogChunkRecorded {
+            process,
+            task,
+            sequence,
+            next_offset: end,
+        })
+    }
+
     pub(super) fn handle_report_vfs_metadata(
         &mut self,
         tenant: String,
@@ -85,7 +261,9 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant,
                 project,
                 process: process.clone(),
@@ -177,20 +355,37 @@ impl CoordinatorService {
             artifact_size_bytes,
             result,
         };
-        let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota.can_charge_log_bytes(
+        let stdout_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stdout,
+            event.stdout_bytes,
+            &event.stdout_tail,
+            event.stdout_truncated,
             now_epoch_seconds,
         )?;
-        self.quota.charge_log_bytes(
+        let stderr_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stderr,
+            event.stderr_bytes,
+            &event.stderr_tail,
+            event.stderr_truncated,
             now_epoch_seconds,
         )?;
+        if !stdout_retained {
+            event.stdout_tail = "[log output truncated at project log quota]".to_owned();
+            event.stdout_truncated = true;
+        }
+        if !stderr_retained {
+            event.stderr_tail = "[log output truncated at project log quota]".to_owned();
+            event.stderr_truncated = true;
+        }
         let task_key = task_control_key(
             &event.tenant,
             &event.project,
@@ -203,7 +398,9 @@ impl CoordinatorService {
         event.placement = self.task_placements.remove(&task_key);
         if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant: event.tenant.clone(),
                 project: event.project.clone(),
                 process: event.process.clone(),
@@ -217,26 +414,22 @@ impl CoordinatorService {
         self.task_aborts.remove(&task_key);
         self.debug_commands.remove(&task_key);
         self.active_tasks.remove(&task_key);
-        let no_active_tasks =
+        self.clear_recent_log_offsets_for_task(
-            !self
+            &event.tenant,
-                .active_tasks
+            &event.project,
-                .iter()
+            &event.process,
-                .any(|(task_tenant, task_project, task_process, _, _)| {
+            &event.task,
-                    task_tenant == &event.tenant
+        );
-                        && task_project == &event.project
+        self.task_assignments.retain(|_, assignments| {
-                        && task_process == &event.process
+            assignments.retain(|assignment| {
-                });
+                assignment.tenant != event.tenant
-        if no_active_tasks {
+                    || assignment.project != event.project
-            self.process_aborts.remove(&process_key);
+                    || assignment.process != event.process
-            if self.process_cancellations.remove(&process_key)
+                    || assignment.node != event.node
-                && !self.main_runtime.controls.contains_key(&process_key)
+                    || assignment.task != event.task
-            {
+            });
-                self.coordinator
+            !assignments.is_empty()
-                    .abort_process(&event.tenant, &event.project, &event.process)?;
+        });
-                self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
-                self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
-            }
-        }
         if process_was_aborted {
             let checkpoint_key = super::keys::task_restart_key(
                 &event.tenant,
@@ -253,10 +446,20 @@ impl CoordinatorService {
         if !awaiting_operator {
             self.notify_coordinator_main_waiters(&event);
         }
+        self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
+        let events_recorded = self
+            .task_events
+            .iter()
+            .filter(|recorded| {
+                recorded.tenant == event.tenant
+                    && recorded.project == event.project
+                    && recorded.process == event.process
+            })
+            .count();
         Ok(CoordinatorResponse::TaskRecorded {
             process: event.process,
             task: event.task,
-            events_recorded: self.task_events.len(),
+            events_recorded,
         })
     }
 
@@ -316,7 +519,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::TaskSnapshots { snapshots })
     }
 
-    fn authorize_task_event_process_scope(
+    pub(super) fn authorize_task_event_process_scope(
         &self,
         tenant: &TenantId,
         project: &ProjectId,
@@ -356,6 +559,47 @@ impl CoordinatorService {
         Ok(())
     }
 
+    pub(super) fn handle_list_recent_logs(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: Option<String>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = task.map(TaskInstanceId::new);
+        self.authorize_task_event_process_scope(&tenant, &project, &process)?;
+        let after_sequence = after_sequence.unwrap_or(0);
+        let retained = self.recent_logs.get(&(tenant.clone(), project.clone()));
+        let history_truncated = self
+            .recent_log_dropped_through
+            .get(&process_control_key(&tenant, &project, &process))
+            .is_some_and(|dropped_through| *dropped_through > after_sequence);
+        let entries = retained
+            .into_iter()
+            .flatten()
+            .filter(|entry| {
+                entry.process == process
+                    && entry.sequence > after_sequence
+                    && task.as_ref().is_none_or(|task| &entry.task == task)
+            })
+            .take(limit as usize)
+            .cloned()
+            .collect::<Vec<_>>();
+        let next_sequence = entries.last().map(|entry| entry.sequence);
+        Ok(CoordinatorResponse::RecentLogs {
+            entries,
+            next_sequence,
+            history_truncated,
+        })
+    }
+
     pub(super) fn handle_join_task(
         &mut self,
         tenant: String,
@@ -474,6 +718,22 @@ impl CoordinatorService {
     pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
         event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
         event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
+        match event.executor {
+            super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
+                &event.tenant,
+                &event.project,
+                &event.process,
+                event.task_definition.clone(),
+                event.task.clone(),
+                event.terminal_state.clone(),
+            ),
+            super::TaskExecutor::Node => {
+                self.task_terminal_states.insert(
+                    task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
+                    event.terminal_state.clone(),
+                );
+            }
+        }
         let process_scope = (
             event.tenant.clone(),
             event.project.clone(),
@@ -511,6 +771,321 @@ impl CoordinatorService {
         self.task_events.push_back(event);
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn record_recent_log(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        task: TaskInstanceId,
+        stream: TaskLogStream,
+        mut text: String,
+        mut truncated: bool,
+        server_timestamp_epoch_seconds: u64,
+    ) -> u64 {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES;
+            while !text.is_char_boundary(boundary) {
+                boundary -= 1;
+            }
+            text.truncate(boundary);
+            truncated = true;
+        }
+        let sequence = self.next_recent_log_sequence;
+        self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1);
+        let logs = self
+            .recent_logs
+            .entry((tenant.clone(), project.clone()))
+            .or_default();
+        let mut dropped = Vec::new();
+        while logs.iter().filter(|entry| entry.process == process).count()
+            >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS
+        {
+            let Some(index) = logs.iter().position(|entry| entry.process == process) else {
+                break;
+            };
+            if let Some(entry) = logs.remove(index) {
+                dropped.push(entry);
+            }
+        }
+        while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT
+            || logs
+                .iter()
+                .map(|entry| entry.text.len())
+                .sum::<usize>()
+                .saturating_add(text.len())
+                > MAX_RECENT_LOG_BYTES_PER_PROJECT
+        {
+            match logs.pop_front() {
+                Some(entry) => dropped.push(entry),
+                None => break,
+            }
+        }
+        logs.push_back(RecentLogEntry {
+            sequence,
+            process,
+            task,
+            stream,
+            text,
+            server_timestamp_epoch_seconds,
+            truncated,
+        });
+        for entry in dropped {
+            let key = process_control_key(&tenant, &project, &entry.process);
+            self.recent_log_dropped_through
+                .entry(key)
+                .and_modify(|dropped_through| {
+                    *dropped_through = (*dropped_through).max(entry.sequence);
+                })
+                .or_insert(entry.sequence);
+        }
+        sequence
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn accept_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| {
+            let stream_name = match stream {
+                TaskLogStream::Stdout => "stdout",
+                TaskLogStream::Stderr => "stderr",
+            };
+            CoordinatorServiceError::Protocol(format!(
+                "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted"
+            ))
+        })?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            return Ok(false);
+        }
+        if self
+            .quota
+            .charge_log_bytes(tenant, project, remaining, now_epoch_seconds)
+            .is_err()
+        {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            self.mark_log_quota_truncated(
+                tenant,
+                project,
+                process,
+                task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(false);
+        }
+        self.reconcile_final_log_stream(
+            tenant,
+            project,
+            process,
+            task,
+            stream,
+            total_source_bytes,
+            final_tail,
+            source_truncated,
+            now_epoch_seconds,
+        );
+        Ok(true)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn mark_log_quota_truncated(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: &TaskLogStream,
+        now_epoch_seconds: u64,
+    ) -> Option<u64> {
+        let key = recent_log_offset_key(tenant, project, process, task, stream);
+        if !self.recent_log_quota_truncated_streams.insert(key.clone()) {
+            return None;
+        }
+        self.recent_log_truncated_streams.insert(key);
+        Some(self.record_recent_log(
+            tenant.clone(),
+            project.clone(),
+            process.clone(),
+            task.clone(),
+            stream.clone(),
+            "[log output truncated at project log quota]".to_owned(),
+            true,
+            now_epoch_seconds,
+        ))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn reconcile_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let mut visible_truncation = false;
+        if total_source_bytes > accounted {
+            let missing = total_source_bytes - accounted;
+            if final_tail.is_empty() {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!("[log output unavailable: {missing} source bytes]"),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            } else if (final_tail.len() as u64) <= total_source_bytes {
+                let tail_source_start = total_source_bytes - final_tail.len() as u64;
+                if accounted < tail_source_start {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        format!(
+                            "[log output lost before final tail: {} source bytes]",
+                            tail_source_start - accounted
+                        ),
+                        true,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation = true;
+                }
+                let source_start = accounted.max(tail_source_start) - tail_source_start;
+                let mut byte_start = usize::try_from(source_start)
+                    .unwrap_or(final_tail.len())
+                    .min(final_tail.len());
+                while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
+                    byte_start += 1;
+                }
+                let suffix = &final_tail[byte_start..];
+                if !suffix.is_empty() {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        suffix.to_owned(),
+                        source_truncated || visible_truncation,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation |= source_truncated;
+                }
+            } else if accounted == 0 {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    final_tail.to_owned(),
+                    source_truncated,
+                    now_epoch_seconds,
+                );
+                visible_truncation |= source_truncated;
+            } else {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!(
+                        "[{missing} additional source bytes could not be merged without duplicating redacted output]"
+                    ),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            }
+            self.recent_log_accounted_bytes
+                .insert(key.clone(), total_source_bytes);
+        }
+
+        let marker_is_new = (source_truncated || visible_truncation)
+            && self.recent_log_truncated_streams.insert(key);
+        if marker_is_new && !visible_truncation {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream,
+                "[log output truncated at source]".to_owned(),
+                true,
+                now_epoch_seconds,
+            );
+        }
+    }
+
+    pub(super) fn clear_recent_log_offsets_for_task(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+    ) {
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _), _| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+    }
+
     fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
         let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
         let Some(attempt) = self
@@ -545,6 +1120,118 @@ impl CoordinatorService {
         awaiting_operator
     }
 
+    pub(super) fn maybe_retire_terminal_process(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let process_key = process_control_key(tenant, project, process);
+        if self.main_runtime.controls.contains_key(&process_key) {
+            return Ok(false);
+        }
+
+        let has_runnable_remote_work =
+            self.active_tasks
+                .iter()
+                .any(|(task_tenant, task_project, task_process, _, _)| {
+                    task_tenant == tenant && task_project == project && task_process == process
+                })
+                || self.pending_task_launches.iter().any(|pending| {
+                    &pending.tenant == tenant
+                        && &pending.project == project
+                        && &pending.process == process
+                })
+                || self.task_assignments.values().any(|assignments| {
+                    assignments.iter().any(|assignment| {
+                        &assignment.tenant == tenant
+                            && &assignment.project == project
+                            && &assignment.process == process
+                    })
+                })
+                || self.task_attempts.iter().any(
+                    |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                        attempt_tenant == tenant
+                            && attempt_project == project
+                            && attempt_process == process
+                            && attempts.iter().rev().any(|attempt| {
+                                attempt.current
+                                    && matches!(
+                                        attempt.state,
+                                        TaskAttemptState::Queued
+                                            | TaskAttemptState::Running
+                                            | TaskAttemptState::FailedAwaitingAction
+                                    )
+                            })
+                    },
+                );
+        if has_runnable_remote_work {
+            return Ok(false);
+        }
+
+        let main_completed = self
+            .process_summaries
+            .get(&process_key)
+            .and_then(|summary| summary.main_terminal_state.as_ref())
+            .is_some_and(|state| matches!(state, TaskTerminalState::Completed));
+        let cancellation_completed = self.process_cancellations.contains(&process_key);
+        if !main_completed && !cancellation_completed {
+            return Ok(false);
+        }
+
+        self.process_aborts.remove(&process_key);
+        self.process_cancellations.remove(&process_key);
+        if self
+            .coordinator
+            .active_process(tenant, project, process)
+            .is_none()
+        {
+            return Ok(false);
+        }
+        let final_result = if cancellation_completed {
+            super::ProcessFinalResult::Cancelled
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Failed
+            },
+        ) {
+            super::ProcessFinalResult::Failed
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Cancelled
+            },
+        ) {
+            super::ProcessFinalResult::Cancelled
+        } else {
+            super::ProcessFinalResult::Completed
+        };
+        self.record_process_terminal(
+            tenant,
+            project,
+            process,
+            final_result,
+            self.current_epoch_seconds()?,
+        );
+        self.coordinator.abort_process(tenant, project, process)?;
+        self.clear_debug_state_for_process(tenant, project, process);
+        self.clear_operator_panel_state(tenant, project, process);
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(tenant, project);
+        self.artifact_registry.enforce_project_metadata_limit(
+            tenant,
+            project,
+            &pinned,
+            &protected_processes,
+        );
+        Ok(true)
+    }
+
     fn flush_artifact_metadata(
         &mut self,
         flush: ArtifactFlush,
@@ -552,23 +1239,59 @@ impl CoordinatorService {
         let now_epoch_seconds = self.current_epoch_seconds()?;
         self.artifact_registry
             .expire_download_links(now_epoch_seconds);
-        let pinned = self
+        let tenant = flush.tenant.clone();
-            .task_restart_checkpoints
+        let project = flush.project.clone();
-            .values()
+        let (pinned, protected_processes) =
-            .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
+            self.artifact_retention_guards_for_project(&tenant, &project);
-            .chain(
-                self.pending_task_launches
-                    .iter()
-                    .flat_map(|pending| pending.task_spec.required_artifacts.iter()),
-            )
-            .cloned()
-            .collect::<std::collections::BTreeSet<ArtifactId>>();
         self.artifact_registry
-            .flush_metadata_bounded(flush, &pinned)
+            .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
             .map(|_| ())
             .map_err(CoordinatorServiceError::Protocol)
     }
 
+    fn artifact_retention_guards_for_project(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> (
+        std::collections::BTreeSet<ArtifactScopeKey>,
+        std::collections::BTreeSet<ProcessId>,
+    ) {
+        let mut pinned = std::collections::BTreeSet::new();
+        for checkpoint in self.task_restart_checkpoints.values() {
+            if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
+            {
+                continue;
+            }
+            for artifact in &checkpoint.assignment.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &checkpoint.assignment.tenant,
+                    &checkpoint.assignment.project,
+                    artifact,
+                ));
+            }
+        }
+        for pending in &self.pending_task_launches {
+            if &pending.tenant != tenant || &pending.project != project {
+                continue;
+            }
+            for artifact in &pending.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &pending.tenant,
+                    &pending.project,
+                    artifact,
+                ));
+            }
+        }
+        let protected_processes = self
+            .coordinator
+            .active_processes_for_project(tenant, project)
+            .into_iter()
+            .map(|process| process.id)
+            .collect();
+        (pinned, protected_processes)
+    }
+
     fn task_is_known_or_active(
         &self,
         tenant: &TenantId,
@@ -601,15 +1324,24 @@ impl CoordinatorService {
     }
 }
 
-fn checked_reported_log_bytes(
+fn recent_log_offset_key(
-    stdout_bytes: u64,
+    tenant: &TenantId,
-    stderr_bytes: u64,
+    project: &ProjectId,
-) -> Result<u64, CoordinatorServiceError> {
+    process: &ProcessId,
-    stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
+    task: &TaskInstanceId,
-        CoordinatorServiceError::Protocol(
+    stream: &TaskLogStream,
-            "reported task log byte counts exceed the supported range".to_owned(),
+) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) {
-        )
+    (
-    })
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        task.clone(),
+        match stream {
+            TaskLogStream::Stdout => "stdout",
+            TaskLogStream::Stderr => "stderr",
+        }
+        .to_owned(),
+    )
 }
 
 fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
@@ -626,11 +1358,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
     if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
         return value;
     }
-    let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
+    let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES;
-    while !value.is_char_boundary(boundary) {
+    while boundary < value.len() && !value.is_char_boundary(boundary) {
-        boundary -= 1;
+        boundary += 1;
     }
-    value.truncate(boundary);
+    value.drain(..boundary);
     *truncated = true;
     value
 }

crates/clusterflux-coordinator/src/service/main_runtime.rs

@@ -30,14 +30,14 @@ use super::{
 };
 
 #[derive(Clone)]
-struct MainScope {
+pub(super) struct MainScope {
-    tenant: TenantId,
+    pub(super) tenant: TenantId,
-    project: ProjectId,
+    pub(super) project: ProjectId,
-    process: ProcessId,
+    pub(super) process: ProcessId,
-    task_definition: TaskDefinitionId,
+    pub(super) task_definition: TaskDefinitionId,
-    task_instance: TaskInstanceId,
+    pub(super) task_instance: TaskInstanceId,
-    epoch: u64,
+    pub(super) epoch: u64,
-    launch_id: u64,
+    pub(super) launch_id: u64,
 }
 
 enum MainCommand {
@@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime {
 }
 
 impl CoordinatorMainRuntime {
+    pub(super) fn active_main_count(&self) -> usize {
+        self.controls.len()
+    }
+
+    pub(super) fn max_active_mains(&self) -> usize {
+        self.max_active_mains
+    }
+
     pub(super) fn configure(
         &mut self,
         configuration: super::CoordinatorMainRuntimeConfiguration,
@@ -729,6 +737,13 @@ impl CoordinatorService {
                 &process,
                 "coordinator main launch failed admission or validation",
             );
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Failed,
+                self.liveness_now_epoch_seconds(),
+            );
             let _ = self.coordinator.abort_process(&tenant, &project, &process);
         }
         result
@@ -889,7 +904,7 @@ impl CoordinatorService {
         }
     }
 
-    fn record_coordinator_main_completion(
+    pub(super) fn record_coordinator_main_completion(
         &mut self,
         scope: MainScope,
         result: Result<WasmTaskResult, String>,
@@ -929,6 +944,7 @@ impl CoordinatorService {
             ),
             Err(error) => (TaskTerminalState::Failed, None, error),
         };
+        let main_completed = matches!(terminal_state, TaskTerminalState::Completed);
         let main_state = match terminal_state {
             TaskTerminalState::Completed => "completed",
             TaskTerminalState::Failed => "failed",
@@ -970,6 +986,12 @@ impl CoordinatorService {
             }
         }
         let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
+        self.main_runtime.controls.remove(&process_key);
+        if main_completed {
+            let _ =
+                self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
+            return;
+        }
         for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
             if task_tenant == scope.tenant
                 && task_project == scope.project
@@ -985,10 +1007,16 @@ impl CoordinatorService {
             }
         }
         self.process_aborts.insert(process_key.clone());
+        self.record_process_terminal(
+            &scope.tenant,
+            &scope.project,
+            &scope.process,
+            super::ProcessFinalResult::Failed,
+            self.liveness_now_epoch_seconds(),
+        );
         let _ = self
             .coordinator
             .abort_process(&scope.tenant, &scope.project, &scope.process);
-        self.main_runtime.controls.remove(&process_key);
         self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process);
         self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process);
     }
@@ -1157,6 +1185,120 @@ mod tests {
         assert!(!service.main_runtime.controls.contains_key(&process_key));
         assert!(!service.debug_epochs.contains_key(&process_key));
         assert!(!service.debug_epoch_runtime.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+    }
+
+    #[test]
+    fn completed_main_keeps_process_and_debug_state_for_active_children() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-current");
+        let main_task = TaskInstanceId::from("ti:vp-current:main");
+        let child_task = TaskInstanceId::from("ti:vp-current:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task.clone(),
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(
+            scope,
+            Ok(WasmTaskResult::completed(
+                main_task,
+                TaskBoundaryValue::SmallJson(serde_json::Value::Null),
+            )),
+        );
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_some());
+        assert!(!service.main_runtime.controls.contains_key(&process_key));
+        assert!(service.debug_epochs.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(!service.task_aborts.contains(&child_key));
+    }
+
+    #[test]
+    fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-failed-main");
+        let main_task = TaskInstanceId::from("ti:vp-failed-main:main");
+        let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task,
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(scope, Err("main crashed".to_owned()));
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_none());
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(service.task_aborts.contains(&child_key));
         assert!(service.process_aborts.contains(&process_key));
+        assert!(!service.debug_epochs.contains_key(&process_key));
+        assert!(service.task_events.iter().any(|event| {
+            event.process == process
+                && event.executor == TaskExecutor::CoordinatorMain
+                && event.terminal_state == TaskTerminalState::Failed
+        }));
     }
 }

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     SourceProviderKind, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{
     bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@@ -27,9 +27,9 @@ impl CoordinatorService {
         unix_timestamp_seconds()
     }
 
-    pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
+    pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
         self.node_last_seen_epoch_seconds
-            .get(node)
+            .get(scope)
             .is_some_and(|last_seen| {
                 self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
                     <= self.node_stale_after_seconds
@@ -41,7 +41,11 @@ impl CoordinatorService {
             .values()
             .cloned()
             .map(|mut descriptor| {
-                descriptor.online = self.node_is_live(&descriptor.id);
+                descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
+                    &descriptor.tenant,
+                    &descriptor.project,
+                    &descriptor.id,
+                ));
                 descriptor
             })
             .collect()
@@ -165,7 +169,11 @@ impl CoordinatorService {
             !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
         });
         self.coordinator.ensure_tenant_active(&tenant)?;
-        if self.coordinator.node_identity(&node).is_none() {
+        if self
+            .coordinator
+            .node_identity(&tenant, &project, &node)
+            .is_none()
+        {
             self.quota.ensure_node_admission(
                 &tenant,
                 self.coordinator.node_identity_count_for_tenant(&tenant),
@@ -197,12 +205,21 @@ impl CoordinatorService {
 
     pub(super) fn handle_node_heartbeat(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         node_signature: Option<NodeSignedRequest>,
         payload_digest: &Digest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
-        self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
+        self.authenticate_node_request(
+            &NodeScopeKey::new(tenant, project, node.clone()),
+            node_signature,
+            "node_heartbeat",
+            payload_digest,
+        )?;
         Ok(CoordinatorResponse::NodeHeartbeat {
             node,
             epoch: self.coordinator.coordinator_epoch(),
@@ -225,16 +242,13 @@ impl CoordinatorService {
         let tenant = TenantId::new(tenant);
         let project = ProjectId::new(project);
         let node = NodeId::new(node);
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "node capability report is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
         capabilities.validate_public_report()?;
         for (kind, count) in [
             ("cached environments", cached_environment_digests.len()),
@@ -274,16 +288,20 @@ impl CoordinatorService {
             .into_iter()
             .map(ArtifactId::new)
             .collect::<BTreeSet<_>>();
-        self.artifact_registry
+        self.artifact_registry.reconcile_node_retention(
-            .reconcile_node_retention(&node, &artifact_locations);
+            &tenant,
+            &project,
+            &node,
+            &artifact_locations,
+        );
 
-        let online = self.node_is_live(&node);
+        let online = self.node_is_live(&node_scope);
         self.node_descriptors.insert(
-            node.clone(),
+            node_scope,
             NodeDescriptor {
                 id: node.clone(),
-                tenant,
+                tenant: tenant.clone(),
-                project,
+                project: project.clone(),
                 capabilities,
                 cached_environments: cached_environment_digests.into_iter().collect(),
                 dependency_caches: dependency_cache_digests.into_iter().collect(),
@@ -293,9 +311,14 @@ impl CoordinatorService {
                 online,
             },
         );
+        let node_descriptors = self
+            .node_descriptors
+            .values()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .count();
         Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
             node,
-            node_descriptors: self.node_descriptors.len(),
+            node_descriptors,
         })
     }
 
@@ -333,9 +356,13 @@ impl CoordinatorService {
             actor: Actor::User(actor.clone()),
         };
         self.coordinator.revoke_node_credential(&context, &node)?;
-        let descriptor_removed = self.node_descriptors.remove(&node).is_some();
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
-        self.node_last_seen_epoch_seconds.remove(&node);
+        let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
-        self.artifact_registry.garbage_collect_node(&node);
+        self.node_last_seen_epoch_seconds.remove(&node_scope);
+        self.node_replay_nonces
+            .retain(|(retained_scope, _), _| retained_scope != &node_scope);
+        self.artifact_registry
+            .garbage_collect_node(&tenant, &project, &node);
         let queued_assignments_removed = self
             .task_assignments
             .remove(&(tenant.clone(), project.clone(), node.clone()))
@@ -369,14 +396,14 @@ impl CoordinatorService {
 
     pub(super) fn authenticate_node_request(
         &mut self,
-        node: &NodeId,
+        scope: &NodeScopeKey,
         node_signature: Option<NodeSignedRequest>,
         request_kind: &str,
         payload_digest: &Digest,
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(&scope.tenant, &scope.project, &scope.node)
             .ok_or(CoordinatorError::UnknownNode)?;
         let signature = node_signature.ok_or_else(|| {
             CoordinatorError::Unauthorized(
@@ -401,7 +428,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        let replay_key = (node.clone(), signature.nonce.clone());
+        let replay_key = (scope.clone(), signature.nonce.clone());
         self.node_replay_nonces.retain(|_, accepted_at| {
             now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
         });
@@ -413,7 +440,7 @@ impl CoordinatorService {
         }
         verify_node_request_signature(
             &identity.public_key,
-            node,
+            &scope.node,
             request_kind,
             payload_digest,
             &signature,
@@ -422,7 +449,7 @@ impl CoordinatorService {
         if self
             .node_replay_nonces
             .keys()
-            .filter(|(retained_node, _)| retained_node == node)
+            .filter(|(retained_scope, _)| retained_scope == scope)
             .count()
             >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
         {
@@ -436,8 +463,8 @@ impl CoordinatorService {
             .insert(replay_key, now_epoch_seconds);
         let seen_at = self.liveness_now_epoch_seconds();
         self.node_last_seen_epoch_seconds
-            .insert(node.clone(), seen_at);
+            .insert(scope.clone(), seen_at);
-        if let Some(descriptor) = self.node_descriptors.get_mut(node) {
+        if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
             descriptor.online = true;
         }
         Ok(())

crates/clusterflux-coordinator/src/service/panels.rs

@@ -254,7 +254,8 @@ impl CoordinatorService {
             .rev()
             .find_map(|event| event.artifact_path.as_ref())
         {
-            let artifact = artifact_id_from_path(path);
+            let artifact = artifact_id_from_path(path)
+                .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
             let context = clusterflux_core::AuthContext {
                 tenant,
                 project,

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -118,14 +118,14 @@ impl CoordinatorService {
         let mut objects = BTreeMap::new();
         let mut missing_required_artifact = false;
         for artifact in &task_spec.required_artifacts {
-            let Some(metadata) = self.artifact_registry.metadata(artifact) else {
+            let Some(metadata) =
+                self.artifact_registry
+                    .metadata(&assignment.tenant, &assignment.project, artifact)
+            else {
                 missing_required_artifact = true;
                 continue;
             };
-            if metadata.tenant != assignment.tenant
+            if metadata.retaining_nodes.is_empty() {
-                || metadata.project != assignment.project
-                || metadata.retaining_nodes.is_empty()
-            {
                 missing_required_artifact = true;
                 continue;
             }
@@ -483,17 +483,14 @@ impl CoordinatorService {
         .validate()
         .map_err(CoordinatorServiceError::Protocol)?;
         for artifact in &task_spec.required_artifacts {
-            let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
+            let metadata = self
-                CoordinatorError::Unauthorized(format!(
+                .artifact_registry
-                    "required artifact {artifact} is unavailable or has expired"
+                .metadata(&tenant, &project, artifact)
-                ))
+                .ok_or_else(|| {
-            })?;
+                    CoordinatorError::Unauthorized(format!(
-            if metadata.tenant != tenant || metadata.project != project {
+                        "required artifact {artifact} is unavailable or has expired in this tenant/project scope"
-                return Err(CoordinatorError::Unauthorized(format!(
+                    ))
-                    "required artifact {artifact} is outside the task tenant/project scope"
+                })?;
-                ))
-                .into());
-            }
             if metadata.retaining_nodes.is_empty() {
                 return Err(CoordinatorError::Unauthorized(format!(
                     "required artifact {artifact} has no retaining node"
@@ -548,13 +545,22 @@ impl CoordinatorService {
                     task_spec,
                     wasm_module_base64,
                 });
+                let queued_tasks = self
+                    .pending_task_launches
+                    .iter()
+                    .filter(|pending| {
+                        pending.tenant == tenant
+                            && pending.project == project
+                            && pending.process == process
+                    })
+                    .count();
                 return Ok(CoordinatorResponse::TaskQueued {
                     process,
                     task,
                     actor,
                     reason,
                     charged_spawns,
-                    queued_tasks: self.pending_task_launches.len(),
+                    queued_tasks,
                 });
             }
             Err(err) => return Err(err.into()),
@@ -665,7 +671,9 @@ impl CoordinatorService {
                 ));
             };
             self.task_attempts.remove(&removable);
+            self.task_terminal_states.remove(&removable);
         }
+        self.task_terminal_states.remove(&key);
         let attempts = self.task_attempts.entry(key).or_default();
         for attempt in attempts.iter_mut() {
             attempt.current = false;

crates/clusterflux-coordinator/src/service/processes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::keys::{process_control_key, task_control_key};
 use super::{
@@ -47,14 +47,10 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "task assignment poll is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
         let assignment_key = (tenant.clone(), project.clone(), node.clone());
         let assignment = self
             .task_assignments
@@ -77,7 +73,11 @@ impl CoordinatorService {
         project: &ProjectId,
         node: &NodeId,
     ) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
-        let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
+        let Some(descriptor) = self
+            .node_descriptors
+            .get(&NodeScopeKey::from_refs(tenant, project, node))
+            .cloned()
+        else {
             return Ok(None);
         };
         let mut remaining = VecDeque::new();
@@ -233,20 +233,18 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
+        debug_assert_eq!(identity.tenant, tenant);
-            return Err(CoordinatorError::Unauthorized(
+        debug_assert_eq!(identity.project, project);
-                "source preparation completion is outside the enrolled tenant/project scope"
+        let descriptor = self
-                    .to_owned(),
+            .node_descriptors
-            )
+            .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
-            .into());
+            .ok_or_else(|| {
-        }
+                CoordinatorError::Unauthorized(
-        let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
+                    "source preparation completion requires a node capability report".to_owned(),
-            CoordinatorError::Unauthorized(
+                )
-                "source preparation completion requires a node capability report".to_owned(),
+            })?;
-            )
-        })?;
         if !descriptor.source_snapshots.contains(&source_snapshot)
             && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
         {
@@ -273,6 +271,7 @@ impl CoordinatorService {
         agent_signature: Option<AgentSignedRequest>,
         request_payload_digest: Option<&Digest>,
         process: String,
+        launch_attempt: Option<String>,
         restart: bool,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let tenant = TenantId::new(tenant);
@@ -383,6 +382,10 @@ impl CoordinatorService {
                     || attempt_project != &project
                     || attempt_process != &process
             });
+        self.task_terminal_states
+            .retain(|(task_tenant, task_project, task_process, _), _| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
         self.restart_launches
             .retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
                 attempt_tenant != &tenant
@@ -392,10 +395,19 @@ impl CoordinatorService {
         self.debug_audit_events.retain(|event| {
             event.tenant != tenant || event.project != project || event.process != process
         });
-        self.coordinator
+        let active = self.coordinator.start_process_for_launch_attempt(
-            .start_process(tenant, project, process.clone());
+            tenant.clone(),
+            project.clone(),
+            process.clone(),
+            launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
+        );
+        self.record_process_started(&tenant, &project, &process, now_epoch_seconds);
         Ok(CoordinatorResponse::ProcessStarted {
             process,
+            launch_attempt: active
+                .launch_attempt
+                .as_ref()
+                .map(|attempt| attempt.as_str().to_owned()),
             epoch: self.coordinator.coordinator_epoch(),
             actor,
             charged_spawns,
@@ -404,14 +416,18 @@ impl CoordinatorService {
 
     pub(super) fn handle_reconnect_node(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         process: String,
         epoch: u64,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
         let process = ProcessId::new(process);
         self.coordinator
-            .reconnect_node(&node, Some((&process, epoch)))?;
+            .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
         Ok(CoordinatorResponse::NodeReconnected { node, process })
     }
 
@@ -502,6 +518,13 @@ impl CoordinatorService {
         }
         let process_key = process_control_key(&tenant, &project, &process);
         if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Cancelled,
+                self.current_epoch_seconds()?,
+            );
             self.coordinator
                 .abort_process(&tenant, &project, &process)?;
             self.clear_operator_panel_state(&tenant, &project, &process);
@@ -519,6 +542,7 @@ impl CoordinatorService {
         project: String,
         actor_user: String,
         process: String,
+        launch_attempt: Option<String>,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let tenant = TenantId::new(tenant);
         let project = ProjectId::new(project);
@@ -534,6 +558,17 @@ impl CoordinatorService {
             })?;
         debug_assert_eq!(active.tenant, tenant);
         debug_assert_eq!(active.project, project);
+        let launch_attempt = launch_attempt.map(clusterflux_core::LaunchAttemptId::new);
+        if let Some(expected) = launch_attempt.as_ref() {
+            if active.launch_attempt.as_ref() != Some(expected) {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "launch rollback denied: attempt {} does not own process {}",
+                    expected.as_str(),
+                    process.as_str()
+                ))
+                .into());
+            }
+        }
 
         let process_key = process_control_key(&tenant, &project, &process);
         self.process_cancellations.remove(&process_key);
@@ -575,8 +610,24 @@ impl CoordinatorService {
             }
         }
 
-        self.coordinator
+        self.record_process_terminal(
-            .abort_process(&tenant, &project, &process)?;
+            &tenant,
+            &project,
+            &process,
+            super::ProcessFinalResult::Cancelled,
+            self.current_epoch_seconds()?,
+        );
+        if let Some(launch_attempt) = launch_attempt.as_ref() {
+            self.coordinator.abort_process_for_launch_attempt(
+                &tenant,
+                &project,
+                &process,
+                launch_attempt,
+            )?;
+        } else {
+            self.coordinator
+                .abort_process(&tenant, &project, &process)?;
+        }
         let active_restart_tasks = aborted_tasks
             .iter()
             .map(|target| target.task.clone())
@@ -624,10 +675,18 @@ impl CoordinatorService {
             .map(|active| {
                 let process_key = process_control_key(&active.tenant, &active.project, &active.id);
                 let main = self.main_runtime.controls.get(&process_key);
+                let stored = self.process_summaries.get(&process_key);
+                let stored_main_state = stored
+                    .and_then(|summary| summary.main_terminal_state.as_ref())
+                    .map(|state| match state {
+                        super::TaskTerminalState::Completed => "completed",
+                        super::TaskTerminalState::Failed => "failed",
+                        super::TaskTerminalState::Cancelled => "cancelled",
+                    });
                 let state = if self.process_cancellations.contains(&process_key) {
                     "cancelling"
                 } else {
-                    main.map_or("running", |main| main.state.as_str())
+                    "running"
                 };
                 let main_wait_state = main.and_then(|main| {
                     if main.state != "running" {
@@ -652,9 +711,15 @@ impl CoordinatorService {
                 VirtualProcessStatus {
                     process: active.id,
                     state: state.to_owned(),
-                    main_task_definition: main.map(|main| main.task_definition.clone()),
+                    main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
-                    main_task_instance: main.map(|main| main.task_instance.clone()),
+                        || stored.and_then(|summary| summary.main_task_definition.clone()),
-                    main_state: main.map(|main| main.state.clone()),
+                    ),
+                    main_task_instance: main
+                        .map(|main| main.task_instance.clone())
+                        .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
+                    main_state: main
+                        .map(|main| main.state.clone())
+                        .or_else(|| stored_main_state.map(str::to_owned)),
                     main_wait_state,
                     main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
                     connected_nodes: active.connected_nodes.into_iter().collect(),

crates/clusterflux-coordinator/src/service/protocol/responses.rs

@@ -183,6 +183,121 @@ pub struct VirtualProcessStatus {
     pub coordinator_epoch: u64,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum SourcePreparationDisposition {
     Pending { reason: String },
@@ -213,7 +328,7 @@ pub enum CoordinatorResponse {
         manual_review: bool,
         sanitized_reason: Option<String>,
         next_actions: Vec<String>,
-        private_moderation_details_exposed: bool,
+        sensitive_moderation_details_exposed: bool,
         signup_failure_details_exposed: bool,
     },
     AdminStatus {
@@ -282,6 +397,11 @@ pub enum CoordinatorResponse {
         descriptors: Vec<NodeDescriptor>,
         actor: UserId,
     },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     NodeCredentialRevoked {
         node: NodeId,
         tenant: TenantId,
@@ -344,6 +464,8 @@ pub enum CoordinatorResponse {
     },
     ProcessStarted {
         process: ProcessId,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        launch_attempt: Option<String>,
         epoch: u64,
         actor: WorkflowActor,
         charged_spawns: u64,
@@ -371,6 +493,11 @@ pub enum CoordinatorResponse {
         processes: Vec<VirtualProcessStatus>,
         actor: UserId,
     },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     QuotaStatus {
         tenant: TenantId,
         project: ProjectId,
@@ -428,6 +555,7 @@ pub enum CoordinatorResponse {
     DebugBreakpoints {
         process: ProcessId,
         actor: UserId,
+        revision: u64,
         probe_symbols: Vec<String>,
         hit_epoch: Option<u64>,
         hit_task: Option<TaskInstanceId>,
@@ -480,6 +608,17 @@ pub enum CoordinatorResponse {
         stderr_tail: String,
         backpressured: bool,
     },
+    TaskLogChunkRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        sequence: Option<u64>,
+        next_offset: u64,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
     VfsMetadataRecorded {
         process: ProcessId,
         task: TaskInstanceId,
@@ -516,6 +655,13 @@ pub enum CoordinatorResponse {
     ArtifactDownloadLink {
         link: DownloadLink,
     },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
     ArtifactDownloadLinkRevoked {
         link: DownloadLink,
     },
@@ -540,6 +686,24 @@ pub enum CoordinatorResponse {
         artifact_size_bytes: u64,
     },
     Error {
-        message: String,
+        #[serde(flatten)]
+        error: clusterflux_core::ApiError,
     },
 }
+
+impl CoordinatorResponse {
+    pub fn error(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        Self::Error {
+            error: clusterflux_core::ApiError::from_message(request_id, message),
+        }
+    }
+
+    pub fn service_error(
+        request_id: impl Into<String>,
+        error: &crate::service::CoordinatorServiceError,
+    ) -> Self {
+        Self::Error {
+            error: error.api_error(request_id),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/quota.rs

@@ -260,22 +260,6 @@ impl CoordinatorQuota {
         self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
     }
 
-    pub(super) fn can_charge_log_bytes(
-        &self,
-        tenant: &TenantId,
-        project: &ProjectId,
-        bytes: u64,
-        now_epoch_seconds: u64,
-    ) -> Result<(), LimitError> {
-        self.can_charge(
-            tenant,
-            project,
-            LimitKind::LogBytes,
-            bytes,
-            now_epoch_seconds,
-        )
-    }
-
     pub(super) fn charge_log_bytes(
         &mut self,
         tenant: &TenantId,

crates/clusterflux-coordinator/src/service/relay.rs

@@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage {
     pub egress_bytes: u64,
     pub abandoned_or_failed_bytes: u64,
     pub reserved_bytes: u64,
+    pub lifetime_ingress_bytes: u64,
+    pub lifetime_egress_bytes: u64,
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    pub completed_transfers: u64,
+    pub failed_transfers: u64,
+    pub cancelled_transfers: u64,
+    pub expired_transfers: u64,
+    pub tracked_scopes: usize,
+    pub max_active_global: usize,
+    pub max_tracked_scopes: usize,
 }
 
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
@@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState {
     pub ingress_used: u64,
     pub egress_used: u64,
     pub abandoned_or_failed_used: u64,
+    #[serde(default)]
+    pub lifetime_ingress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_egress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    #[serde(default)]
+    pub completed_transfers: u64,
+    #[serde(default)]
+    pub failed_transfers: u64,
+    #[serde(default)]
+    pub cancelled_transfers: u64,
+    #[serde(default)]
+    pub expired_transfers: u64,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger {
     ingress_used: u64,
     egress_used: u64,
     abandoned_or_failed_used: u64,
+    lifetime_ingress_bytes: u64,
+    lifetime_egress_bytes: u64,
+    lifetime_abandoned_or_failed_bytes: u64,
+    completed_transfers: u64,
+    failed_transfers: u64,
+    cancelled_transfers: u64,
+    expired_transfers: u64,
 }
 
 impl Default for ArtifactRelayLedger {
@@ -165,6 +196,13 @@ impl ArtifactRelayLedger {
             ingress_used: 0,
             egress_used: 0,
             abandoned_or_failed_used: 0,
+            lifetime_ingress_bytes: 0,
+            lifetime_egress_bytes: 0,
+            lifetime_abandoned_or_failed_bytes: 0,
+            completed_transfers: 0,
+            failed_transfers: 0,
+            cancelled_transfers: 0,
+            expired_transfers: 0,
         }
     }
 
@@ -218,6 +256,13 @@ impl ArtifactRelayLedger {
             ingress_used: state.ingress_used,
             egress_used: state.egress_used,
             abandoned_or_failed_used: state.abandoned_or_failed_used,
+            lifetime_ingress_bytes: state.lifetime_ingress_bytes,
+            lifetime_egress_bytes: state.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: state.completed_transfers,
+            failed_transfers: state.failed_transfers,
+            cancelled_transfers: state.cancelled_transfers,
+            expired_transfers: state.expired_transfers,
         }
     }
 
@@ -260,6 +305,13 @@ impl ArtifactRelayLedger {
             ingress_used: self.ingress_used,
             egress_used: self.egress_used,
             abandoned_or_failed_used: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
         }
     }
 
@@ -491,9 +543,11 @@ impl ArtifactRelayLedger {
         if ingress {
             reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
             self.ingress_used = self.ingress_used.saturating_add(bytes);
+            self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
         } else {
             reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
             self.egress_used = self.egress_used.saturating_add(bytes);
+            self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
         }
         let project_key = (
             reservation.scope.tenant.clone(),
@@ -554,10 +608,29 @@ impl ArtifactRelayLedger {
     pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
         if let Some(reservation) = self.reservations.remove(key) {
             if reason != RelayFinishReason::Completed {
+                let abandoned_or_failed_bytes = reservation
+                    .ingress_bytes
+                    .saturating_add(reservation.egress_bytes);
                 self.abandoned_or_failed_used = self
                     .abandoned_or_failed_used
-                    .saturating_add(reservation.ingress_bytes)
+                    .saturating_add(abandoned_or_failed_bytes);
-                    .saturating_add(reservation.egress_bytes);
+                self.lifetime_abandoned_or_failed_bytes = self
+                    .lifetime_abandoned_or_failed_bytes
+                    .saturating_add(abandoned_or_failed_bytes);
+            }
+            match reason {
+                RelayFinishReason::Completed => {
+                    self.completed_transfers = self.completed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Failed => {
+                    self.failed_transfers = self.failed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Cancelled => {
+                    self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Expired => {
+                    self.expired_transfers = self.expired_transfers.saturating_add(1);
+                }
             }
         }
     }
@@ -580,6 +653,18 @@ impl ArtifactRelayLedger {
             ingress_bytes: self.ingress_used,
             egress_bytes: self.egress_used,
             abandoned_or_failed_bytes: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
+            tracked_scopes: self.project_used.len()
+                + self.tenant_used.len()
+                + self.account_used.len(),
+            max_active_global: self.configuration.max_active_global,
+            max_tracked_scopes: self.configuration.max_tracked_scopes,
             reserved_bytes: self
                 .reservations
                 .values()
@@ -588,3 +673,68 @@ impl ArtifactRelayLedger {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
+        let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
+        ledger
+            .reserve(
+                "transfer".to_owned(),
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                16,
+                16,
+                100,
+                1,
+            )
+            .unwrap();
+        ledger.charge_ingress("transfer", 24, 1).unwrap();
+        ledger.charge_egress("transfer", 24, 1).unwrap();
+        ledger.finish("transfer", RelayFinishReason::Completed);
+
+        let usage = ledger.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+
+        ledger.prepare_period(u64::MAX);
+        let usage = ledger.usage();
+        assert_eq!(usage.ingress_bytes, 0);
+        assert_eq!(usage.egress_bytes, 0);
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+
+        let restored = ArtifactRelayLedger::from_durable(
+            ArtifactRelayConfiguration::unlimited(),
+            ledger.durable_state(),
+        );
+        let usage = restored.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+    }
+
+    #[test]
+    fn older_durable_relay_state_defaults_new_metrics() {
+        let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
+            "reservations": {},
+            "period": 1,
+            "project_used": [],
+            "tenant_used": [],
+            "account_used": [],
+            "ingress_used": 3,
+            "egress_used": 4,
+            "abandoned_or_failed_used": 2
+        }))
+        .unwrap();
+
+        assert_eq!(state.lifetime_ingress_bytes, 0);
+        assert_eq!(state.lifetime_egress_bytes, 0);
+        assert_eq!(state.completed_transfers, 0);
+    }
+}

crates/clusterflux-coordinator/src/service/routing.rs

@@ -5,6 +5,9 @@ impl CoordinatorService {
         &mut self,
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        request
+            .validate_external_identifiers()
+            .map_err(CoordinatorServiceError::Protocol)?;
         self.pump_main_runtime_commands();
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
@@ -42,7 +45,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -271,9 +274,17 @@ impl CoordinatorService {
                 enrollment_grant,
             ),
             CoordinatorRequest::NodeHeartbeat {
+                tenant,
+                project,
                 node,
                 node_signature,
-            } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
+            } => self.handle_node_heartbeat(
+                tenant,
+                project,
+                node,
+                node_signature,
+                &request_payload_digest,
+            ),
             CoordinatorRequest::SignedNode {
                 node,
                 node_signature,
@@ -287,6 +298,13 @@ impl CoordinatorService {
                 project,
                 actor_user,
             } => self.handle_list_node_descriptors(tenant, project, actor_user),
+            CoordinatorRequest::ListNodeSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::RevokeNodeCredential {
                 tenant,
                 project,
@@ -370,6 +388,7 @@ impl CoordinatorService {
                 agent_public_key_fingerprint,
                 agent_signature,
                 process,
+                launch_attempt,
                 restart,
             } => self.handle_start_process(
                 tenant,
@@ -380,6 +399,7 @@ impl CoordinatorService {
                 agent_signature,
                 Some(&request_payload_digest),
                 process,
+                launch_attempt,
                 restart,
             ),
             CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
@@ -401,12 +421,20 @@ impl CoordinatorService {
                 project,
                 actor_user,
                 process,
-            } => self.handle_abort_process(tenant, project, actor_user, process),
+                launch_attempt,
+            } => self.handle_abort_process(tenant, project, actor_user, process, launch_attempt),
             CoordinatorRequest::ListProcesses {
                 tenant,
                 project,
                 actor_user,
             } => self.handle_list_processes(tenant, project, actor_user),
+            CoordinatorRequest::ListProcessSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::QuotaStatus {
                 tenant,
                 project,
@@ -452,7 +480,8 @@ impl CoordinatorService {
             CoordinatorRequest::PollDebugCommand { .. }
             | CoordinatorRequest::ReportDebugState { .. }
             | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
-            CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportTaskLog { .. }
+            | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ListTaskEvents {
@@ -467,6 +496,23 @@ impl CoordinatorService {
                 actor_user,
                 process,
             } => self.handle_list_task_snapshots(tenant, project, actor_user, process),
+            CoordinatorRequest::ListRecentLogs {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             CoordinatorRequest::JoinTask {
                 tenant,
                 project,
@@ -513,6 +559,20 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            CoordinatorRequest::ListArtifacts {
+                tenant,
+                project,
+                actor_user,
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit),
+            CoordinatorRequest::GetArtifact {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+            } => self.handle_get_artifact(tenant, project, actor_user, artifact),
             CoordinatorRequest::OpenArtifactDownloadStream {
                 tenant,
                 project,
@@ -595,7 +655,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -682,6 +742,14 @@ impl CoordinatorService {
                     context.project.as_str().to_owned(),
                     actor.as_str().to_owned(),
                 ),
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self
+                .handle_list_node_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
                 .handle_revoke_node_credential(
                     context.tenant.as_str().to_owned(),
@@ -689,18 +757,22 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     node,
                 ),
-            AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
+            AuthenticatedCoordinatorRequest::StartProcess {
-                .handle_start_process(
+                process,
-                    context.tenant.as_str().to_owned(),
+                launch_attempt,
-                    context.project.as_str().to_owned(),
+                restart,
-                    Some(actor.as_str().to_owned()),
+            } => self.handle_start_process(
-                    None,
+                context.tenant.as_str().to_owned(),
-                    None,
+                context.project.as_str().to_owned(),
-                    None,
+                Some(actor.as_str().to_owned()),
-                    None,
+                None,
-                    process,
+                None,
-                    restart,
+                None,
-                ),
+                None,
+                process,
+                launch_attempt,
+                restart,
+            ),
             request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
                 self.handle_authenticated_schedule_task(&context, request)
             }
@@ -714,17 +786,29 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     process,
                 ),
-            AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
+            AuthenticatedCoordinatorRequest::AbortProcess {
+                process,
+                launch_attempt,
+            } => self.handle_abort_process(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
                 actor.as_str().to_owned(),
                 process,
+                launch_attempt,
             ),
             AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
                 actor.as_str().to_owned(),
             ),
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self
+                .handle_list_process_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -780,6 +864,20 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     process,
                 ),
+            AuthenticatedCoordinatorRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -799,6 +897,24 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            AuthenticatedCoordinatorRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                cursor,
+                limit,
+            ),
+            AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+            ),
             AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
                 artifact,
                 max_bytes,

crates/clusterflux-coordinator/src/service/signed_nodes.rs

@@ -1,6 +1,6 @@
-use clusterflux_core::{NodeId, NodeSignedRequest};
+use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
 
@@ -12,22 +12,24 @@ impl CoordinatorService {
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let request_kind = signed_node_request_kind(&request)?;
-        let request_node = signed_node_request_node(&request)?;
+        let request_scope = signed_node_request_scope(&request)?;
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
                 "failed to canonicalize signed node request: {error}"
             ))
         })?;
         let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
-        let signed_node = NodeId::new(signed_node);
+        let signed_node = NodeId::try_new(signed_node).map_err(|error| {
-        if request_node != signed_node {
+            CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
+        })?;
+        if request_scope.node != signed_node {
             return Err(CoordinatorError::Unauthorized(
                 "signed node request node does not match the wrapped request node".to_owned(),
             )
             .into());
         }
         self.authenticate_node_request(
-            &signed_node,
+            &request_scope,
             Some(node_signature),
             request_kind,
             &payload_digest,
@@ -144,11 +146,26 @@ impl CoordinatorService {
                 provider,
                 source_snapshot,
             ),
+            CoordinatorRequest::RequestRendezvous {
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_request_rendezvous(
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            ),
             CoordinatorRequest::ReconnectNode {
+                tenant,
+                project,
                 node,
                 process,
                 epoch,
-            } => self.handle_reconnect_node(node, process, epoch),
+            } => self.handle_reconnect_node(tenant, project, node, process, epoch),
             CoordinatorRequest::PollTaskControl {
                 tenant,
                 project,
@@ -236,6 +253,29 @@ impl CoordinatorService {
                 stderr_truncated,
                 backpressured,
             ),
+            CoordinatorRequest::ReportTaskLogChunk {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            } => self.handle_report_task_log_chunk(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            ),
             CoordinatorRequest::ReportVfsMetadata {
                 tenant,
                 project,
@@ -322,12 +362,14 @@ fn signed_node_request_kind(
         CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
         CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
         CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
+        CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"),
         CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
         CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
         CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
         CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
         CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
         CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
+        CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"),
         CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
         CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
         _ => Err(CoordinatorError::Unauthorized(
@@ -337,29 +379,135 @@ fn signed_node_request_kind(
     }
 }
 
-fn signed_node_request_node(
+fn signed_node_request_scope(
     request: &CoordinatorRequest,
-) -> Result<NodeId, CoordinatorServiceError> {
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
     match request {
-        CoordinatorRequest::ReportNodeCapabilities { node, .. }
+        CoordinatorRequest::ReportNodeCapabilities {
-        | CoordinatorRequest::PollTaskAssignment { node, .. }
+            tenant,
-        | CoordinatorRequest::PollArtifactTransfer { node, .. }
+            project,
-        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
+            node,
-        | CoordinatorRequest::FailArtifactTransfer { node, .. }
+            ..
-        | CoordinatorRequest::LaunchChildTask { node, .. }
+        }
-        | CoordinatorRequest::JoinChildTask { node, .. }
+        | CoordinatorRequest::PollTaskAssignment {
-        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
+            tenant,
-        | CoordinatorRequest::ReconnectNode { node, .. }
+            project,
-        | CoordinatorRequest::PollTaskControl { node, .. }
+            node,
-        | CoordinatorRequest::PollDebugCommand { node, .. }
+        }
-        | CoordinatorRequest::ReportDebugState { node, .. }
+        | CoordinatorRequest::PollArtifactTransfer {
-        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
+            tenant,
-        | CoordinatorRequest::ReportTaskLog { node, .. }
+            project,
-        | CoordinatorRequest::ReportVfsMetadata { node, .. }
+            node,
-        | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
+        }
+        | CoordinatorRequest::UploadArtifactTransferChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::FailArtifactTransfer {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::LaunchChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::JoinChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::CompleteSourcePreparation {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReconnectNode {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollTaskControl {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollDebugCommand {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugState {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugProbeHit {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLog {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLogChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportVfsMetadata {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::TaskCompleted {
+            tenant,
+            project,
+            node,
+            ..
+        } => node_scope_from_strings(tenant, project, node),
+        CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
+            scope.tenant.clone(),
+            scope.project.clone(),
+            source.node.clone(),
+        )),
         _ => Err(CoordinatorError::Unauthorized(
             "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
         )
         .into()),
     }
 }
+
+fn node_scope_from_strings(
+    tenant: &str,
+    project: &str,
+    node: &str,
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
+    let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
+    })?;
+    let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
+    })?;
+    let node = NodeId::try_new(node.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
+    })?;
+    Ok(NodeScopeKey::new(tenant, project, node))
+}

crates/clusterflux-coordinator/src/service/summaries.rs

@@ -0,0 +1,565 @@
+use std::collections::BTreeSet;
+use std::time::Instant;
+
+use clusterflux_core::{
+    ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId,
+};
+
+use super::keys::{process_control_key, ProcessControlKey};
+use super::{
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse,
+    CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
+    TaskAttemptState,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct StoredProcessSummary {
+    pub(super) started_at_epoch_seconds: u64,
+    pub(super) ended_at_epoch_seconds: Option<u64>,
+    pub(super) final_result: Option<ProcessFinalResult>,
+    pub(super) connected_nodes: Vec<NodeId>,
+    pub(super) main_task_definition: Option<TaskDefinitionId>,
+    pub(super) main_task_instance: Option<TaskInstanceId>,
+    pub(super) main_terminal_state: Option<super::TaskTerminalState>,
+    pub(super) order: u64,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_list_node_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = cursor.as_deref();
+        let mut nodes = self
+            .node_descriptors
+            .iter()
+            .filter(|(scope, descriptor)| {
+                scope.tenant == tenant
+                    && scope.project == project
+                    && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor)
+            })
+            .map(|(scope, descriptor)| {
+                let online = self.node_is_live(scope);
+                NodeSummary {
+                    id: descriptor.id.clone(),
+                    display_name: descriptor.id.as_str().to_owned(),
+                    online,
+                    stale: !online,
+                    last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(),
+                    capabilities: descriptor.capabilities.clone(),
+                    direct_connectivity: descriptor.direct_connectivity,
+                }
+            })
+            .collect::<Vec<_>>();
+        nodes.sort_by(|left, right| left.id.cmp(&right.id));
+        let has_more = nodes.len() > limit as usize;
+        nodes.truncate(limit as usize);
+        let next_cursor = has_more
+            .then(|| nodes.last().map(|node| node.id.as_str().to_owned()))
+            .flatten();
+        Ok(CoordinatorResponse::NodeSummaries {
+            nodes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    pub(super) fn record_process_started(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) {
+            logs.retain(|entry| &entry.process != process);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(tenant.clone(), project.clone()));
+            }
+        }
+        self.recent_log_dropped_through.remove(&key);
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, _, _), _| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.process_summary_order
+            .retain(|retained| retained != &key);
+        self.evict_process_summaries_for_project(tenant, project);
+        self.evict_process_summaries_total();
+        let order = self.next_process_summary_order;
+        self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+        self.process_summaries.insert(
+            key.clone(),
+            StoredProcessSummary {
+                started_at_epoch_seconds: now_epoch_seconds,
+                ended_at_epoch_seconds: None,
+                final_result: None,
+                connected_nodes: Vec::new(),
+                main_task_definition: None,
+                main_task_instance: None,
+                main_terminal_state: None,
+                order,
+            },
+        );
+        self.process_summary_order.push_back(key);
+    }
+
+    pub(super) fn record_process_terminal(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        final_result: ProcessFinalResult,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        let connected_nodes = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or_default();
+        let order = self.next_process_summary_order;
+        let entry = self
+            .process_summaries
+            .entry(key.clone())
+            .or_insert_with(|| {
+                self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+                self.process_summary_order.push_back(key.clone());
+                StoredProcessSummary {
+                    started_at_epoch_seconds: now_epoch_seconds,
+                    ended_at_epoch_seconds: None,
+                    final_result: None,
+                    connected_nodes: Vec::new(),
+                    main_task_definition: None,
+                    main_task_instance: None,
+                    main_terminal_state: None,
+                    order,
+                }
+            });
+        entry.ended_at_epoch_seconds = Some(now_epoch_seconds);
+        entry.final_result = Some(final_result);
+        entry.connected_nodes = connected_nodes;
+    }
+
+    pub(super) fn record_main_terminal_state(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task_definition: TaskDefinitionId,
+        task_instance: TaskInstanceId,
+        terminal_state: super::TaskTerminalState,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if !self.process_summaries.contains_key(&key) {
+            self.record_process_started(
+                tenant,
+                project,
+                process,
+                self.liveness_now_epoch_seconds(),
+            );
+        }
+        if let Some(summary) = self.process_summaries.get_mut(&key) {
+            summary.main_task_definition = Some(task_definition);
+            summary.main_task_instance = Some(task_instance);
+            summary.main_terminal_state = Some(terminal_state);
+        }
+    }
+
+    pub(super) fn handle_list_process_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = parse_order_cursor(cursor.as_deref(), "process")?;
+        let mut stored = self
+            .process_summaries
+            .iter()
+            .filter(|((entry_tenant, entry_project, _), summary)| {
+                entry_tenant == &tenant
+                    && entry_project == &project
+                    && cursor.is_none_or(|cursor| summary.order < cursor)
+            })
+            .map(|(key, summary)| (key.clone(), summary.clone()))
+            .collect::<Vec<_>>();
+        stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order));
+        let has_more = stored.len() > limit as usize;
+        stored.truncate(limit as usize);
+        let processes = stored
+            .into_iter()
+            .map(|(key, stored)| self.process_summary_from_stored(&key, stored))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| processes.last().map(|process| process.order_cursor.clone()))
+            .flatten();
+        Ok(CoordinatorResponse::ProcessSummaries {
+            processes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    fn process_summary_from_stored(
+        &self,
+        key: &ProcessControlKey,
+        stored: StoredProcessSummary,
+    ) -> ProcessSummary {
+        let (tenant, project, process) = key;
+        let active = self.coordinator.active_process(tenant, project, process);
+        let process_key = process_control_key(tenant, project, process);
+        let main_wait_state = active.and_then(|_| {
+            if self.pending_task_launches.iter().any(|pending| {
+                &pending.tenant == tenant
+                    && &pending.project == project
+                    && &pending.process == process
+            }) {
+                Some("waiting_for_node".to_owned())
+            } else if self
+                .main_runtime
+                .is_waiting_for_task(tenant, project, process)
+            {
+                Some("waiting_for_task".to_owned())
+            } else {
+                None
+            }
+        });
+        let current_debug_epoch = self.debug_epoch_summary(&process_key);
+        let awaiting_action = self.task_attempts.iter().any(
+            |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                attempt_tenant == tenant
+                    && attempt_project == project
+                    && attempt_process == process
+                    && attempts.iter().any(|attempt| {
+                        attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction
+                    })
+            },
+        );
+        let activity = if let Some(result) = &stored.final_result {
+            match result {
+                ProcessFinalResult::Completed => ProcessActivityState::Completed,
+                ProcessFinalResult::Failed => ProcessActivityState::Failed,
+                ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled,
+            }
+        } else if self.process_cancellations.contains(&process_key) {
+            ProcessActivityState::Cancelling
+        } else if current_debug_epoch
+            .as_ref()
+            .is_some_and(|epoch| epoch.partially_frozen)
+        {
+            ProcessActivityState::DebugEpochPartial
+        } else if awaiting_action {
+            ProcessActivityState::AwaitingAction
+        } else {
+            match main_wait_state.as_deref() {
+                Some("waiting_for_node") => ProcessActivityState::WaitingForNode,
+                Some("waiting_for_task") => ProcessActivityState::WaitingForTask,
+                _ => ProcessActivityState::Running,
+            }
+        };
+        let connected_nodes = active
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or(stored.connected_nodes);
+        ProcessSummary {
+            process: process.clone(),
+            lifecycle: if active.is_some() {
+                ProcessLifecycleState::Active
+            } else {
+                ProcessLifecycleState::RecentTerminal
+            },
+            activity,
+            main_wait_state,
+            started_at_epoch_seconds: stored.started_at_epoch_seconds,
+            ended_at_epoch_seconds: stored.ended_at_epoch_seconds,
+            final_result: stored.final_result,
+            connected_nodes,
+            current_debug_epoch,
+            order_cursor: format!("process:{}", stored.order),
+        }
+    }
+
+    fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option<DebugEpochSummary> {
+        let runtime = self.debug_epoch_runtime.get(key)?;
+        let acknowledgements = runtime.acknowledgements.values().collect::<Vec<_>>();
+        let all_acknowledged = !runtime.expected.is_empty()
+            && runtime
+                .expected
+                .iter()
+                .all(|participant| runtime.acknowledgements.contains_key(participant));
+        let fully_frozen = runtime.command == "freeze"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Frozen);
+        let freeze_deadline_elapsed =
+            runtime.command == "freeze" && Instant::now() >= runtime.deadline;
+        let frozen_count = acknowledgements
+            .iter()
+            .filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
+            .count();
+        let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen;
+        let fully_resumed = runtime.command == "resume"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Running);
+        let failed = acknowledgements
+            .iter()
+            .any(|ack| ack.state == DebugAcknowledgementState::Failed)
+            || (freeze_deadline_elapsed
+                && runtime
+                    .expected
+                    .iter()
+                    .any(|participant| !runtime.acknowledgements.contains_key(participant)));
+        Some(DebugEpochSummary {
+            epoch: runtime.epoch,
+            command: runtime.command.clone(),
+            fully_frozen,
+            partially_frozen,
+            fully_resumed,
+            failed,
+        })
+    }
+
+    pub(super) fn handle_list_artifacts(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: Option<String>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = process.map(ProcessId::new);
+        if let Some(process) = &process {
+            self.authorize_task_event_process_scope(&tenant, &project, process)?;
+        }
+        let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?;
+        let mut metadata = self
+            .artifact_registry
+            .metadata_for_project(&tenant, &project)
+            .filter(|metadata| {
+                process
+                    .as_ref()
+                    .is_none_or(|process| &metadata.process == process)
+                    && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor)
+            })
+            .cloned()
+            .collect::<Vec<_>>();
+        metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch));
+        let has_more = metadata.len() > limit as usize;
+        metadata.truncate(limit as usize);
+        let artifacts = metadata
+            .into_iter()
+            .map(|metadata| self.artifact_summary(metadata))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| {
+                artifacts
+                    .last()
+                    .map(|artifact| artifact.order_cursor.clone())
+            })
+            .flatten();
+        Ok(CoordinatorResponse::Artifacts {
+            artifacts,
+            next_cursor,
+        })
+    }
+
+    pub(super) fn handle_get_artifact(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let metadata = self
+            .artifact_registry
+            .metadata(&tenant, &project, &artifact)
+            .cloned()
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        Ok(CoordinatorResponse::Artifact {
+            artifact: self.artifact_summary(metadata),
+        })
+    }
+
+    fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary {
+        let live_retaining_nodes = metadata
+            .retaining_nodes
+            .iter()
+            .filter(|node| {
+                self.node_is_live(&crate::NodeScopeKey::from_refs(
+                    &metadata.tenant,
+                    &metadata.project,
+                    node,
+                ))
+            })
+            .cloned()
+            .collect::<BTreeSet<_>>();
+        let safe_node = live_retaining_nodes
+            .iter()
+            .next()
+            .cloned()
+            .or_else(|| metadata.retaining_nodes.iter().next().cloned());
+        let explicit_storage = !metadata.explicit_locations.is_empty();
+        let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage;
+        let availability = if downloadable_now {
+            ArtifactAvailability::Available
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactAvailability::NodeOffline
+        } else {
+            ArtifactAvailability::Unavailable
+        };
+        let retention_state = if explicit_storage {
+            ArtifactRetentionState::ExplicitStorage
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactRetentionState::NodeRetained
+        } else {
+            ArtifactRetentionState::Lost
+        };
+        let display_suffix = metadata.id.as_str().replace(':', "/");
+        let display_path = format!("/vfs/artifacts/{display_suffix}");
+        let display_name = display_suffix
+            .rsplit('/')
+            .next()
+            .unwrap_or(metadata.id.as_str())
+            .to_owned();
+        ArtifactSummary {
+            id: metadata.id,
+            display_path,
+            display_name,
+            process: metadata.process,
+            producer_task: metadata.producer_task,
+            safe_node,
+            digest: metadata.digest,
+            size_bytes: metadata.size,
+            availability,
+            downloadable_now,
+            retention_state,
+            explicit_storage,
+            order_cursor: format!("artifact:{}", metadata.flushed_epoch),
+        }
+    }
+
+    fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) {
+        while self
+            .process_summaries
+            .keys()
+            .filter(|(entry_tenant, entry_project, _)| {
+                entry_tenant == tenant && entry_project == project
+            })
+            .count()
+            >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT
+        {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                &key.0 == tenant
+                    && &key.1 == project
+                    && self
+                        .process_summaries
+                        .get(*key)
+                        .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn evict_process_summaries_total(&mut self) {
+        while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                self.process_summaries
+                    .get(*key)
+                    .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
+        self.process_summaries.remove(key);
+        self.recent_log_dropped_through.remove(key);
+        if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
+            logs.retain(|entry| entry.process != key.2);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
+            }
+        }
+        self.recent_log_accounted_bytes
+            .retain(|(tenant, project, process, _, _), _| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_quota_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.process_summary_order
+            .retain(|retained| retained != key);
+    }
+}
+
+fn parse_order_cursor(
+    cursor: Option<&str>,
+    expected_kind: &str,
+) -> Result<Option<u64>, CoordinatorServiceError> {
+    cursor
+        .map(|cursor| {
+            let (kind, order) = cursor.split_once(':').ok_or_else(|| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })?;
+            if kind != expected_kind {
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                )));
+            }
+            order.parse::<u64>().map_err(|_| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })
+        })
+        .transpose()
+}

crates/clusterflux-coordinator/src/service/tcp.rs

@@ -79,17 +79,15 @@ impl CoordinatorService {
                 continue;
             }
             let response = match decode_wire_request(&line) {
-                Ok(request) => match authorize_client_request(&request, authority_mode)
+                Ok((request_id, request)) => {
-                    .and_then(|()| self.handle_request(request))
+                    match authorize_client_request(&request, authority_mode)
-                {
+                        .and_then(|()| self.handle_request(request))
-                    Ok(response) => response,
+                    {
-                    Err(err) => CoordinatorResponse::Error {
+                        Ok(response) => response,
-                        message: err.to_string(),
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                    },
+                    }
-                },
+                }
-                Err(err) => CoordinatorResponse::Error {
+                Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
-                    message: err.to_string(),
-                },
             };
             serde_json::to_writer(&mut writer, &response)?;
             writer.write_all(b"\n")?;
@@ -114,25 +112,19 @@ fn handle_shared_stream(
             continue;
         }
         let response = match decode_wire_request(&line) {
-            Ok(request) => match authorize_client_request(&request, authority_mode) {
+            Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) {
                 Ok(()) => match service.lock() {
                     Ok(mut service) => match service.handle_request(request) {
                         Ok(response) => response,
-                        Err(err) => CoordinatorResponse::Error {
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                            message: err.to_string(),
-                        },
-                    },
-                    Err(_) => CoordinatorResponse::Error {
-                        message: "coordinator service lock poisoned".to_owned(),
                     },
+                    Err(_) => {
+                        CoordinatorResponse::error(request_id, "coordinator service lock poisoned")
+                    }
                 },
-                Err(err) => CoordinatorResponse::Error {
+                Err(err) => CoordinatorResponse::service_error(request_id, &err),
-                    message: err.to_string(),
-                },
-            },
-            Err(err) => CoordinatorResponse::Error {
-                message: err.to_string(),
             },
+            Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
         };
         serde_json::to_writer(&mut writer, &response)?;
         writer.write_all(b"\n")?;
@@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato
     Ok((listener, addr))
 }
 
-fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
+fn decode_wire_request(
+    line: &str,
+) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> {
     serde_json::from_str::<super::CoordinatorWireRequest>(line)?
-        .into_request()
+        .into_parts()
         .map_err(CoordinatorServiceError::Protocol)
 }
 
+fn wire_request_id_hint(line: &str) -> String {
+    serde_json::from_str::<serde_json::Value>(line)
+        .ok()
+        .and_then(|value| {
+            value
+                .get("request_id")
+                .and_then(|value| value.as_str())
+                .map(str::to_owned)
+        })
+        .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok())
+        .unwrap_or_else(|| "unavailable".to_owned())
+}
+
 fn authorize_client_request(
     request: &CoordinatorRequest,
     authority_mode: ClientAuthorityMode,
@@ -180,15 +187,21 @@ fn authorize_client_request(
         }
         | CoordinatorRequest::AdminStatus { .. }
         | CoordinatorRequest::SuspendTenant { .. } => Ok(()),
-        _ => Err(CoordinatorServiceError::Protocol(
+        _ => Err(crate::CoordinatorError::Unauthorized(
             "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
                 .to_owned(),
-        )),
+        )
+        .into()),
     }
 }
 
 #[cfg(test)]
 mod transport_boundary_tests {
+    use std::io::{BufRead as _, BufReader};
+
+    use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId};
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -197,4 +210,99 @@ mod transport_boundary_tests {
         let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
         assert!(error.to_string().contains("restricted to loopback"));
     }
+
+    #[test]
+    fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() {
+        let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
+        let mut coordinator = CoordinatorService::new(11);
+        coordinator
+            .issue_cli_session(
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                "healthy-session",
+                None,
+            )
+            .unwrap();
+        let shared = Arc::new(Mutex::new(coordinator));
+        let server_shared = Arc::clone(&shared);
+        let server = std::thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap();
+        });
+
+        let mut stream = TcpStream::connect(addr).unwrap();
+        let mut reader = BufReader::new(stream.try_clone().unwrap());
+        for (index, malformed_process) in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0process".to_owned(),
+            "bad process!".to_owned(),
+            "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1),
+        ]
+        .into_iter()
+        .enumerate()
+        {
+            let malformed = coordinator_wire_request(
+                format!("malformed-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": {
+                        "type": "abort_process",
+                        "process": malformed_process,
+                        "launch_attempt": "valid-attempt"
+                    }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &malformed).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            let CoordinatorResponse::Error { error } =
+                serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
+            else {
+                panic!("malformed identifier request unexpectedly succeeded");
+            };
+            assert!(
+                error.message.contains("malformed external identifier")
+                    && error.message.contains("request.request.process"),
+                "unexpected malformed identifier response: {}",
+                error.message
+            );
+            assert_eq!(error.request_id, format!("malformed-{index}"));
+            assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
+
+            let valid = coordinator_wire_request(
+                format!("healthy-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": { "type": "auth_status" }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &valid).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            line.clear();
+            reader.read_line(&mut line).unwrap();
+            assert!(
+                matches!(
+                    serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
+                    CoordinatorResponse::AuthStatus {
+                        authenticated: true,
+                        ..
+                    }
+                ),
+                "valid authenticated traffic failed after malformed request {index}"
+            );
+        }
+
+        stream.shutdown(std::net::Shutdown::Both).unwrap();
+        server.join().unwrap();
+        assert!(!shared.is_poisoned());
+    }
 }

crates/clusterflux-coordinator/src/service/wire_protocol.rs

@@ -1,4 +1,4 @@
-use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
+use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest {
 
 impl CoordinatorWireRequest {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         match self {
-            Self::Envelope(envelope) => envelope.into_request(),
+            Self::Envelope(envelope) => envelope.into_parts(),
         }
     }
 }
@@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope {
 
 impl CoordinatorRequestEnvelope {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
             return Err(format!(
                 "unsupported coordinator wire request type {}; expected {}",
@@ -44,9 +52,9 @@ impl CoordinatorRequestEnvelope {
                 self.protocol_version, COORDINATOR_PROTOCOL_VERSION
             ));
         }
-        if self.request_id.trim().is_empty() {
+        RequestId::try_new(self.request_id.clone())
-            return Err("coordinator wire request_id must be non-empty".to_owned());
+            .map_err(|error| format!("malformed coordinator wire request_id: {error}"))?;
-        }
+        self.payload.validate_external_identifiers()?;
         let payload_operation = self.payload.operation()?;
         if self.operation != payload_operation {
             return Err(format!(
@@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope {
                 self.operation, payload_operation
             ));
         }
-        Ok(self.payload)
+        Ok((self.request_id, self.payload))
     }
 }

crates/clusterflux-core/src/api_error.rs

@@ -0,0 +1,296 @@
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCode {
+    Unauthenticated,
+    SessionExpired,
+    AccountSuspended,
+    Forbidden,
+    ValidationError,
+    NotFound,
+    Conflict,
+    ActiveProcessExists,
+    NodeOffline,
+    NoCapableNode,
+    TaskNotRestartable,
+    ArtifactUnavailable,
+    ArtifactLimitExceeded,
+    QuotaExceeded,
+    TemporaryCapacity,
+    DebugEpochPartial,
+    InternalError,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCategory {
+    Authentication,
+    Authorization,
+    Validation,
+    State,
+    Availability,
+    Resource,
+    Internal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ApiError {
+    pub code: ApiErrorCode,
+    pub category: ApiErrorCategory,
+    pub message: String,
+    pub retryable: bool,
+    pub request_id: String,
+}
+
+impl fmt::Display for ApiError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            formatter,
+            "{} (code {:?}, request {})",
+            self.message, self.code, self.request_id
+        )
+    }
+}
+
+impl std::error::Error for ApiError {}
+
+impl ApiError {
+    pub fn new(
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: impl Into<String>,
+        retryable: bool,
+        request_id: impl Into<String>,
+    ) -> Self {
+        Self {
+            code,
+            category,
+            message: message.into(),
+            retryable,
+            request_id: request_id.into(),
+        }
+    }
+
+    pub fn from_message(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        let request_id = request_id.into();
+        let message = message.into();
+        let normalized = message.to_ascii_lowercase();
+        let (code, category, retryable) = if normalized.contains("session credential has expired")
+            || normalized.contains("session expired")
+        {
+            (
+                ApiErrorCode::SessionExpired,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("tenant is suspended")
+            || normalized.contains("account is suspended")
+            || normalized.contains("suspended by hosted")
+        {
+            (
+                ApiErrorCode::AccountSuspended,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("session credential")
+            || normalized.contains("no authenticated")
+            || normalized.contains("not authenticated")
+            || normalized.contains("credential is not")
+        {
+            (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("project already has active virtual process") {
+            (
+                ApiErrorCode::ActiveProcessExists,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("no capable node") {
+            (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("node offline")
+            || normalized.contains("node is not live")
+            || normalized.contains("source node is not connected")
+            || normalized.contains("direct connectivity unavailable")
+        {
+            (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("restart")
+            && (normalized.contains("not restartable")
+                || normalized.contains("requires whole")
+                || normalized.contains("clean boundary"))
+        {
+            (
+                ApiErrorCode::TaskNotRestartable,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("exceeds download limit")
+                || normalized.contains("download session limit")
+                || normalized.contains("artifact limit"))
+        {
+            (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("does not exist")
+                || normalized.contains("not found")
+                || normalized.contains("unknown artifact"))
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("artifact")
+            && (normalized.contains("unavailable") || normalized.contains("retention"))
+        {
+            (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("resource limit")
+            || normalized.contains("quota")
+            || normalized.contains("limit exceeded")
+        {
+            (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            )
+        } else if normalized.contains("capacity")
+            || normalized.contains("temporarily full")
+            || normalized.contains("replay window is full")
+        {
+            (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("partial debug epoch")
+            || normalized.contains("debug epoch is partially")
+        {
+            (
+                ApiErrorCode::DebugEpochPartial,
+                ApiErrorCategory::State,
+                true,
+            )
+        } else if normalized.contains("malformed")
+            || normalized.contains("invalid ")
+            || normalized.contains("protocol")
+            || normalized.contains("unknown field")
+            || normalized.contains("missing field")
+            || normalized.contains("must ")
+        {
+            (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            )
+        } else if normalized.contains("outside")
+            || normalized.contains("unauthorized")
+            || normalized.contains("denied")
+            || normalized.contains("requires an authenticated")
+            || normalized.contains("may only")
+        {
+            (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("not found")
+            || normalized.contains("does not exist")
+            || normalized.contains("unknown ")
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("already")
+            || normalized.contains("conflict")
+            || normalized.contains("requires an active")
+        {
+            (ApiErrorCode::Conflict, ApiErrorCategory::State, false)
+        } else {
+            (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                false,
+            )
+        };
+        Self::new(code, category, message, retryable, request_id)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn required_machine_codes_are_stably_serialized() {
+        let required = [
+            ApiErrorCode::Unauthenticated,
+            ApiErrorCode::SessionExpired,
+            ApiErrorCode::AccountSuspended,
+            ApiErrorCode::Forbidden,
+            ApiErrorCode::ValidationError,
+            ApiErrorCode::NotFound,
+            ApiErrorCode::Conflict,
+            ApiErrorCode::ActiveProcessExists,
+            ApiErrorCode::NodeOffline,
+            ApiErrorCode::NoCapableNode,
+            ApiErrorCode::TaskNotRestartable,
+            ApiErrorCode::ArtifactUnavailable,
+            ApiErrorCode::ArtifactLimitExceeded,
+            ApiErrorCode::QuotaExceeded,
+            ApiErrorCode::TemporaryCapacity,
+            ApiErrorCode::DebugEpochPartial,
+        ];
+        let serialized = required
+            .into_iter()
+            .map(|code| serde_json::to_value(code).unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            serialized,
+            vec![
+                "unauthenticated",
+                "session_expired",
+                "account_suspended",
+                "forbidden",
+                "validation_error",
+                "not_found",
+                "conflict",
+                "active_process_exists",
+                "node_offline",
+                "no_capable_node",
+                "task_not_restartable",
+                "artifact_unavailable",
+                "artifact_limit_exceeded",
+                "quota_exceeded",
+                "temporary_capacity",
+                "debug_epoch_partial",
+            ]
+        );
+    }
+
+    #[test]
+    fn message_classification_keeps_request_identity() {
+        let error = ApiError::from_message(
+            "request-17",
+            "CLI session credential has expired; run login again",
+        );
+        assert_eq!(error.code, ApiErrorCode::SessionExpired);
+        assert_eq!(error.category, ApiErrorCategory::Authentication);
+        assert_eq!(error.request_id, "request-17");
+        assert!(!error.retryable);
+    }
+}

crates/clusterflux-core/src/artifact.rs

@@ -10,7 +10,7 @@ use crate::{
 
 const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
 const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
-const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
+const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
     pub coordinator_has_large_bytes: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct ArtifactScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub artifact: ArtifactId,
+}
+
+impl ArtifactScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
+        Self {
+            tenant,
+            project,
+            artifact,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
+        Self::new(tenant.clone(), project.clone(), artifact.clone())
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ArtifactFlush {
     pub id: ArtifactId,
@@ -148,7 +169,7 @@ pub struct ArtifactFlush {
 
 #[derive(Clone, Debug, Default)]
 pub struct ArtifactRegistry {
-    artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
+    artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
     issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
     next_epoch: u64,
 }
@@ -162,42 +183,46 @@ impl ArtifactRegistry {
     pub fn flush_metadata_bounded(
         &mut self,
         flush: ArtifactFlush,
-        pinned: &BTreeSet<ArtifactId>,
+        pinned: &BTreeSet<ArtifactScopeKey>,
     ) -> Result<ArtifactMetadata, String> {
-        let replacing_existing = self.artifacts.contains_key(&flush.id);
+        self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
-        while !replacing_existing
+    }
-            && self
+
-                .artifacts
+    pub fn flush_metadata_with_protected_processes(
-                .values()
+        &mut self,
-                .filter(|metadata| {
+        flush: ArtifactFlush,
-                    metadata.tenant == flush.tenant
+        pinned: &BTreeSet<ArtifactScopeKey>,
-                        && metadata.project == flush.project
+        protected_processes: &BTreeSet<ProcessId>,
-                        && metadata.process == flush.process
+    ) -> Result<ArtifactMetadata, String> {
-                })
+        let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
-                .count()
+        let replacing = self.artifacts.contains_key(&key);
-                >= MAX_ARTIFACT_METADATA_PER_PROCESS
+        let retained_for_project = self
-        {
+            .artifacts
-            let candidate = self
+            .values()
-                .artifacts
+            .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
-                .values()
+            .count();
-                .filter(|metadata| {
+        let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
-                    metadata.tenant == flush.tenant
+            None
-                        && metadata.project == flush.project
+        } else {
-                        && metadata.process == flush.process
+            let mut protected_processes = protected_processes.clone();
-                        && !pinned.contains(&metadata.id)
+            protected_processes.insert(flush.process.clone());
-                        && !self.issued_download_links.values().any(|issued| {
+            Some(
-                            issued.link.artifact == metadata.id
+                self.project_metadata_eviction_candidate(
-                        })
+                    &flush.tenant,
-                })
+                    &flush.project,
-                .min_by_key(|metadata| metadata.flushed_epoch)
+                    pinned,
-                .map(|metadata| metadata.id.clone())
+                    &protected_processes,
+                )
                 .ok_or_else(|| {
-                    "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
+                    format!(
-                        .to_owned()
+                        "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
-                })?;
+                         exhausted by active or retained artifacts"
-            self.artifacts.remove(&candidate);
+                    )
-        }
+                })?,
-        self.next_epoch += 1;
+            )
+        };
+
+        self.next_epoch = self.next_epoch.saturating_add(1);
         let metadata = ArtifactMetadata {
             id: flush.id.clone(),
             tenant: flush.tenant,
@@ -212,43 +237,137 @@ impl ArtifactRegistry {
             explicit_locations: Vec::new(),
             coordinator_has_large_bytes: false,
         };
-        self.artifacts.insert(flush.id, metadata.clone());
+        if let Some(eviction) = eviction {
+            self.artifacts.remove(&eviction);
+        }
+        self.artifacts.insert(key, metadata.clone());
         Ok(metadata)
     }
 
+    pub fn enforce_project_metadata_limit(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> usize {
+        let mut evicted = 0;
+        while self
+            .artifacts
+            .values()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+            .count()
+            > MAX_ARTIFACT_METADATA_PER_PROJECT
+        {
+            let candidate = self.project_metadata_eviction_candidate(
+                tenant,
+                project,
+                pinned,
+                protected_processes,
+            );
+            let Some(candidate) = candidate else {
+                break;
+            };
+            self.artifacts.remove(&candidate);
+            evicted += 1;
+        }
+        evicted
+    }
+
+    fn project_metadata_eviction_candidate(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> Option<ArtifactScopeKey> {
+        self.artifacts
+            .values()
+            .filter(|metadata| {
+                &metadata.tenant == tenant
+                    && &metadata.project == project
+                    && !pinned.contains(&ArtifactScopeKey::from_refs(
+                        &metadata.tenant,
+                        &metadata.project,
+                        &metadata.id,
+                    ))
+                    && !protected_processes.contains(&metadata.process)
+                    && metadata.explicit_locations.is_empty()
+                    && !self.issued_download_links.values().any(|issued| {
+                        !issued.revoked
+                            && issued.link.tenant == metadata.tenant
+                            && issued.link.project == metadata.project
+                            && issued.link.artifact == metadata.id
+                    })
+            })
+            .min_by_key(|metadata| metadata.flushed_epoch)
+            .map(|metadata| {
+                ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
+            })
+    }
+
     pub fn sync_to_explicit_store(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         artifact: &ArtifactId,
         location: impl Into<String>,
     ) -> Result<(), ArtifactUnavailable> {
         let metadata = self
             .artifacts
-            .get_mut(artifact)
+            .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
             .ok_or(ArtifactUnavailable)?;
         metadata.explicit_locations.push(location.into());
         Ok(())
     }
 
-    pub fn garbage_collect_node(&mut self, node: &NodeId) {
+    pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
-        for metadata in self.artifacts.values_mut() {
+        for metadata in self
+            .artifacts
+            .values_mut()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+        {
             metadata.retaining_nodes.remove(node);
         }
     }
 
     pub fn reconcile_node_retention(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         retained_artifacts: &BTreeSet<ArtifactId>,
     ) {
-        for (artifact, metadata) in &mut self.artifacts {
+        for (key, metadata) in &mut self.artifacts {
-            if !retained_artifacts.contains(artifact) {
+            if &key.tenant != tenant || &key.project != project {
+                continue;
+            }
+            if retained_artifacts.contains(&key.artifact) {
+                metadata.retaining_nodes.insert(node.clone());
+            } else {
                 metadata.retaining_nodes.remove(node);
             }
         }
     }
 
-    pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
+    pub fn metadata(
-        self.artifacts.get(artifact)
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        artifact: &ArtifactId,
+    ) -> Option<&ArtifactMetadata> {
+        self.artifacts
+            .get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
+    }
+
+    pub fn metadata_for_project<'a>(
+        &'a self,
+        tenant: &'a TenantId,
+        project: &'a ProjectId,
+    ) -> impl Iterator<Item = &'a ArtifactMetadata> + 'a {
+        self.artifacts
+            .values()
+            .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project)
     }
 
     pub fn download_action(
@@ -259,7 +378,11 @@ impl ArtifactRegistry {
     ) -> Result<DownloadAction, DownloadError> {
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let scope = Scope {
             tenant: metadata.tenant.clone(),
@@ -313,7 +436,11 @@ impl ArtifactRegistry {
         self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         Ok(metadata.size)
     }
@@ -331,7 +458,11 @@ impl ArtifactRegistry {
         if self
             .issued_download_links
             .values()
-            .filter(|issued| issued.link.artifact == *artifact)
+            .filter(|issued| {
+                issued.link.tenant == context.tenant
+                    && issued.link.project == context.project
+                    && issued.link.artifact == *artifact
+            })
             .count()
             >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
         {
@@ -343,7 +474,11 @@ impl ArtifactRegistry {
         let action = self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
         let policy_context_digest =
@@ -396,7 +531,11 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact || issued.link.actor != context.actor {
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
+            || issued.link.actor != context.actor
+        {
             return Err(DownloadError::InvalidToken);
         }
         self.download_action(
@@ -432,7 +571,9 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
             || issued.link.max_bytes != policy.max_bytes
             || issued.link.actor != context.actor
         {
@@ -450,7 +591,11 @@ impl ArtifactRegistry {
         }
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if download_policy_context_digest(metadata, &action.source, policy)
             != issued.link.policy_context_digest
@@ -475,7 +620,11 @@ impl ArtifactRegistry {
     ) -> Result<(), DownloadError> {
         let metadata = self
             .artifacts
-            .get(&stream.link.artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &stream.link.tenant,
+                &stream.link.project,
+                &stream.link.artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if !source_is_available(metadata, &stream.link.source) {
             return Err(DownloadError::Unavailable);
@@ -594,7 +743,13 @@ mod tests {
     #[test]
     fn flush_publishes_metadata_without_coordinator_bytes() {
         let registry = registry_with_artifact();
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(!metadata.coordinator_has_large_bytes);
         assert_eq!(metadata.id, ArtifactId::from("artifact"));
@@ -613,7 +768,11 @@ mod tests {
     #[test]
     fn unsynced_node_loss_surfaces_as_unavailable() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -634,20 +793,68 @@ mod tests {
     #[test]
     fn signed_node_retention_inventory_removes_garbage_collected_location() {
         let mut registry = registry_with_artifact();
-        registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+            &BTreeSet::new(),
+        );
 
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(metadata.retaining_nodes.is_empty());
     }
 
+    #[test]
+    fn signed_node_retention_inventory_restores_readvertised_location() {
+        let mut registry = registry_with_artifact();
+        let node = NodeId::from("node");
+        let artifact = ArtifactId::from("artifact");
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+        );
+
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+            &BTreeSet::from([artifact.clone()]),
+        );
+
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &artifact,
+            )
+            .unwrap();
+        assert!(metadata.retaining_nodes.contains(&node));
+    }
+
     #[test]
     fn explicit_user_storage_location_survives_node_retention_loss() {
         let mut registry = registry_with_artifact();
         registry
-            .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
+            .sync_to_explicit_store(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+                "s3://bucket/app",
+            )
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -668,7 +875,11 @@ mod tests {
         );
         assert!(
             !registry
-                .metadata(&ArtifactId::from("artifact"))
+                .metadata(
+                    &TenantId::from("tenant"),
+                    &ProjectId::from("project"),
+                    &ArtifactId::from("artifact"),
+                )
                 .unwrap()
                 .coordinator_has_large_bytes
         );
@@ -699,7 +910,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
     }
 
     #[test]
@@ -719,13 +930,206 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
+    }
+
+    #[test]
+    fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
+        let artifact = ArtifactId::from("shared-artifact");
+        let tenant_a = TenantId::from("tenant-a");
+        let project_a = ProjectId::from("project-a");
+        let tenant_b = TenantId::from("tenant-b");
+        let project_b = ProjectId::from("project-b");
+        let project_c = ProjectId::from("project-c");
+        let node_a = NodeId::from("node-a");
+        let node_b = NodeId::from("node-b");
+        let node_c = NodeId::from("node-c");
+        let mut registry = ArtifactRegistry::default();
+        for (tenant, project, process, node, content, size) in [
+            (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
+            (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
+            (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
+        ] {
+            registry.flush_metadata(ArtifactFlush {
+                id: artifact.clone(),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: ProcessId::from(process),
+                producer_task: TaskInstanceId::from("producer"),
+                retaining_node: node.clone(),
+                digest: Digest::sha256(content),
+                size,
+            });
+        }
+
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_c, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-c")
+        );
+        registry.flush_metadata(ArtifactFlush {
+            id: artifact.clone(),
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            process: ProcessId::from("process-a"),
+            producer_task: TaskInstanceId::from("producer-repeat"),
+            retaining_node: node_a.clone(),
+            digest: Digest::sha256("bytes-a-repeat"),
+            size: 18,
+        });
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a-repeat")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+
+        registry
+            .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
+            .unwrap();
+        registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
+        let context_a = AuthContext {
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let context_b = AuthContext {
+            tenant: tenant_b.clone(),
+            project: project_b.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let policy = DownloadPolicy { max_bytes: 100 };
+        assert_eq!(
+            registry
+                .download_action(&context_a, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::ExplicitStore("store://tenant-a".to_owned())
+        );
+        assert_eq!(
+            registry
+                .download_action(&context_b, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::RetainedNode(node_b)
+        );
+
+        let first_a = registry
+            .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        let first_b = registry
+            .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        assert_ne!(
+            first_a.scoped_token_digest, first_b.scoped_token_digest,
+            "the full artifact scope must contribute to download tokens"
+        );
+        for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
+            registry
+                .create_download_link(
+                    &context_a,
+                    &artifact,
+                    &policy,
+                    &format!("tenant-a-{index}"),
+                    10,
+                    60,
+                )
+                .unwrap();
+        }
+        assert!(matches!(
+            registry.create_download_link(
+                &context_a,
+                &artifact,
+                &policy,
+                "tenant-a-over-limit",
+                10,
+                60,
+            ),
+            Err(DownloadError::Usage(_))
+        ));
+        registry
+            .create_download_link(
+                &context_b,
+                &artifact,
+                &policy,
+                "tenant-b-still-independent",
+                10,
+                60,
+            )
+            .unwrap();
+        registry
+            .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
+            .unwrap();
+
+        let limits = ResourceLimits {
+            limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
+        };
+        let mut meter = ResourceMeter::default();
+        registry
+            .open_download_stream(
+                DownloadStreamRequest {
+                    context: &context_b,
+                    artifact: &artifact,
+                    policy: &policy,
+                    presented_token_digest: &first_b.scoped_token_digest,
+                    now_epoch_seconds: 11,
+                    limits: &limits,
+                },
+                &mut meter,
+            )
+            .expect("revoking tenant A's link must not revoke tenant B's link");
+        assert_eq!(
+            registry
+                .open_download_stream(
+                    DownloadStreamRequest {
+                        context: &context_b,
+                        artifact: &artifact,
+                        policy: &policy,
+                        presented_token_digest: &first_a.scoped_token_digest,
+                        now_epoch_seconds: 11,
+                        limits: &limits,
+                    },
+                    &mut meter,
+                )
+                .unwrap_err(),
+            DownloadError::InvalidToken
+        );
     }
 
     #[test]
     fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -1002,51 +1406,200 @@ mod tests {
     }
 
     #[test]
-    fn artifact_metadata_is_bounded_without_evicting_pins() {
+    fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
         let mut registry = ArtifactRegistry::default();
-        let mut pinned = BTreeSet::new();
+        let tenant = TenantId::from("tenant");
-        for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
+        let project = ProjectId::from("project");
+        registry.flush_metadata(ArtifactFlush {
+            id: ArtifactId::from("artifact-0"),
+            tenant: TenantId::from("other-tenant"),
+            project: ProjectId::from("other-project"),
+            process: ProcessId::from("process"),
+            producer_task: TaskInstanceId::from("other-task"),
+            retaining_node: NodeId::from("other-node"),
+            digest: Digest::sha256("other-content"),
+            size: 1,
+        });
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
             let id = ArtifactId::new(format!("artifact-{index}"));
-            pinned.insert(id.clone());
+            registry.flush_metadata(ArtifactFlush {
-            registry
+                id,
-                .flush_metadata_bounded(
+                tenant: tenant.clone(),
-                    ArtifactFlush {
+                project: project.clone(),
-                        id,
+                process: ProcessId::new(if index == 3 {
-                        tenant: TenantId::from("tenant"),
+                    "active-process-3".to_owned()
-                        project: ProjectId::from("project"),
+                } else {
-                        process: ProcessId::from("process"),
+                    format!("completed-process-{index}")
-                        producer_task: TaskInstanceId::new(format!("task-{index}")),
+                }),
-                        retaining_node: NodeId::from("node"),
+                producer_task: TaskInstanceId::new(format!("task-{index}")),
-                        digest: Digest::sha256(format!("content-{index}")),
+                retaining_node: NodeId::from("node"),
-                        size: 1,
+                digest: Digest::sha256(format!("content-{index}")),
-                    },
+                size: 1,
-                    &pinned,
+            });
-                )
-                .unwrap();
         }
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        let pinned = BTreeSet::from([ArtifactScopeKey::new(
+            tenant.clone(),
+            project.clone(),
+            ArtifactId::from("artifact-0"),
+        )]);
+        registry
+            .sync_to_explicit_store(
+                &tenant,
+                &project,
+                &ArtifactId::from("artifact-1"),
+                "store://retained-export",
+            )
+            .unwrap();
+        let context = AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        registry
+            .create_download_link(
+                &context,
+                &ArtifactId::from("artifact-2"),
+                &DownloadPolicy { max_bytes: 1 },
+                "active-download",
+                10,
+                60,
+            )
+            .unwrap();
+        let protected_processes = BTreeSet::from([
+            ProcessId::from("active-process-3"),
+            ProcessId::from("active-process"),
+        ]);
         let next = ArtifactFlush {
             id: ArtifactId::from("artifact-next"),
-            tenant: TenantId::from("tenant"),
+            tenant: tenant.clone(),
-            project: ProjectId::from("project"),
+            project: project.clone(),
-            process: ProcessId::from("process"),
+            process: ProcessId::from("active-process"),
             producer_task: TaskInstanceId::from("task-next"),
             retaining_node: NodeId::from("node"),
             digest: Digest::sha256("next"),
             size: 1,
         };
+        registry
+            .flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
+            .unwrap();
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
+            assert!(
+                registry
+                    .metadata(&tenant, &project, &ArtifactId::from(id))
+                    .is_some(),
+                "{id} was evicted despite being pinned, exported, downloaded, or live"
+            );
+        }
+        assert!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
+                .is_none(),
+            "the oldest completed unprotected metadata should be evicted"
+        );
         assert!(registry
-            .flush_metadata_bounded(next.clone(), &pinned)
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
-            .unwrap_err()
-            .contains("pinned"));
-
-        pinned.remove(&ArtifactId::from("artifact-0"));
-        registry.flush_metadata_bounded(next, &pinned).unwrap();
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
-        assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
-        assert!(registry
-            .metadata(&ArtifactId::from("artifact-next"))
             .is_some());
+        assert_eq!(
+            registry
+                .metadata(
+                    &TenantId::from("other-tenant"),
+                    &ProjectId::from("other-project"),
+                    &ArtifactId::from("artifact-0"),
+                )
+                .unwrap()
+                .digest,
+            Digest::sha256("other-content"),
+            "eviction and pins in one scope must not affect an equal ID in another scope"
+        );
+    }
+
+    #[test]
+    fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
+        let mut registry = ArtifactRegistry::default();
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let active_process = ProcessId::from("active-process");
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
+            registry.flush_metadata(ArtifactFlush {
+                id: ArtifactId::new(format!("artifact-{index}")),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::new(format!("task-{index}")),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256(format!("content-{index}")),
+                size: 1,
+            });
+        }
+
+        let replacement = ArtifactFlush {
+            id: ArtifactId::from("artifact-0"),
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: active_process.clone(),
+            producer_task: TaskInstanceId::from("replacement-task"),
+            retaining_node: NodeId::from("node"),
+            digest: Digest::sha256("replacement"),
+            size: 2,
+        };
+        registry
+            .flush_metadata_with_protected_processes(
+                replacement,
+                &BTreeSet::new(),
+                &BTreeSet::from([active_process.clone()]),
+            )
+            .expect("replacement must remain possible at the metadata bound");
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement")
+        );
+
+        let epoch_before_rejection = registry.next_epoch;
+        let result = registry.flush_metadata_with_protected_processes(
+            ArtifactFlush {
+                id: ArtifactId::from("artifact-rejected"),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::from("rejected-task"),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256("rejected"),
+                size: 3,
+            },
+            &BTreeSet::new(),
+            &BTreeSet::from([active_process]),
+        );
+
+        assert!(result
+            .unwrap_err()
+            .contains("exhausted by active or retained artifacts"));
+        assert_eq!(registry.next_epoch, epoch_before_rejection);
+        assert_eq!(
+            registry.metadata_for_project(&tenant, &project).count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT
+        );
+        assert!(registry
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
+            .is_none());
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement"),
+            "rejection must not modify existing metadata"
+        );
     }
 
     #[test]
@@ -1135,7 +1688,11 @@ mod tests {
         registry
             .stream_download_chunk(&mut stream, &limits, &mut meter, 16)
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
 
         assert_eq!(
             registry

crates/clusterflux-core/src/auth.rs

@@ -364,7 +364,13 @@ pub fn agent_workflow_request_scope_from_payload(
                 .get("task_instance")
                 .and_then(Value::as_str)
                 .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
-            (process, Some(TaskInstanceId::from(task)))
+            (
+                process,
+                Some(
+                    TaskInstanceId::try_new(task)
+                        .map_err(|error| format!("malformed launch_task task instance: {error}"))?,
+                ),
+            )
         }
         _ => {
             return Err(format!(
@@ -373,10 +379,13 @@ pub fn agent_workflow_request_scope_from_payload(
         }
     };
     AgentWorkflowRequestScope::new(
-        TenantId::from(tenant),
+        TenantId::try_new(tenant)
-        ProjectId::from(project),
+            .map_err(|error| format!("malformed agent workflow tenant: {error}"))?,
+        ProjectId::try_new(project)
+            .map_err(|error| format!("malformed agent workflow project: {error}"))?,
         request_kind,
-        ProcessId::from(process),
+        ProcessId::try_new(process)
+            .map_err(|error| format!("malformed agent workflow process: {error}"))?,
         task,
     )
 }

crates/clusterflux-core/src/execution.rs

@@ -300,6 +300,27 @@ impl TaskBoundaryValue {
             _ => Vec::new(),
         }
     }
+
+    fn rebind_source_snapshot(&mut self, replacement: &Digest) -> usize {
+        match self {
+            Self::SourceSnapshot(snapshot) => {
+                *snapshot = replacement.clone();
+                1
+            }
+            Self::Structured(structured) => structured
+                .handles
+                .iter_mut()
+                .map(|handle| match handle {
+                    TaskBoundaryHandle::SourceSnapshot(snapshot) => {
+                        *snapshot = replacement.clone();
+                        1
+                    }
+                    _ => 0,
+                })
+                .sum(),
+            _ => 0,
+        }
+    }
 }
 
 pub const WASM_TASK_ABI_VERSION: u32 = 1;
@@ -846,6 +867,23 @@ impl TaskSpec {
         }
         Ok(())
     }
+
+    pub fn rebind_source_snapshot(&mut self, replacement: Digest) -> Result<(), String> {
+        self.validate_boundary_authority()?;
+        if self.source_snapshot.is_none() {
+            return Err("task has no SourceSnapshot argument handle to rebind".to_owned());
+        }
+        let rebound = self
+            .args
+            .iter_mut()
+            .map(|argument| argument.rebind_source_snapshot(&replacement))
+            .sum::<usize>();
+        if rebound == 0 {
+            return Err("task source dependency has no canonical argument handle".to_owned());
+        }
+        self.source_snapshot = Some(replacement);
+        self.validate_boundary_authority()
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -1044,6 +1082,11 @@ mod tests {
             bundle_digest: Some(Digest::sha256("bundle")),
         };
         spec.validate_boundary_authority().unwrap();
+        let replacement_source = Digest::sha256("replacement-source");
+        spec.rebind_source_snapshot(replacement_source.clone())
+            .unwrap();
+        assert_eq!(spec.source_snapshot, Some(replacement_source.clone()));
+        assert_eq!(spec.args[0].source_snapshots(), vec![replacement_source]);
         spec.required_artifacts = vec![ArtifactId::from("forged.bin")];
         assert!(spec
             .validate_boundary_authority()

crates/clusterflux-core/src/ids.rs

@@ -1,18 +1,102 @@
+#[cfg(not(target_arch = "wasm32"))]
+use serde::{de::Error as _, Deserializer};
 use serde::{Deserialize, Serialize};
 
+pub const MAX_EXTERNAL_ID_BYTES: usize = 255;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OpaqueTokenError {
+    reason: String,
+}
+
+impl std::fmt::Display for OpaqueTokenError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.reason)
+    }
+}
+
+impl std::error::Error for OpaqueTokenError {}
+
+pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> {
+    let reason = if value.trim().is_empty() {
+        Some("value must not be empty or whitespace-only".to_owned())
+    } else if value.len() > max_bytes {
+        Some(format!("value exceeds the {max_bytes}-byte limit"))
+    } else if value.chars().any(char::is_control) {
+        Some("control characters are forbidden".to_owned())
+    } else {
+        None
+    };
+    reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason }))
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IdParseError {
+    id_type: &'static str,
+    reason: &'static str,
+}
+
+impl IdParseError {
+    fn new(id_type: &'static str, reason: &'static str) -> Self {
+        Self { id_type, reason }
+    }
+}
+
+impl std::fmt::Display for IdParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} is invalid: {}", self.id_type, self.reason)
+    }
+}
+
+impl std::error::Error for IdParseError {}
+
+fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> {
+    if value.is_empty() || value.trim().is_empty() {
+        return Err(IdParseError::new(
+            id_type,
+            "value must not be empty or whitespace-only",
+        ));
+    }
+    if value.len() > MAX_EXTERNAL_ID_BYTES {
+        return Err(IdParseError::new(
+            id_type,
+            "value exceeds the 255-byte limit",
+        ));
+    }
+    if value.chars().any(char::is_control) {
+        return Err(IdParseError::new(
+            id_type,
+            "control characters are forbidden",
+        ));
+    }
+    if !value.bytes().all(|byte| {
+        byte.is_ascii_alphanumeric()
+            || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+')
+    }) {
+        return Err(IdParseError::new(
+            id_type,
+            "only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed",
+        ));
+    }
+    Ok(())
+}
+
 macro_rules! id_type {
     ($name:ident) => {
-        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+        #[cfg_attr(target_arch = "wasm32", derive(Deserialize))]
+        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
         pub struct $name(String);
 
         impl $name {
-            pub fn new(value: impl Into<String>) -> Self {
+            pub fn try_new(value: impl Into<String>) -> Result<Self, IdParseError> {
                 let value = value.into();
-                assert!(
+                validate_id(&value, stringify!($name))?;
-                    !value.trim().is_empty(),
+                Ok(Self(value))
-                    concat!(stringify!($name), " cannot be empty")
+            }
-                );
+
-                Self(value)
+            /// Constructs an identifier from a trusted, internally generated value.
+            pub fn new(value: impl Into<String>) -> Self {
+                Self::try_new(value).unwrap_or_else(|error| panic!("{error}"))
             }
 
             pub fn as_str(&self) -> &str {
@@ -26,6 +110,17 @@ macro_rules! id_type {
             }
         }
 
+        #[cfg(not(target_arch = "wasm32"))]
+        impl<'de> Deserialize<'de> for $name {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                let value = String::deserialize(deserializer)?;
+                Self::try_new(value).map_err(D::Error::custom)
+            }
+        }
+
         impl std::fmt::Display for $name {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                 f.write_str(&self.0)
@@ -38,8 +133,79 @@ id_type!(AgentId);
 id_type!(ArtifactId);
 id_type!(NodeId);
 id_type!(ProcessId);
+id_type!(LaunchAttemptId);
 id_type!(ProjectId);
 id_type!(TaskDefinitionId);
 id_type!(TaskInstanceId);
 id_type!(TenantId);
 id_type!(UserId);
+
+pub type DebugSessionId = ProcessId;
+pub type RequestId = LaunchAttemptId;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_hostile_values<T>(parse: impl Fn(String) -> Result<T, IdParseError>) {
+        for value in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0id".to_owned(),
+            "bad id".to_owned(),
+            "x".repeat(MAX_EXTERNAL_ID_BYTES + 1),
+        ] {
+            assert!(parse(value).is_err());
+        }
+    }
+
+    #[test]
+    fn every_external_identifier_type_rejects_hostile_values() {
+        assert_hostile_values(AgentId::try_new);
+        assert_hostile_values(ArtifactId::try_new);
+        assert_hostile_values(DebugSessionId::try_new);
+        assert_hostile_values(NodeId::try_new);
+        assert_hostile_values(ProcessId::try_new);
+        assert_hostile_values(LaunchAttemptId::try_new);
+        assert_hostile_values(ProjectId::try_new);
+        assert_hostile_values(RequestId::try_new);
+        assert_hostile_values(TaskDefinitionId::try_new);
+        assert_hostile_values(TaskInstanceId::try_new);
+        assert_hostile_values(TenantId::try_new);
+        assert_hostile_values(UserId::try_new);
+    }
+
+    #[test]
+    fn every_identifier_type_validates_during_deserialization() {
+        macro_rules! assert_deserialization {
+            ($id:ty) => {
+                assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok());
+                let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err();
+                assert!(
+                    error.to_string().contains("is invalid"),
+                    "{} produced an unexpected error: {error}",
+                    stringify!($id)
+                );
+            };
+        }
+
+        assert_deserialization!(AgentId);
+        assert_deserialization!(ArtifactId);
+        assert_deserialization!(NodeId);
+        assert_deserialization!(ProcessId);
+        assert_deserialization!(LaunchAttemptId);
+        assert_deserialization!(ProjectId);
+        assert_deserialization!(TaskDefinitionId);
+        assert_deserialization!(TaskInstanceId);
+        assert_deserialization!(TenantId);
+        assert_deserialization!(UserId);
+    }
+
+    #[test]
+    fn opaque_tokens_are_bounded_without_using_identifier_syntax() {
+        validate_opaque_token("opaque secret/+==", 64).unwrap();
+        assert!(validate_opaque_token("", 64).is_err());
+        assert!(validate_opaque_token("bad\0token", 64).is_err());
+        assert!(validate_opaque_token(&"x".repeat(65), 64).is_err());
+    }
+}

crates/clusterflux-core/src/lib.rs

@@ -1,3 +1,4 @@
+mod api_error;
 pub mod artifact;
 pub mod auth;
 pub mod bundle;
@@ -18,10 +19,11 @@ pub mod transport;
 pub mod vfs;
 pub mod wire;
 
+pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode};
 pub use artifact::{
     ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
-    ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
+    ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
-    DownloadStreamRequest, RetentionPolicy, StorageLocation,
+    DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
 };
 pub use auth::{
     admin_request_proof, admin_request_proof_from_token_digest,
@@ -66,8 +68,9 @@ pub use execution::{
     WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
 };
 pub use ids::{
-    AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
+    validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId,
-    UserId,
+    NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId, MAX_EXTERNAL_ID_BYTES,
 };
 pub use limits::{
     LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,

crates/clusterflux-core/src/vfs.rs

@@ -1,18 +1,48 @@
 use std::collections::BTreeMap;
 
-use serde::{Deserialize, Serialize};
+use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
 use thiserror::Error;
 
 use crate::{Digest, NodeId, TaskInstanceId};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub const MAX_VFS_PATH_BYTES: usize = 4096;
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 pub struct VfsPath(String);
 
 impl VfsPath {
     pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
         let path = path.into();
         if !path.starts_with("/vfs/") {
-            return Err(VfsError::InvalidPath(path));
+            return Err(VfsError::invalid(path, "path must start with /vfs/"));
+        }
+        if path.len() > MAX_VFS_PATH_BYTES {
+            return Err(VfsError::invalid(
+                path,
+                "path exceeds the 4096-byte protocol limit",
+            ));
+        }
+        if path.chars().any(char::is_control) {
+            return Err(VfsError::invalid(path, "control characters are forbidden"));
+        }
+        if path.contains('\\') {
+            return Err(VfsError::invalid(path, "backslashes are forbidden"));
+        }
+        let relative = &path["/vfs/".len()..];
+        if relative.is_empty() {
+            return Err(VfsError::invalid(
+                path,
+                "path after /vfs/ must not be empty",
+            ));
+        }
+        if relative
+            .split('/')
+            .any(|component| component.is_empty() || matches!(component, "." | ".."))
+        {
+            return Err(VfsError::invalid(
+                path,
+                "empty, '.', and '..' path components are forbidden",
+            ));
         }
         Ok(Self(path))
     }
@@ -22,6 +52,16 @@ impl VfsPath {
     }
 }
 
+impl<'de> Deserialize<'de> for VfsPath {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let path = String::deserialize(deserializer)?;
+        Self::new(path).map_err(D::Error::custom)
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct VfsObject {
     pub path: VfsPath,
@@ -63,12 +103,18 @@ pub enum ReuseDecision {
 
 #[derive(Clone, Debug, Error, PartialEq, Eq)]
 pub enum VfsError {
-    #[error("VFS path must start with /vfs/: {0}")]
+    #[error("invalid VFS path {path:?}: {reason}")]
-    InvalidPath(String),
+    InvalidPath { path: String, reason: &'static str },
     #[error("path is not visible in the published VFS manifest: {0}")]
     NotVisible(String),
 }
 
+impl VfsError {
+    fn invalid(path: String, reason: &'static str) -> Self {
+        Self::InvalidPath { path, reason }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct VfsOverlay {
     task: TaskInstanceId,
@@ -238,4 +284,39 @@ mod tests {
 
         assert_eq!(overlay.pending_len(), 0);
     }
+
+    #[test]
+    fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
+        for invalid in [
+            "vfs/artifacts/app".to_owned(),
+            "/vfs/".to_owned(),
+            "/vfs/artifacts//app".to_owned(),
+            "/vfs/artifacts/app/".to_owned(),
+            "/vfs/artifacts/./app".to_owned(),
+            "/vfs/artifacts/../app".to_owned(),
+            "/vfs/artifacts\\app".to_owned(),
+            "/vfs/artifacts/bad\0app".to_owned(),
+            format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
+        ] {
+            assert!(
+                VfsPath::new(&invalid).is_err(),
+                "hostile VFS path unexpectedly passed: {invalid:?}"
+            );
+            assert!(
+                serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
+                "hostile VFS path unexpectedly deserialized"
+            );
+        }
+    }
+
+    #[test]
+    fn vfs_path_accepts_the_4096_byte_boundary() {
+        let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
+        assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
+        assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
+
+        let overlong = format!("{path}x");
+        assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
+        assert!(VfsPath::new(overlong).is_err());
+    }
 }

crates/clusterflux-dap/src/adapter.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::io::{self, BufReader};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{mpsc, Arc};
@@ -19,13 +20,13 @@ use crate::demo_backend::{
 };
 use crate::runtime_client::{
     attach_services_runtime, create_debug_epoch, debug_epoch_runtime_record,
-    relaunch_services_main_runtime, restart_task, resume_debug_epoch, run_live_services_runtime,
+    observe_services_runtime, relaunch_services_main_runtime, restart_task, resume_debug_epoch,
-    run_local_services_runtime, set_services_debug_breakpoints, wait_for_debug_epoch_frozen,
+    run_live_services_runtime, run_local_services_runtime, set_services_debug_breakpoints,
-    wait_for_debug_epoch_resumed, wait_for_services_runtime_outcome, LocalRuntimeSession,
+    wait_for_debug_epoch_frozen, wait_for_debug_epoch_resumed, LocalRuntimeSession,
     RuntimeContinuationOutcome,
 };
 use crate::variables::variables_response;
-use crate::virtual_model::{AdapterState, DapSessionMode, RuntimeBackend};
+use crate::virtual_model::{AdapterState, DapSessionMode, RuntimeBackend, VirtualThread};
 
 enum AdapterEvent {
     Request(Value),
@@ -35,9 +36,12 @@ enum AdapterEvent {
         generation: u64,
         result: std::result::Result<LaunchCompletion, String>,
     },
+    ObservationUpdate {
+        generation: u64,
+        outcome: RuntimeContinuationOutcome,
+    },
     ObservationFinished {
         generation: u64,
-        result: std::result::Result<Option<RuntimeContinuationOutcome>, String>,
     },
     PauseFinished {
         generation: u64,
@@ -45,10 +49,13 @@ enum AdapterEvent {
     },
     ResumeFinished {
         generation: u64,
+        request: Value,
+        thread_id: i64,
         result: std::result::Result<AdapterState, String>,
     },
     BreakpointsUpdated {
         generation: u64,
+        revision: u64,
         result: std::result::Result<(), String>,
     },
 }
@@ -56,6 +63,8 @@ enum AdapterEvent {
 struct LaunchCompletion {
     state: AdapterState,
     local_runtime_session: Option<LocalRuntimeSession>,
+    breakpoint_revision: u64,
+    observation_diagnostics: Vec<String>,
 }
 
 pub(crate) fn run_adapter() -> Result<()> {
@@ -87,22 +96,39 @@ pub(crate) fn run_adapter() -> Result<()> {
                 runtime_starting = false;
                 match result {
                     Ok(mut completion) => {
-                        completion.state.breakpoints = state.breakpoints.clone();
+                        let requested_breakpoints = state.breakpoints.clone();
+                        let requested_revision = state.breakpoint_revision;
+                        let installed_revision = completion.breakpoint_revision;
+                        completion.state.breakpoints = requested_breakpoints;
+                        completion.state.breakpoint_revision = requested_revision;
+                        completion.state.breakpoints_installed =
+                            installed_revision == requested_revision;
+                        let previous_threads = state.threads.clone();
                         state = completion.state;
+                        emit_thread_lifecycle(&mut writer, &previous_threads, &state.threads)?;
                         if let Some(session) = completion.local_runtime_session {
                             _local_runtime_session = Some(session);
                         }
+                        for diagnostic in completion.observation_diagnostics {
+                            writer.output("stderr", format!("{diagnostic}\n"))?;
+                        }
                         runtime_started = true;
-                        if !state.breakpoints.is_empty() {
+                        if state.breakpoints_installed {
+                            emit_verified_breakpoints(&mut writer, &state)?;
+                        } else {
                             let breakpoint_state = state.clone();
                             let breakpoint_tx = event_tx.clone();
+                            let revision = state.breakpoint_revision;
                             thread::spawn(move || {
                                 let result = set_services_debug_breakpoints(&breakpoint_state)
                                     .map_err(|error| {
                                         format!("coordinator breakpoint update failed: {error:#}")
                                     });
-                                let _ = breakpoint_tx
+                                let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
-                                    .send(AdapterEvent::BreakpointsUpdated { generation, result });
+                                    generation,
+                                    revision,
+                                    result,
+                                });
                             });
                         }
                         writer.output(
@@ -129,41 +155,24 @@ pub(crate) fn run_adapter() -> Result<()> {
                     Err(message) => {
                         runtime_started = false;
                         writer.output("stderr", format!("{message}\n"))?;
-                        writer.event(
+                        writer.event("terminated", json!({ "restart": false }))?;
-                            "stopped",
-                            json!({
-                                "reason": "exception",
-                                "description": message,
-                                "threadId": MAIN_THREAD,
-                                "allThreadsStopped": false,
-                            }),
-                        )?;
                     }
                 }
                 continue;
             }
-            AdapterEvent::ObservationFinished { generation, result } => {
+            AdapterEvent::ObservationUpdate {
+                generation,
+                outcome,
+            } => {
                 if generation != runtime_generation {
                     continue;
                 }
-                observation_cancel = None;
+                emit_runtime_outcome(&mut writer, &mut state, outcome)?;
-                match result {
+                continue;
-                    Ok(Some(outcome)) => {
+            }
-                        emit_runtime_outcome(&mut writer, &mut state, outcome)?;
+            AdapterEvent::ObservationFinished { generation } => {
-                    }
+                if generation == runtime_generation {
-                    Ok(None) => {}
+                    observation_cancel = None;
-                    Err(message) => {
-                        writer.output("stderr", format!("{message}\n"))?;
-                        writer.event(
-                            "stopped",
-                            json!({
-                                "reason": "exception",
-                                "description": message,
-                                "threadId": default_thread_id(&state),
-                                "allThreadsStopped": false,
-                            }),
-                        )?;
-                    }
                 }
                 continue;
             }
@@ -197,13 +206,31 @@ pub(crate) fn run_adapter() -> Result<()> {
                 }
                 continue;
             }
-            AdapterEvent::ResumeFinished { generation, result } => {
+            AdapterEvent::ResumeFinished {
+                generation,
+                request,
+                thread_id,
+                result,
+            } => {
                 if generation != runtime_generation {
                     continue;
                 }
                 match result {
                     Ok(resumed_state) => {
                         state = resumed_state;
+                        for thread in state.threads.values_mut() {
+                            if thread.state == DebugRuntimeState::Frozen {
+                                thread.state = DebugRuntimeState::Running;
+                            }
+                        }
+                        writer.response(&request, true, json!({ "allThreadsContinued": true }))?;
+                        writer.event(
+                            "continued",
+                            json!({
+                                "threadId": thread_id,
+                                "allThreadsContinued": true,
+                            }),
+                        )?;
                         spawn_runtime_observer(
                             &event_tx,
                             &state,
@@ -213,23 +240,38 @@ pub(crate) fn run_adapter() -> Result<()> {
                     }
                     Err(message) => {
                         writer.output("stderr", format!("{message}\n"))?;
-                        writer.event(
+                        writer.error_response(&request, message)?;
-                            "stopped",
+                        spawn_runtime_observer(
-                            json!({
+                            &event_tx,
-                                "reason": "exception",
+                            &state,
-                                "description": message,
+                            runtime_generation,
-                                "threadId": default_thread_id(&state),
+                            &mut observation_cancel,
-                                "allThreadsStopped": false,
+                        );
-                            }),
-                        )?;
                     }
                 }
                 continue;
             }
-            AdapterEvent::BreakpointsUpdated { generation, result } => {
+            AdapterEvent::BreakpointsUpdated {
-                if generation == runtime_generation {
+                generation,
-                    if let Err(message) = result {
+                revision,
-                        writer.output("stderr", format!("{message}\n"))?;
+                result,
+            } => {
+                if breakpoint_update_is_current(
+                    generation,
+                    runtime_generation,
+                    revision,
+                    state.breakpoint_revision,
+                ) {
+                    match result {
+                        Ok(()) => {
+                            state.breakpoints_installed = true;
+                            emit_verified_breakpoints(&mut writer, &state)?;
+                        }
+                        Err(message) => {
+                            state.breakpoints_installed = false;
+                            emit_unverified_breakpoints(&mut writer, &state, &message)?;
+                            writer.output("stderr", format!("{message}\n"))?;
+                        }
                     }
                 }
                 continue;
@@ -293,7 +335,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                     .and_then(Value::as_bool)
                     .unwrap_or(false);
                 if let Some(process_id) = args.get("processId").and_then(Value::as_str) {
-                    state.process = clusterflux_core::ProcessId::new(process_id);
+                    match clusterflux_core::ProcessId::try_new(process_id.to_owned()) {
+                        Ok(process) => state.process = process,
+                        Err(error) => {
+                            writer.error_response(
+                                &request,
+                                format!("invalid DAP processId: {error}"),
+                            )?;
+                            continue;
+                        }
+                    }
                 }
                 if command == "attach" {
                     state.session_mode = DapSessionMode::Attach;
@@ -320,14 +371,31 @@ pub(crate) fn run_adapter() -> Result<()> {
                             .collect::<Vec<_>>()
                     })
                     .unwrap_or_default();
-                let response = resolve_breakpoints_for_source(
+                let resolved = resolve_breakpoints_for_source(
                     &mut state,
                     requested_source_path,
                     requested_lines,
-                )
+                );
-                .iter()
+                state.breakpoint_revision = state.breakpoint_revision.saturating_add(1);
-                .map(|breakpoint| breakpoint.to_dap())
+                let breakpoint_revision = state.breakpoint_revision;
-                .collect::<Vec<_>>();
+                let coordinator_backed = matches!(
+                    state.runtime_backend,
+                    RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
+                );
+                state.breakpoints_installed = !coordinator_backed;
+                let response = resolved
+                    .iter()
+                    .map(|breakpoint| {
+                        let mut dap = breakpoint.to_dap();
+                        if coordinator_backed && breakpoint.verified {
+                            dap["verified"] = Value::Bool(false);
+                            dap["message"] = Value::String(
+                                "Pending coordinator breakpoint installation".to_owned(),
+                            );
+                        }
+                        dap
+                    })
+                    .collect::<Vec<_>>();
                 writer.response(&request, true, json!({ "breakpoints": response }))?;
                 if runtime_started
                     && matches!(
@@ -343,8 +411,11 @@ pub(crate) fn run_adapter() -> Result<()> {
                             set_services_debug_breakpoints(&breakpoint_state).map_err(|error| {
                                 format!("coordinator breakpoint update failed: {error:#}")
                             });
-                        let _ = breakpoint_tx
+                        let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
-                            .send(AdapterEvent::BreakpointsUpdated { generation, result });
+                            generation,
+                            revision: breakpoint_revision,
+                            result,
+                        });
                     });
                 }
             }
@@ -405,11 +476,18 @@ pub(crate) fn run_adapter() -> Result<()> {
                     let mut completed_state = launch_state;
                     let result = if completed_state.session_mode == DapSessionMode::Attach {
                         attach_services_runtime(&completed_state)
-                            .map(|record| {
+                            .and_then(|record| {
                                 completed_state.apply_attach_record(record);
+                                set_services_debug_breakpoints(&completed_state)?;
+                                Ok(())
+                            })
+                            .map(|()| {
+                                let breakpoint_revision = completed_state.breakpoint_revision;
                                 LaunchCompletion {
                                     state: completed_state,
                                     local_runtime_session: None,
+                                    breakpoint_revision,
+                                    observation_diagnostics: Vec::new(),
                                 }
                             })
                             .map_err(|error| format!("services runtime attach failed: {error:#}"))
@@ -418,10 +496,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                             RuntimeBackend::LocalServices => {
                                 run_local_services_runtime(&completed_state)
                                     .map(|(record, session)| {
+                                        let observation_diagnostics =
+                                            runtime_observation_diagnostics(&record);
                                         completed_state.apply_runtime_record(record);
+                                        let breakpoint_revision =
+                                            completed_state.breakpoint_revision;
                                         LaunchCompletion {
                                             state: completed_state,
                                             local_runtime_session: Some(session),
+                                            breakpoint_revision,
+                                            observation_diagnostics,
                                         }
                                     })
                                     .map_err(|error| {
@@ -431,10 +515,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                             RuntimeBackend::LiveServices => {
                                 run_live_services_runtime(&completed_state)
                                     .map(|record| {
+                                        let observation_diagnostics =
+                                            runtime_observation_diagnostics(&record);
                                         completed_state.apply_runtime_record(record);
+                                        let breakpoint_revision =
+                                            completed_state.breakpoint_revision;
                                         LaunchCompletion {
                                             state: completed_state,
                                             local_runtime_session: None,
+                                            breakpoint_revision,
+                                            observation_diagnostics,
                                         }
                                     })
                                     .map_err(|error| {
@@ -677,28 +767,21 @@ pub(crate) fn run_adapter() -> Result<()> {
                     cancel_observation(&mut observation_cancel);
                     runtime_generation = runtime_generation.saturating_add(1);
                     let generation = runtime_generation;
-                    for thread in state.threads.values_mut() {
-                        if thread.state == DebugRuntimeState::Frozen {
-                            thread.state = DebugRuntimeState::Running;
-                        }
-                    }
-                    writer.response(&request, true, json!({ "allThreadsContinued": true }))?;
-                    writer.event(
-                        "continued",
-                        json!({
-                            "threadId": thread_id,
-                            "allThreadsContinued": true,
-                        }),
-                    )?;
                     let mut resume_state = state.clone();
                     let resume_tx = event_tx.clone();
+                    let resume_request = request.clone();
                     thread::spawn(move || {
                         let result = resume_coordinator_epoch(&mut resume_state)
                             .map(|()| resume_state)
                             .map_err(|error| {
                                 format!("coordinator debug epoch resume failed: {error:#}")
                             });
-                        let _ = resume_tx.send(AdapterEvent::ResumeFinished { generation, result });
+                        let _ = resume_tx.send(AdapterEvent::ResumeFinished {
+                            generation,
+                            request: resume_request,
+                            thread_id,
+                            result,
+                        });
                     });
                     continue;
                 }
@@ -959,24 +1042,145 @@ fn spawn_runtime_observer(
     let observer_tx = event_tx.clone();
     let previous_debug_epoch = state.coordinator_debug_epoch.unwrap_or(state.epoch);
     thread::spawn(move || {
-        let result = wait_for_services_runtime_outcome(
+        let result = observe_services_runtime(
             &observer_state,
             previous_debug_epoch,
             cancelled.as_ref(),
-        )
+            |outcome| {
-        .map_err(|error| format!("runtime observation failed: {error:#}"));
+                observer_tx
-        let _ = observer_tx.send(AdapterEvent::ObservationFinished { generation, result });
+                    .send(AdapterEvent::ObservationUpdate {
+                        generation,
+                        outcome,
+                    })
+                    .is_ok()
+            },
+        );
+        if let Err(error) = result {
+            let _ = observer_tx.send(AdapterEvent::ObservationUpdate {
+                generation,
+                outcome: RuntimeContinuationOutcome::Diagnostic(format!(
+                    "runtime observer stopped unexpectedly: {error:#}"
+                )),
+            });
+        }
+        let _ = observer_tx.send(AdapterEvent::ObservationFinished { generation });
     });
 }
 
+pub(crate) fn breakpoint_update_is_current(
+    update_generation: u64,
+    runtime_generation: u64,
+    update_revision: u64,
+    current_revision: u64,
+) -> bool {
+    update_generation == runtime_generation && update_revision == current_revision
+}
+
+fn runtime_observation_diagnostics(
+    record: &crate::virtual_model::RuntimeLaunchRecord,
+) -> Vec<String> {
+    record
+        .node_report
+        .get("observation_diagnostics")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .filter_map(Value::as_str)
+        .map(str::to_owned)
+        .collect()
+}
+
+fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> {
+    if !state.breakpoints_installed {
+        return Ok(());
+    }
+    for (index, line) in state.breakpoints.iter().enumerate() {
+        writer.event(
+            "breakpoint",
+            json!({
+                "reason": "changed",
+                "breakpoint": {
+                    "id": index + 1,
+                    "verified": true,
+                    "line": line,
+                    "message": "Installed by the Clusterflux coordinator",
+                }
+            }),
+        )?;
+    }
+    Ok(())
+}
+
+fn emit_unverified_breakpoints(
+    writer: &mut DapWriter,
+    state: &AdapterState,
+    coordinator_error: &str,
+) -> Result<()> {
+    for (index, line) in state.breakpoints.iter().enumerate() {
+        writer.event(
+            "breakpoint",
+            json!({
+                "reason": "changed",
+                "breakpoint": {
+                    "id": index + 1,
+                    "verified": false,
+                    "line": line,
+                    "message": format!("Coordinator breakpoint installation failed: {coordinator_error}"),
+                }
+            }),
+        )?;
+    }
+    Ok(())
+}
+
+fn emit_thread_lifecycle(
+    writer: &mut DapWriter,
+    previous: &BTreeMap<i64, VirtualThread>,
+    current: &BTreeMap<i64, VirtualThread>,
+) -> Result<()> {
+    for (id, thread) in previous {
+        if current
+            .get(id)
+            .is_none_or(|current| current.task != thread.task)
+        {
+            writer.event("thread", json!({ "reason": "exited", "threadId": id }))?;
+        }
+    }
+    for (id, thread) in current {
+        if previous
+            .get(id)
+            .is_none_or(|previous| previous.task != thread.task)
+        {
+            writer.event("thread", json!({ "reason": "started", "threadId": id }))?;
+        }
+    }
+    Ok(())
+}
+
+fn apply_runtime_record_with_thread_events(
+    writer: &mut DapWriter,
+    state: &mut AdapterState,
+    record: crate::virtual_model::RuntimeLaunchRecord,
+) -> Result<()> {
+    let previous = state.threads.clone();
+    state.apply_runtime_record(record);
+    emit_thread_lifecycle(writer, &previous, &state.threads)
+}
+
 fn emit_runtime_outcome(
     writer: &mut DapWriter,
     state: &mut AdapterState,
     outcome: RuntimeContinuationOutcome,
 ) -> Result<()> {
     match outcome {
+        RuntimeContinuationOutcome::Snapshot(record) => {
+            apply_runtime_record_with_thread_events(writer, state, record)?;
+        }
+        RuntimeContinuationOutcome::Diagnostic(message) => {
+            writer.output("stderr", format!("{message}\n"))?;
+        }
         RuntimeContinuationOutcome::Breakpoint(record) => {
-            state.apply_runtime_record(record);
+            apply_runtime_record_with_thread_events(writer, state, record)?;
             let stopped_thread = stopped_thread_for_breakpoint(state);
             position_confirmed_breakpoint_stop(state, stopped_thread);
             writer.event(
@@ -990,7 +1194,7 @@ fn emit_runtime_outcome(
             )?;
         }
         RuntimeContinuationOutcome::Exception(record) => {
-            state.apply_runtime_record(record);
+            apply_runtime_record_with_thread_events(writer, state, record)?;
             let stopped_thread = stopped_thread_for_breakpoint(state);
             writer.output("stderr", format!("{}\n", state.command_status))?;
             writer.event(
@@ -1004,10 +1208,12 @@ fn emit_runtime_outcome(
             )?;
         }
         RuntimeContinuationOutcome::Terminal(record) => {
-            state.apply_runtime_record(record);
+            let exit_code = record.status_code.unwrap_or(1);
+            apply_runtime_record_with_thread_events(writer, state, record)?;
             if state.last_task_failed {
                 writer.output("stderr", format!("{}\n", state.command_status))?;
             }
+            writer.event("exited", json!({ "exitCode": exit_code }))?;
             writer.event("terminated", json!({}))?;
         }
     }
@@ -1018,7 +1224,6 @@ fn restart_task_through_coordinator(state: &mut AdapterState, thread_id: i64) ->
     let task = state
         .threads
         .get(&thread_id)
-        .or_else(|| state.threads.get(&default_thread_id(state)))
         .map(|thread| thread.task.clone())
         .ok_or_else(|| anyhow::anyhow!("selected debug thread does not map to a virtual task"))?;
     let record = restart_task(state, &task)?;

crates/clusterflux-dap/src/main.rs

@@ -22,7 +22,7 @@ use breakpoints::{
 #[cfg(test)]
 use dap_protocol::{initialize_capabilities, read_message};
 #[cfg(test)]
-use runtime_client::{client_user_request, parse_task_restart_response};
+use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code};
 #[cfg(test)]
 use variables::variables_response;
 #[cfg(test)]
@@ -36,6 +36,26 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
 use virtual_model::{process_id, RuntimeBackend};
 
 fn main() -> Result<()> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux Debug Adapter Protocol server.\n\n\
+                 Usage: clusterflux-debug-dap\n\n\
+                 The adapter communicates over standard input and output.\n\n\
+                 Options:\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        [] => {}
+        [argument, ..] => anyhow::bail!("unknown argument: {argument}"),
+    }
     adapter::run_adapter()
 }
 

crates/clusterflux-dap/src/runtime_client/debug_protocol.rs

@@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestart
         restarted_task_instance: response
             .get("restarted_task_instance")
             .and_then(Value::as_str)
-            .map(TaskInstanceId::new),
+            .map(TaskInstanceId::try_new)
+            .transpose()
+            .map_err(|error| anyhow!("malformed restarted task instance: {error}"))?,
         restarted_attempt_id: response
             .get("restarted_attempt_id")
             .and_then(Value::as_str)

crates/clusterflux-dap/src/runtime_client/transport.rs

@@ -21,18 +21,41 @@ pub(crate) fn client_user_request(state: &AdapterState, mut request: Value) -> V
     })
 }
 
-pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
+pub(super) struct CoordinatorSession {
-    let mut session = ControlSession::connect(addr)?;
+    session: ControlSession,
-    let wire_request = coordinator_wire_request("dap-1", request);
+}
-    let response = session.request(&wire_request)?;
+
-    if response.get("type").and_then(Value::as_str) == Some("error") {
+impl CoordinatorSession {
-        return Err(anyhow!(
+    pub(super) fn connect(addr: &str) -> Result<Self> {
-            "{}",
+        Ok(Self {
-            response
+            session: ControlSession::connect(addr)?,
-                .get("message")
+        })
-                .and_then(Value::as_str)
+    }
-                .unwrap_or("coordinator returned an error")
+
-        ));
+    pub(super) fn request(&mut self, request: Value) -> Result<Value> {
-    }
+        let response = self.request_allow_error(request)?;
-    Ok(response)
+        if response.get("type").and_then(Value::as_str) == Some("error") {
+            return Err(anyhow!(
+                "{}",
+                response
+                    .get("message")
+                    .and_then(Value::as_str)
+                    .unwrap_or("coordinator returned an error")
+            ));
+        }
+        Ok(response)
+    }
+
+    pub(super) fn request_allow_error(&mut self, request: Value) -> Result<Value> {
+        let wire_request = coordinator_wire_request("dap-1", request);
+        Ok(self.session.request(&wire_request)?)
+    }
+}
+
+pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
+    CoordinatorSession::connect(addr)?.request(request)
+}
+
+pub(super) fn coordinator_request_allow_error(addr: &str, request: Value) -> Result<Value> {
+    CoordinatorSession::connect(addr)?.request_allow_error(request)
 }

crates/clusterflux-dap/src/source.rs

@@ -7,12 +7,12 @@ use serde_json::{json, Value};
 use crate::virtual_model::AdapterState;
 
 pub(crate) fn infer_source_path(project: &str) -> String {
-    if Path::new(project).join("src/build.rs").is_file() {
+    if Path::new(project).join("src/lib.rs").is_file() {
-        "src/build.rs".to_owned()
+        "src/lib.rs".to_owned()
     } else if Path::new(project).join("src/main.rs").is_file() {
         "src/main.rs".to_owned()
-    } else if Path::new(project).join("src/lib.rs").is_file() {
+    } else if Path::new(project).join("src/build.rs").is_file() {
-        "src/lib.rs".to_owned()
+        "src/build.rs".to_owned()
     } else {
         "src/main.rs".to_owned()
     }

crates/clusterflux-dap/src/tests.rs

@@ -45,6 +45,13 @@ fn initialize_capabilities_use_standard_dap_flags() {
         .all(|key| key.starts_with("supports") && !key.contains("clusterflux")));
 }
 
+#[test]
+fn stale_breakpoint_success_and_failure_completions_are_not_current() {
+    assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9));
+}
+
 #[test]
 fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() {
     assert_eq!(
@@ -189,10 +196,19 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
     state.apply_runtime_record(RuntimeLaunchRecord {
         coordinator: "127.0.0.1:7999".to_owned(),
         node: "node".to_owned(),
-        node_report: json!({ "terminal_event": {
+        node_report: json!({
-            "task": "compile-linux-1",
+            "task_snapshots": { "snapshots": [{
-            "task_definition": "compile-linux"
+                "task": "compile-linux-1",
-        } }),
+                "task_definition": "compile-linux",
+                "attempt_id": "attempt-1",
+                "current": true,
+                "state": "failed_awaiting_action"
+            }] },
+            "terminal_event": {
+                "task": "compile-linux-1",
+                "task_definition": "compile-linux"
+            }
+        }),
         task_events: json!({ "events": [] }),
         placed_task_launched: true,
         status_code: Some(1),
@@ -227,10 +243,7 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
         .recent_output
         .iter()
         .any(|line| line.contains("task failed")));
-    assert_eq!(
+    assert_eq!(state.threads.len(), 1);
-        state.threads[&LINUX_THREAD].state,
-        DebugRuntimeState::Running
-    );
 }
 
 #[test]
@@ -283,7 +296,7 @@ fn breakpoint_resolution_uses_bundle_debug_probe_metadata() {
     let mut state = AdapterState::default();
     state.debug_probes = vec![BundleDebugProbe {
         id: "probe-linux".to_owned(),
-        source_path: "src/build.rs".to_owned(),
+        source_path: "src/lib.rs".to_owned(),
         line_start: 20,
         line_end: 30,
         function: "compile_linux".to_owned(),
@@ -311,7 +324,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
 
     let mut state = AdapterState::default();
     state.project = project.path().to_string_lossy().into_owned();
-    state.source_path = "src/build.rs".to_owned();
+    state.source_path = "src/lib.rs".to_owned();
     state.breakpoints = vec![1];
 
     let requested_source = source_dir.join("main.rs");
@@ -319,9 +332,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
 
     assert_eq!(resolved.len(), 1);
     assert!(!resolved[0].verified);
-    assert!(resolved[0]
+    assert!(resolved[0].message.contains("configured for `src/lib.rs`"));
-        .message
-        .contains("configured for `src/build.rs`"));
     assert_eq!(state.breakpoints, vec![1]);
 }
 
@@ -381,10 +392,19 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
     state.apply_runtime_record(RuntimeLaunchRecord {
         coordinator: "127.0.0.1:7999".to_owned(),
         node: "node".to_owned(),
-        node_report: json!({ "terminal_event": {
+        node_report: json!({
-            "task": "compile-linux-1",
+            "task_snapshots": { "snapshots": [{
-            "task_definition": "compile-linux"
+                "task": "compile-linux-1",
-        } }),
+                "task_definition": "compile-linux",
+                "attempt_id": "attempt-1",
+                "current": true,
+                "state": "running"
+            }] },
+            "terminal_event": {
+                "task": "compile-linux-1",
+                "task_definition": "compile-linux"
+            }
+        }),
         task_events: json!({ "events": [] }),
         placed_task_launched: true,
         status_code: Some(0),
@@ -653,14 +673,73 @@ fn terminal_events_do_not_resurrect_threads_absent_from_current_snapshots() {
 }
 
 #[test]
-fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
+fn terminal_record_without_snapshots_clears_active_threads() {
     let mut state = AdapterState::default();
-    let project = Path::new(env!("CARGO_MANIFEST_DIR"))
+    state.runtime_backend = RuntimeBackend::LiveServices;
-        .join("../../examples/launch-build-demo")
+    assert!(!state.threads.is_empty());
-        .canonicalize()
+
+    state.apply_runtime_record(RuntimeLaunchRecord {
+        coordinator: "127.0.0.1:9443".to_owned(),
+        node: "coordinator-main".to_owned(),
+        node_report: json!({ "terminal_event": { "terminal_state": "completed" } }),
+        task_events: json!({ "events": [] }),
+        placed_task_launched: true,
+        status_code: Some(0),
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        artifact_path: None,
+        event_count: 0,
+        debug_epoch: None,
+        stopped_task: None,
+        stopped_probe_symbol: None,
+        all_participants_frozen: false,
+    });
+
+    assert!(state.threads.is_empty());
+}
+
+#[test]
+fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-source-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        r#"async fn run_build_workflow() {
+    let source = prepare_source_snapshot();
+    let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
+        .name("compile linux")
+        .env(linux_env())
+        .start()
+        .await
         .unwrap();
+    let linux_thread = linux.virtual_thread_id();
+    let linux_artifact = linux.join().await.unwrap();
+    let package = clusterflux::spawn::async_task_with_arg(
+        vec![linux_artifact.clone()],
+        package_release,
+    )
+    .name("package artifacts")
+    .env(linux_env())
+    .start()
+    .await
+    .unwrap();
+    let package_artifact = package.join().await.unwrap();
+}
+"#,
+    )
+    .unwrap();
+
+    let mut state = AdapterState::default();
     state.project = project.to_string_lossy().into_owned();
-    state.source_path = "src/build.rs".to_owned();
+    state.source_path = "src/lib.rs".to_owned();
     let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
     let inspection_line = source
         .lines()
@@ -694,6 +773,8 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
         variable["name"] == "unavailable-local-diagnostic"
             && variable["type"] == "unavailable-local"
     }));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -813,12 +894,21 @@ fn package_release(inputs: Vec<Artifact>) -> Artifact {
 
 #[test]
 fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-wasm-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n    input + 1\n}\n",
+    )
+    .unwrap();
+
     let mut state = AdapterState::default();
-    state.project = Path::new(env!("CARGO_MANIFEST_DIR"))
+    state.project = project.to_string_lossy().into_owned();
-        .join("../../examples/launch-build-demo")
+    state.source_path = "src/lib.rs".to_owned();
-        .to_string_lossy()
-        .into_owned();
-    state.source_path = "src/build.rs".to_owned();
     let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
     state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
         .lines()
@@ -845,6 +935,8 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
     assert!(!locals
         .iter()
         .any(|variable| variable["name"] == "wasm-local-diagnostic"));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -871,3 +963,188 @@ fn detects_current_failed_attempt_awaiting_operator_action() {
         Some(("task-current", "attempt-current"))
     );
 }
+
+#[test]
+fn whole_process_terminal_status_covers_every_current_logical_task_attempt() {
+    let snapshots = |items: serde_json::Value| json!({ "snapshots": items });
+
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful main and final child must succeed"
+    );
+    for (state, status_code) in [("failed", 23), ("cancelled", 1)] {
+        assert_eq!(
+            whole_process_status_code(
+                Some(0),
+                &snapshots(json!([
+                    {
+                        "task": "child",
+                        "current": true,
+                        "state": state,
+                        "status_code": status_code
+                    }
+                ]))
+            ),
+            Some(status_code),
+            "a terminal child must determine the whole-process result"
+        );
+    }
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "accepted-failure",
+                    "current": true,
+                    "state": "failed",
+                    "command_state": "failure_accepted",
+                    "status_code": 17
+                }
+            ]))
+        ),
+        Some(17),
+        "accepting an AwaitOperator failure releases the process but does not turn failure into success"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-1",
+                    "current": false,
+                    "state": "failed",
+                    "status_code": 7
+                },
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-2",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful replacement attempt is authoritative over stale failed attempts"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(9),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(9),
+        "a failed coordinator main cannot be masked by successful children"
+    );
+}
+
+#[test]
+fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
+    let mut state = AdapterState::default();
+    state.runtime_backend = RuntimeBackend::LiveServices;
+    let record = |snapshots: serde_json::Value| RuntimeLaunchRecord {
+        coordinator: "127.0.0.1:9443".to_owned(),
+        node: "node-a".to_owned(),
+        node_report: json!({
+            "task_snapshots": { "snapshots": snapshots },
+            "process_status": null
+        }),
+        task_events: json!({ "events": [] }),
+        placed_task_launched: true,
+        status_code: None,
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        artifact_path: None,
+        event_count: 0,
+        debug_epoch: None,
+        stopped_task: None,
+        stopped_probe_symbol: None,
+        all_participants_frozen: false,
+    };
+    state.apply_attach_record(record(json!([
+        {
+            "task": "lane-stable",
+            "task_definition": "build_lane",
+            "attempt_id": "stable-1",
+            "current": true,
+            "state": "running"
+        },
+        {
+            "task": "lane-recovering",
+            "task_definition": "build_lane",
+            "attempt_id": "recovering-1",
+            "current": true,
+            "state": "failed_awaiting_action"
+        }
+    ])));
+    let stable_id = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
+        .unwrap()
+        .id;
+    let recovering_id = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-recovering"))
+        .unwrap()
+        .id;
+    assert_ne!(stable_id, recovering_id);
+
+    state.apply_runtime_record(record(json!([
+        {
+            "task": "lane-new",
+            "task_definition": "build_lane",
+            "attempt_id": "new-1",
+            "current": true,
+            "state": "running"
+        },
+        {
+            "task": "lane-stable",
+            "task_definition": "build_lane",
+            "attempt_id": "stable-2",
+            "current": true,
+            "state": "running"
+        }
+    ])));
+
+    let stable = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
+        .unwrap();
+    let new_lane = state
+        .threads
+        .values()
+        .find(|thread| thread.task == TaskInstanceId::from("lane-new"))
+        .unwrap();
+    assert_eq!(stable.id, stable_id);
+    assert_eq!(stable.attempt_id, "stable-2");
+    assert!(state
+        .threads
+        .values()
+        .all(|thread| thread.task != TaskInstanceId::from("lane-recovering")));
+    assert!(new_lane.id > recovering_id);
+}

crates/clusterflux-dap/src/virtual_model.rs

@@ -66,12 +66,16 @@ pub(crate) struct AdapterState {
     pub(crate) last_task_failed: bool,
     pub(crate) runtime_artifact_path: Option<String>,
     pub(crate) runtime_event_count: usize,
+    pub(crate) runtime_snapshot_fingerprint: String,
+    pub(crate) next_thread_id: i64,
     pub(crate) coordinator_debug_epoch: Option<u64>,
     pub(crate) debug_all_threads_stopped: bool,
     pub(crate) stopped_task: Option<TaskInstanceId>,
     pub(crate) stopped_probe_symbol: Option<String>,
     pub(crate) debug_probes: Vec<BundleDebugProbe>,
     pub(crate) breakpoints: Vec<i64>,
+    pub(crate) breakpoints_installed: bool,
+    pub(crate) breakpoint_revision: u64,
     pub(crate) threads: BTreeMap<i64, VirtualThread>,
 }
 
@@ -108,9 +112,9 @@ impl Default for AdapterState {
             threads: launch_threads(&entry),
             entry,
             project,
-            source_path: "src/build.rs".to_owned(),
+            source_path: "src/lib.rs".to_owned(),
             runtime_backend: RuntimeBackend::Simulated,
-            coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(),
+            coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(),
             tenant: TenantId::from("tenant"),
             project_id: ProjectId::from("project"),
             actor_user: UserId::from("dap"),
@@ -121,12 +125,16 @@ impl Default for AdapterState {
             last_task_failed: false,
             runtime_artifact_path: None,
             runtime_event_count: 0,
+            runtime_snapshot_fingerprint: String::new(),
+            next_thread_id: LINUX_THREAD,
             coordinator_debug_epoch: None,
             debug_all_threads_stopped: true,
             stopped_task: None,
             stopped_probe_symbol: None,
             debug_probes: Vec::new(),
             breakpoints: Vec::new(),
+            breakpoints_installed: false,
+            breakpoint_revision: 0,
         }
     }
 }
@@ -192,18 +200,24 @@ impl AdapterState {
             }
             self.coordinator_endpoint =
                 crate::view_state::normalize_coordinator_endpoint(&session.coordinator);
-            self.tenant = TenantId::new(session.tenant);
+            self.tenant = TenantId::try_new(session.tenant)
-            self.project_id = ProjectId::new(session.project);
+                .map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?;
-            self.actor_user = UserId::new(session.user);
+            self.project_id = ProjectId::try_new(session.project)
+                .map_err(|error| anyhow!("invalid project in CLI session: {error}"))?;
+            self.actor_user = UserId::try_new(session.user)
+                .map_err(|error| anyhow!("invalid user in CLI session: {error}"))?;
             self.client_session_secret = Some(session.session_secret);
         } else {
             self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(
                 coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"),
             );
             if let Some(scope) = project_scope {
-                self.tenant = TenantId::new(scope.tenant);
+                self.tenant = TenantId::try_new(scope.tenant)
-                self.project_id = ProjectId::new(scope.project);
+                    .map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?;
-                self.actor_user = UserId::new(scope.user);
+                self.project_id = ProjectId::try_new(scope.project)
+                    .map_err(|error| anyhow!("invalid project in project scope: {error}"))?;
+                self.actor_user = UserId::try_new(scope.user)
+                    .map_err(|error| anyhow!("invalid user in project scope: {error}"))?;
             }
             self.client_session_secret = None;
         }
@@ -213,6 +227,8 @@ impl AdapterState {
         self.last_task_failed = false;
         self.runtime_artifact_path = None;
         self.runtime_event_count = 0;
+        self.runtime_snapshot_fingerprint.clear();
+        self.next_thread_id = LINUX_THREAD;
         self.coordinator_debug_epoch = None;
         self.debug_all_threads_stopped = true;
         self.stopped_task = None;
@@ -221,6 +237,8 @@ impl AdapterState {
             crate::breakpoints::load_bundle_debug_probes(&self.project, &self.source_path);
         self.epoch = 0;
         self.breakpoints.clear();
+        self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated;
+        self.breakpoint_revision = 0;
         self.threads = if self.runtime_backend == RuntimeBackend::Simulated {
             launch_threads(&self.entry)
         } else {
@@ -233,13 +251,18 @@ impl AdapterState {
         self.coordinator_endpoint = record.coordinator.clone();
         if self.runtime_backend != RuntimeBackend::Simulated {
             if let Some(snapshots) = record.node_report.get("task_snapshots") {
-                self.threads = coordinator_threads_from_snapshots(
+                self.reconcile_runtime_threads(snapshots, record.node_report.get("process_status"));
-                    &self.entry,
+            } else if record.node_report.get("terminal_event").is_some() {
-                    snapshots,
+                self.threads.clear();
-                    record.node_report.get("process_status"),
-                );
             }
         }
+        if let Some(fingerprint) = record
+            .node_report
+            .get("snapshot_fingerprint")
+            .and_then(Value::as_str)
+        {
+            self.runtime_snapshot_fingerprint = fingerprint.to_owned();
+        }
         if !record.placed_task_launched {
             self.command_status = format!(
                 "virtual process started through {}; no runtime task event observed yet",
@@ -290,7 +313,10 @@ impl AdapterState {
             self.debug_all_threads_stopped = record.all_participants_frozen;
             self.epoch = record.debug_epoch.unwrap_or(self.epoch);
             self.coordinator_debug_epoch = record.debug_epoch;
-            self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new);
+            self.stopped_task = record
+                .stopped_task
+                .as_deref()
+                .and_then(|task| TaskInstanceId::try_new(task).ok());
             self.stopped_probe_symbol = record.stopped_probe_symbol.clone();
             if let Some(acknowledgements) = record
                 .node_report
@@ -307,6 +333,12 @@ impl AdapterState {
                     else {
                         continue;
                     };
+                    let Ok(task_id) = TaskInstanceId::try_new(task) else {
+                        continue;
+                    };
+                    let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+                        continue;
+                    };
                     let coordinator_main = acknowledgement.get("node").and_then(Value::as_str)
                         == Some("coordinator-main");
                     let thread_id = if coordinator_main {
@@ -316,14 +348,19 @@ impl AdapterState {
                             .values()
                             .find(|thread| thread.task.as_str() == task)
                             .map(|thread| thread.id)
-                            .unwrap_or_else(|| self.insert_runtime_thread(task, task_definition))
+                            .unwrap_or_else(|| {
+                                self.insert_runtime_thread(
+                                    task_id.clone(),
+                                    task_definition_id.clone(),
+                                )
+                            })
                     };
                     let thread = self
                         .threads
                         .get_mut(&thread_id)
                         .expect("runtime thread was found or inserted");
-                    thread.task = TaskInstanceId::new(task);
+                    thread.task = task_id;
-                    thread.task_definition = TaskDefinitionId::new(task_definition);
+                    thread.task_definition = task_definition_id;
                     thread.runtime_stack_frames = acknowledgement
                         .get("stack_frames")
                         .and_then(Value::as_array)
@@ -366,20 +403,12 @@ impl AdapterState {
                 .and_then(Value::as_str)
                 .unwrap_or(self.entry.as_str())
                 .to_owned();
-            let terminal_task_definition = terminal_event
-                .get("task_definition")
-                .and_then(Value::as_str)
-                .unwrap_or(self.entry.as_str())
-                .to_owned();
             let terminal_thread_id = self
                 .threads
                 .values()
                 .find(|thread| thread.task.as_str() == terminal_task.as_str())
-                .map(|thread| thread.id)
+                .map(|thread| thread.id);
-                .unwrap_or_else(|| {
+            if let Some(thread) = terminal_thread_id.and_then(|id| self.threads.get_mut(&id)) {
-                    self.insert_runtime_thread(&terminal_task, &terminal_task_definition)
-                });
-            if let Some(thread) = self.threads.get_mut(&terminal_thread_id) {
                 thread.stdout_bytes = record.stdout_bytes;
                 thread.stderr_bytes = record.stderr_bytes;
                 thread.stdout_tail = record.stdout_tail.clone();
@@ -413,24 +442,22 @@ impl AdapterState {
         let _ = crate::view_state::write_view_state(self, &record);
     }
 
-    fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 {
+    fn insert_runtime_thread(
-        let id = self
+        &mut self,
-            .threads
+        task: TaskInstanceId,
-            .keys()
+        task_definition: TaskDefinitionId,
-            .next_back()
+    ) -> i64 {
-            .copied()
+        let id = self.allocate_runtime_thread_id();
-            .unwrap_or(MAIN_THREAD)
-            + 1;
         let line = self
             .debug_probes
             .iter()
             .find(|probe| {
-                probe.task.as_str() == task_definition || probe.function == task_definition
+                probe.task == task_definition || probe.function == task_definition.as_str()
             })
             .map(|probe| i64::from(probe.line_start))
             .unwrap_or(1);
-        let definition_name = task_definition.replace(['_', '-'], " ");
+        let definition_name = task_definition.as_str().replace(['_', '-'], " ");
-        let name = if task == task_definition {
+        let name = if task.as_str() == task_definition.as_str() {
             definition_name
         } else {
             format!("{definition_name} ({task})")
@@ -448,9 +475,9 @@ impl AdapterState {
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::new(task),
+                task,
                 attempt_id: "unknown-attempt".to_owned(),
-                task_definition: TaskDefinitionId::new(task_definition),
+                task_definition,
                 name,
                 line,
                 state: DebugRuntimeState::Running,
@@ -476,6 +503,51 @@ impl AdapterState {
         id
     }
 
+    fn allocate_runtime_thread_id(&mut self) -> i64 {
+        while self.threads.contains_key(&self.next_thread_id) || self.next_thread_id == MAIN_THREAD
+        {
+            self.next_thread_id += 1;
+        }
+        let id = self.next_thread_id;
+        self.next_thread_id += 1;
+        id
+    }
+
+    fn reconcile_runtime_threads(
+        &mut self,
+        task_snapshots: &Value,
+        process_status: Option<&Value>,
+    ) {
+        let desired =
+            coordinator_threads_from_snapshots(&self.entry, task_snapshots, process_status);
+        let previous = std::mem::take(&mut self.threads);
+        let mut reconciled = BTreeMap::new();
+        for (_, mut thread) in desired {
+            let coordinator_main = process_status
+                .and_then(|status| status.get("main_task_instance"))
+                .and_then(Value::as_str)
+                .is_some_and(|task| task == thread.task.as_str());
+            let existing = if coordinator_main {
+                previous.get(&MAIN_THREAD)
+            } else {
+                previous
+                    .values()
+                    .find(|current| current.task == thread.task)
+            };
+            if let Some(existing) = existing {
+                preserve_thread_identity(&mut thread, existing);
+            } else if !coordinator_main {
+                thread.id = self.allocate_runtime_thread_id();
+                assign_thread_references(&mut thread);
+            } else {
+                thread.id = MAIN_THREAD;
+                assign_thread_references(&mut thread);
+            }
+            reconciled.insert(thread.id, thread);
+        }
+        self.threads = reconciled;
+    }
+
     pub(crate) fn apply_attach_record(&mut self, record: RuntimeLaunchRecord) {
         self.coordinator_endpoint = record.coordinator.clone();
         self.command_status = format!(
@@ -486,8 +558,7 @@ impl AdapterState {
         self.last_task_failed = false;
         self.runtime_artifact_path = None;
         self.runtime_event_count = record.event_count;
-        self.threads = coordinator_threads_from_snapshots(
+        self.reconcile_runtime_threads(
-            &self.entry,
             record
                 .node_report
                 .get("task_snapshots")
@@ -508,6 +579,35 @@ impl AdapterState {
     }
 }
 
+fn preserve_thread_identity(thread: &mut VirtualThread, existing: &VirtualThread) {
+    thread.id = existing.id;
+    thread.frame_id = existing.frame_id;
+    thread.locals_ref = existing.locals_ref;
+    thread.wasm_locals_ref = existing.wasm_locals_ref;
+    thread.args_ref = existing.args_ref;
+    thread.runtime_ref = existing.runtime_ref;
+    thread.output_ref = existing.output_ref;
+    thread.target_ref = existing.target_ref;
+    thread.vfs_ref = existing.vfs_ref;
+    thread.command_ref = existing.command_ref;
+    if thread.recent_output.is_empty() {
+        thread.recent_output = existing.recent_output.clone();
+    }
+}
+
+fn assign_thread_references(thread: &mut VirtualThread) {
+    let id = thread.id;
+    thread.frame_id = 1000 + id;
+    thread.locals_ref = 5000 + id;
+    thread.wasm_locals_ref = 9000 + id;
+    thread.args_ref = 2000 + id;
+    thread.runtime_ref = 3000 + id;
+    thread.output_ref = 4000 + id;
+    thread.target_ref = 6000 + id;
+    thread.vfs_ref = 7000 + id;
+    thread.command_ref = 8000 + id;
+}
+
 fn value_string_pairs(value: Option<&Value>) -> Vec<(String, String)> {
     value
         .and_then(Value::as_array)
@@ -643,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId {
     ProcessId::new(format!("vp-{suffix}"))
 }
 
-fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread {
+fn coordinator_main_thread(
+    entry: &str,
+    task: TaskInstanceId,
+    task_definition: TaskDefinitionId,
+) -> VirtualThread {
+    let name = format!("{entry} coordinator main ({task})");
     VirtualThread {
         id: MAIN_THREAD,
         frame_id: 1_001,
@@ -655,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi
         target_ref: 4_501,
         vfs_ref: 5_001,
         command_ref: 5_501,
-        task: TaskInstanceId::from(task),
+        task,
         attempt_id: "main:unknown".to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
+        task_definition,
-        name: format!("{entry} coordinator main ({task})"),
+        name,
         line: 0,
         state: DebugRuntimeState::Running,
         freeze_supported: true,
@@ -694,10 +799,14 @@ fn coordinator_threads_from_events(
         .expect("launch thread model should include main");
     if let Some(status) = process_status {
         if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) {
-            main.task = TaskInstanceId::from(task);
+            if let Ok(task) = TaskInstanceId::try_new(task) {
+                main.task = task;
+            }
         }
         if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) {
-            main.task_definition = TaskDefinitionId::from(definition);
+            if let Ok(definition) = TaskDefinitionId::try_new(definition) {
+                main.task_definition = definition;
+            }
         }
         main.name = format!("{entry} coordinator main ({})", main.task);
         main.state = if status
@@ -759,6 +868,12 @@ fn coordinator_threads_from_snapshots(
                 .get("main_task_definition")
                 .and_then(Value::as_str)
                 .unwrap_or(entry);
+            let (Ok(task), Ok(task_definition)) = (
+                TaskInstanceId::try_new(task),
+                TaskDefinitionId::try_new(task_definition),
+            ) else {
+                return threads;
+            };
             let mut main = coordinator_main_thread(entry, task, task_definition);
             main.attempt_id = format!(
                 "main:{}",
@@ -813,6 +928,12 @@ fn coordinator_threads_from_snapshots(
         let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else {
             continue;
         };
+        let Ok(task_id) = TaskInstanceId::try_new(task) else {
+            continue;
+        };
+        let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+            continue;
+        };
         let attempt_id = snapshot
             .get("attempt_id")
             .and_then(Value::as_str)
@@ -863,9 +984,9 @@ fn coordinator_threads_from_snapshots(
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::from(task),
+                task: task_id,
                 attempt_id,
-                task_definition: TaskDefinitionId::from(task_definition),
+                task_definition: task_definition_id,
                 name: format!("{display_name} ({task}, attempt {short_attempt})"),
                 line: snapshot
                     .get("source_line")
@@ -912,6 +1033,8 @@ fn coordinator_threads_from_snapshots(
 fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option<VirtualThread> {
     let task = event.get("task").and_then(Value::as_str)?;
     let task_definition = event.get("task_definition").and_then(Value::as_str)?;
+    let task_id = TaskInstanceId::try_new(task).ok()?;
+    let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?;
     let definition_name = task_definition.replace(['_', '-'], " ");
     let name = if task == task_definition {
         definition_name
@@ -957,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti
         target_ref: 6000 + id,
         vfs_ref: 7000 + id,
         command_ref: 8000 + id,
-        task: TaskInstanceId::from(task),
+        task: task_id,
         attempt_id: event
             .get("attempt_id")
             .and_then(Value::as_str)
             .unwrap_or("unknown-attempt")
             .to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
+        task_definition: task_definition_id,
         name,
         line: 0,
         state,

crates/clusterflux-node/src/assignment_runner.rs

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::io::Read;
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant};
@@ -46,6 +46,40 @@ use validation::{
     resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
 };
 
+#[derive(Debug)]
+struct AssignmentExecutionError {
+    message: String,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
+}
+
+impl std::fmt::Display for AssignmentExecutionError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for AssignmentExecutionError {}
+
+fn execution_error_with_log_bytes(
+    message: impl Into<String>,
+    stdout_source_bytes: &AtomicU64,
+    stderr_source_bytes: &AtomicU64,
+) -> Box<dyn std::error::Error> {
+    Box::new(AssignmentExecutionError {
+        message: message.into(),
+        stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed),
+        stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed),
+    })
+}
+
+pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) {
+    error
+        .downcast_ref::<AssignmentExecutionError>()
+        .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes))
+        .unwrap_or((0, 0))
+}
+
 pub(crate) fn run_verified_wasmtime_assignment(
     args: &Args,
     task: &RuntimeTask,
@@ -95,6 +129,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
             return Err("Wasm entrypoint assignment omitted its descriptor export".into());
         }
     };
+    let command_stdout_source_bytes = Arc::new(AtomicU64::new(0));
+    let command_stderr_source_bytes = Arc::new(AtomicU64::new(0));
     let (stdout, boundary_result) = match abi {
         WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
             let invocation = WasmTaskInvocation::new(
@@ -102,18 +138,28 @@ pub(crate) fn run_verified_wasmtime_assignment(
                 task_spec.task_instance.clone(),
                 task_spec.args.clone(),
             );
-            let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
+            let result = WasmtimeTaskRuntime::new()?
-                &module,
+                .run_task_export_verified_with_task_host(
-                expected_bundle_digest,
-                export,
-                &invocation,
-                Box::new(CoordinatorWasmTaskHost::new(
-                    args,
-                    task,
-                    node_private_key,
                     &module,
-                )?),
+                    expected_bundle_digest,
-            )?;
+                    export,
+                    &invocation,
+                    Box::new(CoordinatorWasmTaskHost::new(
+                        args,
+                        task,
+                        node_private_key,
+                        &module,
+                        Arc::clone(&command_stdout_source_bytes),
+                        Arc::clone(&command_stderr_source_bytes),
+                    )?),
+                )
+                .map_err(|error| {
+                    execution_error_with_log_bytes(
+                        error.to_string(),
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    )
+                })?;
             if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
                 eprintln!(
                     "clusterflux debug control: Wasm assignment returned for task {}",
@@ -122,17 +168,35 @@ pub(crate) fn run_verified_wasmtime_assignment(
             }
             match result.outcome {
                 WasmTaskOutcome::Completed => {
-                    let boundary = result.result.ok_or("completed Wasm task omitted result")?;
+                    let boundary = result.result.ok_or_else(|| {
+                        execution_error_with_log_bytes(
+                            "completed Wasm task omitted result",
+                            &command_stdout_source_bytes,
+                            &command_stderr_source_bytes,
+                        )
+                    })?;
                     (
-                        format!("{}\n", serde_json::to_string(&boundary)?),
+                        format!(
+                            "{}\n",
+                            serde_json::to_string(&boundary).map_err(|error| {
+                                execution_error_with_log_bytes(
+                                    error.to_string(),
+                                    &command_stdout_source_bytes,
+                                    &command_stderr_source_bytes,
+                                )
+                            })?
+                        ),
                         Some(boundary),
                     )
                 }
                 WasmTaskOutcome::Failed => {
-                    return Err(result
+                    return Err(execution_error_with_log_bytes(
-                        .error
+                        result
-                        .unwrap_or_else(|| "Wasm task failed without an error".to_owned())
+                            .error
-                        .into())
+                            .unwrap_or_else(|| "Wasm task failed without an error".to_owned()),
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    ))
                 }
             }
         }
@@ -140,12 +204,18 @@ pub(crate) fn run_verified_wasmtime_assignment(
     let task_id = TaskInstanceId::new(task.task.clone());
     let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
     let manifest = artifacts.flush();
+    let stdout_source_bytes = command_stdout_source_bytes
+        .load(Ordering::Relaxed)
+        .saturating_add(stdout.len() as u64);
+    let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed);
     Ok((
         CommandOutput {
             virtual_thread: task_id,
             status_code: Some(0),
             stdout,
             stderr: String::new(),
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: false,
             stderr_truncated: false,
             log_backpressured: false,
@@ -176,6 +246,8 @@ struct CoordinatorWasmTaskHost {
     next_handle_id: u64,
     handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
     command_status: Arc<Mutex<Option<String>>>,
+    command_stdout_source_bytes: Arc<AtomicU64>,
+    command_stderr_source_bytes: Arc<AtomicU64>,
     cancellation_requested: Arc<AtomicBool>,
     abort_requested: Arc<AtomicBool>,
     debug_control: Arc<WasmDebugControl>,
@@ -188,6 +260,8 @@ impl CoordinatorWasmTaskHost {
         parent: &RuntimeTask,
         node_private_key: &str,
         module: &[u8],
+        command_stdout_source_bytes: Arc<AtomicU64>,
+        command_stderr_source_bytes: Arc<AtomicU64>,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let task_spec = parent
             .task_spec
@@ -268,6 +342,8 @@ impl CoordinatorWasmTaskHost {
             next_handle_id: 1,
             handles,
             command_status,
+            command_stdout_source_bytes,
+            command_stderr_source_bytes,
             cancellation_requested,
             abort_requested,
             debug_control,
@@ -293,7 +369,16 @@ impl CoordinatorWasmTaskHost {
             runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
         let execution =
             run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
-        let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
+        let (
+            terminal_state,
+            status_code,
+            stdout,
+            stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
+            result,
+            retained,
+        ) = match execution {
             Ok((output, _manifest, result)) => {
                 let retained = retained_result_artifact(
                     self.args.project_root.as_deref(),
@@ -306,20 +391,40 @@ impl CoordinatorWasmTaskHost {
                         output.status_code,
                         output.stdout,
                         output.stderr,
+                        output.stdout_source_bytes,
+                        output.stderr_source_bytes,
                         result,
                         retained,
                     ),
-                    Err(error) => ("failed", Some(1), String::new(), error, None, None),
+                    Err(error) => (
+                        "failed",
+                        Some(1),
+                        String::new(),
+                        error.clone(),
+                        output.stdout_source_bytes,
+                        output
+                            .stderr_source_bytes
+                            .saturating_add(error.len() as u64),
+                        None,
+                        None,
+                    ),
                 }
             }
-            Err(error) => (
+            Err(error) => {
-                "failed",
+                let (stdout_source_bytes, stderr_source_bytes) =
-                Some(1),
+                    assignment_error_log_bytes(error.as_ref());
-                String::new(),
+                let error = error.to_string();
-                error.to_string(),
+                (
-                None,
+                    "failed",
-                None,
+                    Some(1),
-            ),
+                    String::new(),
+                    error.clone(),
+                    stdout_source_bytes,
+                    stderr_source_bytes.saturating_add(error.len() as u64),
+                    None,
+                    None,
+                )
+            }
         };
         let artifact_path = retained
             .as_ref()
@@ -339,8 +444,8 @@ impl CoordinatorWasmTaskHost {
                     "task": runtime_task.task,
                     "terminal_state": terminal_state,
                     "status_code": status_code,
-                    "stdout_bytes": stdout.len(),
+                    "stdout_bytes": stdout_source_bytes,
-                    "stderr_bytes": stderr.len(),
+                    "stderr_bytes": stderr_source_bytes,
                     "stdout_tail": stdout,
                     "stderr_tail": stderr,
                     "stdout_truncated": false,
@@ -676,6 +781,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost {
         let mut runner = CoordinatorControlledProcessRunner::new(
             self,
             Duration::from_millis(request.timeout_ms),
+            configured_secrets.clone(),
         );
         let output = LinuxRootlessPodmanBackend
             .execute_local_checkout_task(

crates/clusterflux-node/src/assignment_runner/process_runner.rs

@@ -1,4 +1,83 @@
 use super::*;
+use std::sync::mpsc::{self, Receiver, SyncSender};
+
+struct LiveLogChunk {
+    stream: &'static str,
+    offset: u64,
+    source_bytes: u64,
+    bytes: Vec<u8>,
+    truncated: bool,
+}
+
+fn append_bounded_tail(tail: &mut Vec<u8>, bytes: &[u8], maximum: usize) {
+    if maximum == 0 {
+        tail.clear();
+        return;
+    }
+    if bytes.len() >= maximum {
+        tail.clear();
+        tail.extend_from_slice(&bytes[bytes.len() - maximum..]);
+        return;
+    }
+    let overflow = tail
+        .len()
+        .saturating_add(bytes.len())
+        .saturating_sub(maximum);
+    if overflow > 0 {
+        tail.drain(..overflow);
+    }
+    tail.extend_from_slice(bytes);
+}
+
+fn redact_safe_live_log_prefix(
+    pending: &[u8],
+    configured_secrets: &[String],
+    final_chunk: bool,
+) -> Option<(usize, String)> {
+    if pending.is_empty() {
+        return None;
+    }
+    let secret_bytes = configured_secrets
+        .iter()
+        .filter(|secret| secret.len() >= 4)
+        .map(String::as_bytes)
+        .collect::<Vec<_>>();
+    let maximum_secret_bytes = secret_bytes
+        .iter()
+        .map(|secret| secret.len())
+        .max()
+        .unwrap_or(0);
+    if !final_chunk && pending.len() <= maximum_secret_bytes {
+        return None;
+    }
+    let mut consumed = if final_chunk {
+        pending.len()
+    } else {
+        pending.len() - maximum_secret_bytes
+    };
+    loop {
+        let previous = consumed;
+        for secret in &secret_bytes {
+            for start in 0..=pending.len().saturating_sub(secret.len()) {
+                let end = start + secret.len();
+                if start < consumed && end > consumed && &pending[start..end] == *secret {
+                    consumed = end;
+                }
+            }
+        }
+        if consumed == previous {
+            break;
+        }
+    }
+    if consumed == 0 {
+        return None;
+    }
+    let text = redact_configured_values(
+        String::from_utf8_lossy(&pending[..consumed]).into_owned(),
+        configured_secrets,
+    );
+    Some((consumed, text))
+}
 
 pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) args: Args,
@@ -7,13 +86,20 @@ pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) node_private_key: String,
     pub(super) debug_control: Arc<WasmDebugControl>,
     pub(super) command_status: Arc<Mutex<Option<String>>>,
+    pub(super) stdout_source_bytes: Arc<AtomicU64>,
+    pub(super) stderr_source_bytes: Arc<AtomicU64>,
     pub(super) timeout: Duration,
+    pub(super) configured_secrets: Vec<String>,
 }
 
 impl CoordinatorControlledProcessRunner {
     const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
 
-    pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
+    pub(super) fn new(
+        host: &CoordinatorWasmTaskHost,
+        timeout: Duration,
+        configured_secrets: Vec<String>,
+    ) -> Self {
         Self {
             args: host.args.clone(),
             process: host.process.clone(),
@@ -21,7 +107,10 @@ impl CoordinatorControlledProcessRunner {
             node_private_key: host.node_private_key.clone(),
             debug_control: Arc::clone(&host.debug_control),
             command_status: Arc::clone(&host.command_status),
+            stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes),
+            stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes),
             timeout,
+            configured_secrets,
         }
     }
 
@@ -201,10 +290,18 @@ impl CoordinatorControlledProcessRunner {
     fn drain_bounded(
         mut reader: impl Read + Send + 'static,
         maximum: usize,
+        stream: &'static str,
+        sender: SyncSender<LiveLogChunk>,
+        configured_secrets: Vec<String>,
+        source_bytes_total: Arc<AtomicU64>,
     ) -> thread::JoinHandle<Result<Vec<u8>, String>> {
         thread::spawn(move || {
             let mut captured = Vec::new();
             let mut buffer = [0_u8; 16 * 1024];
+            let stream_base = source_bytes_total.load(Ordering::Relaxed);
+            let mut source_bytes_read = 0_u64;
+            let mut pending_offset = stream_base;
+            let mut pending = Vec::new();
             loop {
                 let count = reader
                     .read(&mut buffer)
@@ -212,12 +309,128 @@ impl CoordinatorControlledProcessRunner {
                 if count == 0 {
                     break;
                 }
-                let remaining = maximum.saturating_sub(captured.len());
+                let _ = source_bytes_total.fetch_update(
-                captured.extend_from_slice(&buffer[..count.min(remaining)]);
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                    |current| Some(current.saturating_add(count as u64)),
+                );
+                source_bytes_read = source_bytes_read.saturating_add(count as u64);
+                append_bounded_tail(&mut captured, &buffer[..count], maximum);
+                pending.extend_from_slice(&buffer[..count]);
+                if let Some((consumed, text)) =
+                    redact_safe_live_log_prefix(&pending, &configured_secrets, false)
+                {
+                    let _ = sender.try_send(LiveLogChunk {
+                        stream,
+                        offset: pending_offset,
+                        source_bytes: consumed as u64,
+                        bytes: text.into_bytes(),
+                        truncated: false,
+                    });
+                    pending.drain(..consumed);
+                    pending_offset = pending_offset.saturating_add(consumed as u64);
+                }
+            }
+            if let Some((consumed, text)) =
+                redact_safe_live_log_prefix(&pending, &configured_secrets, true)
+            {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: pending_offset,
+                    source_bytes: consumed as u64,
+                    bytes: text.into_bytes(),
+                    truncated: false,
+                });
+            }
+            if source_bytes_read > maximum as u64 {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: stream_base.saturating_add(source_bytes_read),
+                    source_bytes: 0,
+                    bytes: b"[log output truncated at node capture limit]".to_vec(),
+                    truncated: true,
+                });
             }
             Ok(captured)
         })
     }
+
+    fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
+        let args = self.args.clone();
+        let process = self.process.clone();
+        let task = self.task.clone();
+        let node_private_key = self.node_private_key.clone();
+        let configured_secrets = self.configured_secrets.clone();
+        let command_status = Arc::clone(&self.command_status);
+        thread::spawn(move || {
+            let mut log_session = None;
+            let mut delivery_available = true;
+            while let Ok(chunk) = receiver.recv() {
+                if !delivery_available {
+                    continue;
+                }
+                let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
+                text = redact_configured_values(text, &configured_secrets);
+                let mut delivered = false;
+                for _ in 0..2 {
+                    if log_session.is_none() {
+                        log_session = CoordinatorSession::connect_with_timeouts(
+                            &args.coordinator,
+                            Duration::from_millis(500),
+                            Duration::from_millis(500),
+                        )
+                        .ok();
+                    }
+                    let Some(session) = log_session.as_mut() else {
+                        continue;
+                    };
+                    let request = signed_node_request_json(
+                        &args,
+                        &node_private_key,
+                        "report_task_log_chunk",
+                        serde_json::json!({
+                            "type": "report_task_log_chunk",
+                            "tenant": &args.tenant,
+                            "project": &args.project,
+                            "process": &process,
+                            "node": &args.node,
+                            "task": &task,
+                            "stream": chunk.stream,
+                            "offset": chunk.offset,
+                            "source_bytes": chunk.source_bytes,
+                            "text": &text,
+                            "truncated": chunk.truncated,
+                        }),
+                    );
+                    let result = match request {
+                        Ok(request) => session
+                            .request(request)
+                            .map(|_| ())
+                            .map_err(|error| error.to_string()),
+                        Err(error) => Err(error.to_string()),
+                    };
+                    match result {
+                        Ok(()) => {
+                            delivered = true;
+                            break;
+                        }
+                        Err(_) => {
+                            log_session = None;
+                        }
+                    }
+                }
+                if !delivered {
+                    delivery_available = false;
+                    if let Ok(mut current) = command_status.lock() {
+                        *current = Some(
+                            "live log delivery was interrupted; final bounded output remains available"
+                                .to_owned(),
+                        );
+                    }
+                }
+            }
+        })
+    }
 }
 
 impl ProcessRunner for CoordinatorControlledProcessRunner {
@@ -245,12 +458,17 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             command.program,
             command.args.join(" ")
         ));
+        let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
         let stdout = Self::drain_bounded(
             child
                 .stdout
                 .take()
                 .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stdout",
+            live_log_sender.clone(),
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stdout_source_bytes),
         );
         let stderr = Self::drain_bounded(
             child
@@ -258,11 +476,19 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 .take()
                 .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stderr",
+            live_log_sender,
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stderr_source_bytes),
         );
+        let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
         let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
             Ok(session) => session,
             Err(error) => {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
+                let _ = stdout.join();
+                let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "establish execution control channel: {error}"
                 )));
@@ -277,6 +503,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Ok(None) => {}
                 Err(error) => {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
+                    let _ = stdout.join();
+                    let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Command(error.to_string()));
                 }
             }
@@ -289,6 +518,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
                 let _ = stdout.join();
                 let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "native command exceeded wall-clock timeout of {} ms",
                     self.timeout.as_millis()
@@ -303,6 +533,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Cancelled(
                         "coordinator requested cancellation or abort".to_owned(),
                     ));
@@ -312,6 +543,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(error);
                 }
             }
@@ -360,6 +592,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             .join()
             .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
             .map_err(BackendError::Command)?;
+        live_log_reporter
+            .join()
+            .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
         self.set_command_status(format!(
             "native command exited with status {:?}",
             status.code()
@@ -371,3 +606,52 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner};
+    use std::io::Cursor;
+    use std::sync::atomic::{AtomicU64, Ordering};
+    use std::sync::{mpsc, Arc};
+
+    #[test]
+    fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
+        let secrets = vec!["correct-horse".to_owned()];
+        let mut pending = b"prefix correct-".to_vec();
+        let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
+        pending.drain(..consumed);
+        pending.extend_from_slice(b"horse suffix");
+        let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
+
+        let combined = format!("{first}{second}");
+        assert_eq!(combined, "prefix [REDACTED] suffix");
+        assert!(!combined.contains("correct-"));
+        assert!(!combined.contains("horse"));
+    }
+
+    #[test]
+    fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() {
+        let source_bytes = Arc::new(AtomicU64::new(5));
+        let (sender, receiver) = mpsc::sync_channel(8);
+        let reader = CoordinatorControlledProcessRunner::drain_bounded(
+            Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()),
+            8,
+            "stdout",
+            sender,
+            Vec::new(),
+            Arc::clone(&source_bytes),
+        );
+        let captured = reader.join().unwrap().unwrap();
+        let chunks = receiver.into_iter().collect::<Vec<_>>();
+
+        assert_eq!(captured, b"opqrstuv");
+        assert_eq!(source_bytes.load(Ordering::Relaxed), 37);
+        assert_eq!(
+            chunks.iter().map(|chunk| chunk.source_bytes).sum::<u64>(),
+            32
+        );
+        assert_eq!(chunks[0].offset, 5);
+        assert_eq!(chunks.last().unwrap().offset, 37);
+        assert!(chunks.iter().any(|chunk| chunk.truncated));
+    }
+}

crates/clusterflux-node/src/assignment_runner/tests.rs

@@ -48,7 +48,10 @@ fn test_controlled_runner(
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout,
+        configured_secrets: Vec::new(),
     }
 }
 
@@ -89,7 +92,10 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout: Duration::from_secs(30),
+        configured_secrets: Vec::new(),
     };
     let started = Instant::now();
     let error = runner

crates/clusterflux-node/src/command_runner.rs

@@ -1,6 +1,6 @@
 use clusterflux_core::{
-    CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
+    CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay,
-    VfsOverlay, VfsPath,
+    VfsPath,
 };
 use serde::{Deserialize, Serialize};
 
@@ -42,6 +42,8 @@ pub struct CommandOutput {
     pub status_code: Option<i32>,
     pub stdout: String,
     pub stderr: String,
+    pub stdout_source_bytes: u64,
+    pub stderr_source_bytes: u64,
     pub stdout_truncated: bool,
     pub stderr_truncated: bool,
     pub log_backpressured: bool,
@@ -83,6 +85,8 @@ impl LocalCommandExecutor {
             &output.stderr,
             max_log_bytes,
         );
+        let stdout_source_bytes = output.stdout.len() as u64;
+        let stderr_source_bytes = output.stderr.len() as u64;
         let staged_artifact = if let Some(path) = command.stage_stdout_as {
             Some(overlay.write(
                 path,
@@ -98,6 +102,8 @@ impl LocalCommandExecutor {
             status_code: output.status.code(),
             stdout: logs.stdout,
             stderr: logs.stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: logs.stdout_truncated,
             stderr_truncated: logs.stderr_truncated,
             log_backpressured: logs.backpressured,
@@ -107,24 +113,20 @@ impl LocalCommandExecutor {
 }
 
 pub(super) fn capture_command_logs(
-    task: &TaskInstanceId,
+    _task: &TaskInstanceId,
     stdout: &[u8],
     stderr: &[u8],
     max_log_bytes: usize,
 ) -> CapturedCommandLogs {
-    let mut logs = LogBuffer::new(max_log_bytes);
+    let stdout_truncated = stdout.len() > max_log_bytes;
-    logs.push(task.clone(), stdout);
+    let stderr_truncated = stderr.len() > max_log_bytes;
-    logs.push(task.clone(), stderr);
+    let stdout_start = stdout.len().saturating_sub(max_log_bytes);
-    let records = logs.records();
+    let stderr_start = stderr.len().saturating_sub(max_log_bytes);
-    let stdout_record = &records[0];
-    let stderr_record = &records[1];
-    debug_assert_eq!(&stdout_record.task, task);
-    debug_assert_eq!(&stderr_record.task, task);
     CapturedCommandLogs {
-        stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
+        stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(),
-        stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
+        stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(),
-        stdout_truncated: stdout_record.truncated,
+        stdout_truncated,
-        stderr_truncated: stderr_record.truncated,
+        stderr_truncated,
-        backpressured: logs.backpressured(),
+        backpressured: stdout_truncated || stderr_truncated,
     }
 }

crates/clusterflux-node/src/coordinator_session.rs

@@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity;
 use clusterflux_control::ControlSession;
 use clusterflux_core::coordinator_wire_request;
 use serde_json::Value;
+use std::time::Duration;
 
 pub(crate) struct CoordinatorSession {
     inner: ControlSession,
@@ -15,6 +16,16 @@ impl CoordinatorSession {
         })
     }
 
+    pub(crate) fn connect_with_timeouts(
+        addr: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        Ok(Self {
+            inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?,
+        })
+    }
+
     pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
         let request_id = format!("node-{}", self.inner.requests() + 1);
         let wire_request = coordinator_wire_request(request_id, value);

crates/clusterflux-node/src/daemon.rs

@@ -7,11 +7,11 @@ use std::time::{Duration, Instant};
 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
 use clusterflux_core::{
     sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
-    TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
+    ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
 };
 use serde_json::{json, Value};
 
-use crate::assignment_runner::run_verified_wasmtime_assignment;
+use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment};
 #[cfg(test)]
 use crate::coordinator_session::control_endpoint_identity;
 use crate::coordinator_session::CoordinatorSession;
@@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
     let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
     });
     let heartbeat_signature = sign_node_request(
@@ -222,7 +224,7 @@ fn worker_loop(
             let expiry = Instant::now()
                 .checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
                 .unwrap_or_else(Instant::now);
-            restart_pins.insert(ArtifactId::new(artifact), expiry);
+            restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry);
         }
         println!("{}", serde_json::to_string(&report)?);
         std::io::stdout().flush()?;
@@ -289,7 +291,7 @@ fn service_pending_artifact_transfer(
         return Ok(false);
     };
     let transfer_id = required_string(transfer, "transfer_id")?;
-    let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
+    let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?;
     let expected_digest: Digest = serde_json::from_value(
         transfer
             .get("expected_digest")
@@ -369,9 +371,11 @@ pub(crate) fn runtime_task_from_assignment(
             .cloned()
             .ok_or("task assignment missing task_spec")?,
     )?;
+    let process = ProcessId::try_new(required_string(value, "process")?)?;
+    let task = TaskInstanceId::try_new(required_string(value, "task")?)?;
     Ok(RuntimeTask {
-        process: required_string(value, "process")?,
+        process: process.to_string(),
-        task: required_string(value, "task")?,
+        task: task.to_string(),
         epoch: value.get("epoch").and_then(Value::as_u64),
         bundle_digest: task_spec.bundle_digest.clone(),
         task_spec: Some(task_spec),
@@ -418,6 +422,8 @@ fn run_runtime_task(
         "reconnect_node",
         json!({
         "type": "reconnect_node",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
         "process": &task.process,
         "epoch": epoch,
@@ -458,6 +464,8 @@ fn run_runtime_task(
             capability_report,
             debug_command,
             node_private_key,
+            0,
+            0,
         );
     }
 
@@ -492,9 +500,13 @@ fn run_runtime_task(
                 debug_command,
                 node_private_key,
                 &error,
+                output.stdout_source_bytes,
+                output.stderr_source_bytes,
             ),
         },
         Err(error) => {
+            let (stdout_source_bytes, stderr_source_bytes) =
+                assignment_error_log_bytes(error.as_ref());
             let error = error.to_string();
             if error.contains("task execution cancelled:") {
                 record_cancelled_task(
@@ -506,6 +518,8 @@ fn run_runtime_task(
                     capability_report,
                     debug_command,
                     node_private_key,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             } else {
                 record_failed_task(
@@ -518,6 +532,8 @@ fn run_runtime_task(
                     debug_command,
                     node_private_key,
                     &error,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             }
         }
@@ -570,6 +586,12 @@ fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
         }
     }
 
+    TenantId::try_new(tenant.clone())?;
+    ProjectId::try_new(project.clone())?;
+    NodeId::try_new(node.clone())?;
+    if let Some(grant) = enrollment_grant.as_ref() {
+        clusterflux_core::validate_opaque_token(grant, 512)?;
+    }
     Ok(Args {
         coordinator: coordinator.ok_or("--coordinator is required")?,
         tenant,
@@ -622,13 +644,12 @@ mod tests {
     #[test]
     fn hosted_url_remains_an_https_control_endpoint() {
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
+            control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(),
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
+            "https://clusterflux.lesstuff.com/api/v1/control"
         );
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
+            control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
-                .unwrap(),
+            "https://clusterflux.lesstuff.com/api/v1/control"
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
         );
         assert_eq!(
             control_endpoint_identity("127.0.0.1:7999").unwrap(),

crates/clusterflux-node/src/lib.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clusterflux_core::{
     Capability, CommandBackendKind, CommandInvocation, CommandPlan, Digest, EnvironmentKind,
@@ -9,6 +9,7 @@ use sha2::{Digest as ShaDigest, Sha256};
 use thiserror::Error;
 
 mod command_runner;
+mod source_snapshot;
 mod windows_dev;
 pub use clusterflux_wasm_runtime::{
     WasmDebugControl, WasmTaskError, WasmTaskHost, WasmtimeDebugProbe, WasmtimeTaskRuntime,
@@ -20,6 +21,10 @@ pub use command_runner::{
 };
 pub use windows_dev::{WindowsCommandDevBackend, WindowsSandboxStubBackend};
 
+pub fn snapshot_project_digest(project_root: &Path) -> Result<Digest, String> {
+    source_snapshot::snapshot_project(project_root).map(|snapshot| snapshot.digest)
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct MaterializedEnvironment {
     pub name: String,
@@ -736,7 +741,7 @@ mod tests {
     }
 
     #[test]
-    fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() {
+    fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() {
         let invocation = CommandInvocation {
             program: "cargo".to_owned(),
             args: vec!["build".to_owned()],
@@ -769,7 +774,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
+        assert_eq!(output.stdout, "cdef");
         assert!(output.stdout_truncated);
         assert!(output.log_backpressured);
         assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
@@ -987,7 +992,7 @@ mod tests {
 
     #[cfg(unix)]
     #[test]
-    fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() {
+    fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() {
         let executor = LocalCommandExecutor {
             node: clusterflux_core::NodeId::from("node"),
             hosted_control_plane: false,
@@ -1016,15 +1021,15 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
+        assert_eq!(output.stdout, "cdef");
-        assert_eq!(output.stderr, "");
+        assert_eq!(output.stderr, "err");
         assert!(output.stdout_truncated);
-        assert!(output.stderr_truncated);
+        assert!(!output.stderr_truncated);
         assert!(output.log_backpressured);
     }
 
     #[test]
-    fn public_node_crate_does_not_require_hosted_private_types() {
+    fn public_node_crate_does_not_require_hosted_service_types() {
         let _tenant = TenantId::from("tenant");
         let _project = ProjectId::from("project");
         let _backend = LinuxRootlessPodmanBackend;

crates/clusterflux-node/src/main.rs

@@ -8,5 +8,34 @@ mod task_artifacts;
 mod task_reports;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-node {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux node worker.\n\n\
+                 Usage: clusterflux-node --coordinator <URL> [OPTIONS]\n\n\
+                 Options:\n  \
+                   --coordinator <URL>\n  \
+                   --tenant <TENANT>                   [default: tenant]\n  \
+                   --project-id <PROJECT>              [default: project]\n  \
+                   --node <NODE>                       [default: node]\n  \
+                   --project-root <PATH>\n  \
+                   --enrollment-grant <GRANT>\n  \
+                   --public-key <KEY>\n  \
+                   --worker\n  \
+                   --emit-ready\n  \
+                   --control-poll-ms <MILLISECONDS>\n  \
+                   --assignment-poll-ms <MILLISECONDS> [default: 500]\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
     daemon::run()
 }

crates/clusterflux-node/src/task_reports.rs

@@ -47,8 +47,8 @@ pub(crate) fn record_completed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": output.stdout.len(),
+            "stdout_bytes": output.stdout_source_bytes,
-            "stderr_bytes": output.stderr.len(),
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -85,8 +85,8 @@ pub(crate) fn record_completed_task(
             "node": &args.node,
             "task": &task.task,
             "status_code": output.status_code,
-            "stdout_bytes": output.stdout.len(),
+            "stdout_bytes": output.stdout_source_bytes,
-            "stderr_bytes": output.stderr.len(),
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -123,8 +123,11 @@ pub(crate) fn record_failed_task(
     debug_command: Value,
     node_private_key: &str,
     error: &str,
+    stdout_source_bytes: u64,
+    command_stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let error = bounded_runtime_error(error);
+    let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64);
     let log_event = session.request(signed_node_request_json(
         args,
         node_private_key,
@@ -136,8 +139,8 @@ pub(crate) fn record_failed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": error.len(),
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -175,8 +178,8 @@ pub(crate) fn record_failed_task(
             "task": &task.task,
             "terminal_state": "failed",
             "status_code": -1,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": error.len(),
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -189,6 +192,8 @@ pub(crate) fn record_failed_task(
     Ok(failed_node_report(
         task,
         &error,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -210,6 +215,8 @@ pub(crate) fn record_cancelled_task(
     capability_report: Value,
     debug_command: Value,
     node_private_key: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let recorded = session.request(signed_node_request_json(
         args,
@@ -224,12 +231,12 @@ pub(crate) fn record_cancelled_task(
             "task": &task.task,
             "terminal_state": "cancelled",
             "status_code": null,
-            "stdout_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
-            "stderr_bytes": 0,
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": "",
-            "stdout_truncated": false,
+            "stdout_truncated": stdout_source_bytes > 0,
-            "stderr_truncated": false,
+            "stderr_truncated": stderr_source_bytes > 0,
             "artifact_path": null,
             "artifact_digest": null,
             "artifact_size_bytes": null,
@@ -238,6 +245,8 @@ pub(crate) fn record_cancelled_task(
     )?)?;
     Ok(cancelled_node_report(
         task,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -281,8 +290,8 @@ pub(crate) fn completed_node_report(
         "virtual_thread": output.virtual_thread,
         "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
         "status_code": output.status_code,
-        "stdout_bytes": output.stdout.len(),
+        "stdout_bytes": output.stdout_source_bytes,
-        "stderr_bytes": output.stderr.len(),
+        "stderr_bytes": output.stderr_source_bytes,
         "stdout_tail": &output.stdout,
         "stderr_tail": &output.stderr,
         "stdout_truncated": output.stdout_truncated,
@@ -305,6 +314,8 @@ pub(crate) fn completed_node_report(
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn cancelled_node_report(
     task: &RuntimeTask,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -318,12 +329,12 @@ pub(crate) fn cancelled_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "cancelled",
         "status_code": null,
-        "stdout_bytes": 0,
+        "stdout_bytes": stdout_source_bytes,
-        "stderr_bytes": 0,
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": "",
-        "stdout_truncated": false,
+        "stdout_truncated": stdout_source_bytes > 0,
-        "stderr_truncated": false,
+        "stderr_truncated": stderr_source_bytes > 0,
         "log_backpressured": false,
         "staged_artifact": null,
         "large_bytes_uploaded": false,
@@ -343,6 +354,8 @@ pub(crate) fn cancelled_node_report(
 pub(crate) fn failed_node_report(
     task: &RuntimeTask,
     error: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -357,8 +370,8 @@ pub(crate) fn failed_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "failed",
         "status_code": -1,
-        "stdout_bytes": 0,
+        "stdout_bytes": stdout_source_bytes,
-        "stderr_bytes": error.len(),
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": error,
         "stdout_truncated": false,
@@ -377,3 +390,40 @@ pub(crate) fn failed_node_report(
         "coordinator_response": coordinator_response,
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use clusterflux_core::TaskInstanceId;
+
+    #[test]
+    fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() {
+        let report = completed_node_report(
+            CommandOutput {
+                virtual_thread: TaskInstanceId::from("task"),
+                status_code: Some(0),
+                stdout: "bounded tail".to_owned(),
+                stderr: String::new(),
+                stdout_source_bytes: 519,
+                stderr_source_bytes: 0,
+                stdout_truncated: true,
+                stderr_truncated: false,
+                log_backpressured: false,
+                staged_artifact: None,
+            },
+            false,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            1,
+        );
+
+        assert_eq!(report["stdout_bytes"], 519);
+        assert_eq!(report["stdout_tail"], "bounded tail");
+    }
+}

crates/clusterflux-sdk/src/lib.rs

@@ -1,3 +1,5 @@
+extern crate self as clusterflux;
+
 use serde::Serialize;
 
 pub use clusterflux_core as core;
@@ -10,6 +12,26 @@ pub use task_args::{
     TaskArgBudget, TaskArgError, TaskArgKind, TaskArgValidation, Vfs,
 };
 
+pub use clusterflux_core::TaskFailurePolicy;
+pub use sdk_runtime::TaskRuntimeError as Error;
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub mod prelude {
+    pub use crate::{
+        command::Command, fs, source, Artifact, Result, SourceSnapshot, TaskFailurePolicy,
+    };
+}
+
+#[macro_export]
+macro_rules! spawn {
+    ($task:ident()) => {
+        $crate::spawn::__product_async_task($task, stringify!($task))
+    };
+    ($task:ident($argument:expr)) => {
+        $crate::spawn::__product_async_task_with_arg($argument, $task, stringify!($task))
+    };
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
 pub struct EntrypointDescriptor {
     pub name: &'static str,
@@ -63,6 +85,8 @@ impl RegisteredProgram {
 pub mod __private;
 
 pub mod spawn {
+    use std::future::{Future, IntoFuture};
+    use std::pin::Pin;
     use std::sync::atomic::{AtomicU64, Ordering};
     use std::sync::{Mutex, OnceLock};
 
@@ -115,6 +139,273 @@ pub mod spawn {
         runtime_threads().lock().unwrap().clone()
     }
 
+    #[doc(hidden)]
+    pub fn __product_async_task<F, Fut, R>(
+        entry: F,
+        task_id: &'static str,
+    ) -> ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        ProductAsyncTaskBuilder {
+            entry: Some(entry),
+            env: None,
+            name: task_id,
+            task_id,
+            failure_policy: TaskFailurePolicy::FailFast,
+            marker: std::marker::PhantomData,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn __product_async_task_with_arg<A, F, Fut, R>(
+        arg: A,
+        entry: F,
+        task_id: &'static str,
+    ) -> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        ProductAsyncTaskWithArgBuilder {
+            arg: Some(arg),
+            entry: Some(entry),
+            env: None,
+            name: task_id,
+            task_id,
+            arg_budget: TaskArgBudget::default(),
+            failure_policy: TaskFailurePolicy::FailFast,
+            marker: std::marker::PhantomData,
+        }
+    }
+
+    pub struct ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+        entry: Option<F>,
+        env: Option<EnvRef>,
+        name: &'static str,
+        task_id: &'static str,
+        failure_policy: TaskFailurePolicy,
+        marker: std::marker::PhantomData<fn() -> (Fut, R)>,
+    }
+
+    impl<F, Fut, R> ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg + DeserializeOwned,
+    {
+        pub fn on(mut self, env: EnvRef) -> Self {
+            self.env = Some(env);
+            self
+        }
+
+        pub fn name(mut self, name: &'static str) -> Self {
+            self.name = name;
+            self
+        }
+
+        pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
+            self.failure_policy = failure_policy;
+            self
+        }
+
+        pub async fn start(self) -> crate::Result<TaskHandle<R>> {
+            let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
+            let runtime_event = register_runtime_thread(id, self.name, self.env);
+            #[cfg(target_arch = "wasm32")]
+            {
+                let remote = start_guest_host_task(
+                    clusterflux_core::TaskDefinitionId::from(self.task_id),
+                    self.env,
+                    Vec::new(),
+                    self.failure_policy,
+                )?;
+                return Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: None,
+                    remote: Some(remote),
+                });
+            }
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                if let Some(config) = ProductRuntimeConfig::from_env()? {
+                    let remote = start_remote_task(
+                        config,
+                        self.task_id.to_owned(),
+                        self.env,
+                        Vec::new(),
+                        self.failure_policy,
+                    )?;
+                    return Ok(TaskHandle {
+                        virtual_thread_id: id,
+                        name: self.name,
+                        env: self.env,
+                        debugger_visible: runtime_event.debugger_visible,
+                        result: None,
+                        remote: Some(remote),
+                    });
+                }
+                let result = self.entry.expect("task entry used once")().await?;
+                Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: Some(result),
+                    remote: None,
+                })
+            }
+        }
+    }
+
+    impl<F, Fut, R> IntoFuture for ProductAsyncTaskBuilder<F, Fut, R>
+    where
+        F: FnOnce() -> Fut + 'static,
+        Fut: Future<Output = crate::Result<R>> + 'static,
+        R: TaskArg + DeserializeOwned + 'static,
+    {
+        type Output = crate::Result<TaskHandle<R>>;
+        type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
+
+        fn into_future(self) -> Self::IntoFuture {
+            Box::pin(self.start())
+        }
+    }
+
+    pub struct ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg,
+    {
+        arg: Option<A>,
+        #[cfg_attr(target_arch = "wasm32", allow(dead_code))]
+        entry: Option<F>,
+        env: Option<EnvRef>,
+        name: &'static str,
+        task_id: &'static str,
+        arg_budget: TaskArgBudget,
+        failure_policy: TaskFailurePolicy,
+        marker: std::marker::PhantomData<fn() -> (Fut, R)>,
+    }
+
+    impl<A, F, Fut, R> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg,
+        F: FnOnce(A) -> Fut,
+        Fut: Future<Output = crate::Result<R>>,
+        R: TaskArg + DeserializeOwned,
+    {
+        pub fn on(mut self, env: EnvRef) -> Self {
+            self.env = Some(env);
+            self
+        }
+
+        pub fn name(mut self, name: &'static str) -> Self {
+            self.name = name;
+            self
+        }
+
+        pub fn arg_budget(mut self, arg_budget: TaskArgBudget) -> Self {
+            self.arg_budget = arg_budget;
+            self
+        }
+
+        pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
+            self.failure_policy = failure_policy;
+            self
+        }
+
+        pub async fn start(self) -> crate::Result<TaskHandle<R>> {
+            let arg = self.arg.expect("task argument used once");
+            validate_task_arg(&arg, self.arg_budget)
+                .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+            let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
+            let runtime_event = register_runtime_thread(id, self.name, self.env);
+            #[cfg(target_arch = "wasm32")]
+            {
+                let boundary = arg
+                    .task_boundary_value()
+                    .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+                let remote = start_guest_host_task(
+                    clusterflux_core::TaskDefinitionId::from(self.task_id),
+                    self.env,
+                    vec![boundary],
+                    self.failure_policy,
+                )?;
+                return Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: None,
+                    remote: Some(remote),
+                });
+            }
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                if let Some(config) = ProductRuntimeConfig::from_env()? {
+                    let boundary = arg
+                        .task_boundary_value()
+                        .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
+                    let remote = start_remote_task(
+                        config,
+                        self.task_id.to_owned(),
+                        self.env,
+                        vec![boundary],
+                        self.failure_policy,
+                    )?;
+                    return Ok(TaskHandle {
+                        virtual_thread_id: id,
+                        name: self.name,
+                        env: self.env,
+                        debugger_visible: runtime_event.debugger_visible,
+                        result: None,
+                        remote: Some(remote),
+                    });
+                }
+                let result = self.entry.expect("task entry used once")(arg).await?;
+                Ok(TaskHandle {
+                    virtual_thread_id: id,
+                    name: self.name,
+                    env: self.env,
+                    debugger_visible: runtime_event.debugger_visible,
+                    result: Some(result),
+                    remote: None,
+                })
+            }
+        }
+    }
+
+    impl<A, F, Fut, R> IntoFuture for ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
+    where
+        A: TaskArg + 'static,
+        F: FnOnce(A) -> Fut + 'static,
+        Fut: Future<Output = crate::Result<R>> + 'static,
+        R: TaskArg + DeserializeOwned + 'static,
+    {
+        type Output = crate::Result<TaskHandle<R>>;
+        type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
+
+        fn into_future(self) -> Self::IntoFuture {
+            Box::pin(self.start())
+        }
+    }
+
     pub fn task<F, R>(entry: F) -> TaskBuilder<F, R>
     where
         F: FnOnce() -> R,
@@ -711,6 +1002,10 @@ pub mod command {
             self
         }
 
+        pub fn cwd(self, directory: impl Into<String>) -> Self {
+            self.current_dir(directory)
+        }
+
         pub fn env(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
             self.environment_variables.insert(name.into(), value.into());
             self
@@ -761,6 +1056,34 @@ pub mod command {
                     .to_owned(),
             ))
         }
+
+        pub async fn run(self) -> Result<Output, TaskRuntimeError> {
+            let program = self.program.clone();
+            let output = self.output().await?;
+            if output.status_code == Some(0) {
+                return Ok(output);
+            }
+            Err(TaskRuntimeError::CommandFailed {
+                program,
+                status_code: output.status_code,
+                stdout: bounded_tail(output.stdout),
+                stderr: bounded_tail(output.stderr),
+                stdout_truncated: output.stdout_truncated,
+                stderr_truncated: output.stderr_truncated,
+            })
+        }
+    }
+
+    fn bounded_tail(value: String) -> String {
+        const LIMIT: usize = 4 * 1024;
+        if value.len() <= LIMIT {
+            return value;
+        }
+        let mut start = value.len() - LIMIT;
+        while !value.is_char_boundary(start) {
+            start += 1;
+        }
+        value[start..].to_owned()
     }
 }
 
@@ -782,6 +1105,34 @@ pub mod source {
             "source snapshots are produced only by a source-capable Clusterflux node".to_owned(),
         ))
     }
+
+    pub struct CurrentProject;
+
+    pub fn current_project() -> CurrentProject {
+        CurrentProject
+    }
+
+    impl CurrentProject {
+        pub async fn snapshot(self) -> Result<SourceSnapshot, TaskRuntimeError> {
+            crate::spawn::__product_async_task(snapshot_current_project, "snapshot_current_project")
+                .await?
+                .join()
+                .await
+        }
+    }
+
+    #[clusterflux::task(capabilities = "source_filesystem")]
+    async fn snapshot_current_project() -> crate::Result<SourceSnapshot> {
+        snapshot().await
+    }
+}
+
+impl SourceSnapshot {
+    pub fn mount(&self) -> Result<&'static str> {
+        crate::task_args::parse_handle_digest(&self.digest)
+            .map_err(|error| Error::Argument(error.to_string()))?;
+        Ok("/workspace")
+    }
 }
 
 pub mod fs {
@@ -814,6 +1165,10 @@ pub mod fs {
         })
     }
 
+    pub fn output(relative: impl Into<String>) -> Result<OutputPath, TaskRuntimeError> {
+        output_path(relative)
+    }
+
     pub async fn flush(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
         #[cfg(target_arch = "wasm32")]
         {
@@ -838,6 +1193,10 @@ pub mod fs {
         }
     }
 
+    pub async fn publish(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
+        flush(path).await
+    }
+
     pub async fn materialize(
         artifact: &Artifact,
         relative: impl Into<String>,

crates/clusterflux-sdk/src/sdk_runtime.rs

@@ -106,6 +106,14 @@ pub enum TaskRuntimeError {
         waited: Duration,
     },
     ResultDecode(String),
+    CommandFailed {
+        program: String,
+        status_code: Option<i32>,
+        stdout: String,
+        stderr: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+    },
 }
 
 impl std::fmt::Display for TaskRuntimeError {
@@ -124,6 +132,21 @@ impl std::fmt::Display for TaskRuntimeError {
                 );
             }
             Self::ResultDecode(message) => ("remote result", message),
+            Self::CommandFailed {
+                program,
+                status_code,
+                stdout,
+                stderr,
+                stdout_truncated,
+                stderr_truncated,
+            } => {
+                return write!(
+                    formatter,
+                    "command `{program}` failed with status {status_code:?}; stdout{}: {stdout:?}; stderr{}: {stderr:?}",
+                    if *stdout_truncated { " (truncated)" } else { "" },
+                    if *stderr_truncated { " (truncated)" } else { "" },
+                );
+            }
         };
         write!(formatter, "{kind} error: {message}")
     }

crates/clusterflux-sdk/tests/product_api.rs

@@ -0,0 +1,37 @@
+#[clusterflux::task]
+async fn plus_one(value: i32) -> clusterflux::Result<i32> {
+    Ok(value + 1)
+}
+
+#[test]
+fn unified_spawn_defaults_to_function_identity_and_unwraps_task_result() {
+    futures_executor::block_on(async {
+        let handle = clusterflux::spawn!(plus_one(41))
+            .on(clusterflux::env!("linux"))
+            .await
+            .unwrap();
+        assert_eq!(handle.name(), "plus_one");
+        assert_eq!(handle.env().unwrap().name, "linux");
+        assert_eq!(handle.join().await.unwrap(), 42);
+    });
+}
+
+#[test]
+fn source_mount_and_command_failure_are_public_typed_contracts() {
+    let source = clusterflux::SourceSnapshot {
+        digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+            .to_owned(),
+    };
+    assert_eq!(source.mount().unwrap(), "/workspace");
+
+    let error = clusterflux::Error::CommandFailed {
+        program: "cc".to_owned(),
+        status_code: Some(2),
+        stdout: "out".to_owned(),
+        stderr: "bad input".to_owned(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+    };
+    assert!(error.to_string().contains("command"));
+    assert!(error.to_string().contains("cc"));
+}

docs/artifacts.md

@@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context.
 The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
 transfers. The CLI verifies the final digest. Hosted policy may impose
 per-project, per-tenant, and per-account size, concurrency, and period limits.
-Operators may disable relay traffic as an emergency safety control; one tenant's
+There is no hosted global circuit breaker, and one tenant's period usage does
-period usage does not consume a shared customer byte quota.
+not consume a shared customer byte quota.

docs/debugging.md

@@ -1,5 +1,11 @@
 # Debugging
 
+The DAP runtime observer keeps one coordinator session open and reconnects it
+with a 100 ms to 5 second exponential backoff. Polling is responsive around
+debug actions (100 ms) and backs off to one observation cycle every 5 seconds
+while idle. A cycle makes at most four coordinator requests, so steady-state
+idle traffic is bounded at 0.8 requests per second per debug session.
+
 The VS Code extension uses the Debug Adapter Protocol and the coordinator's
 authoritative process, task, attempt, and Debug Epoch APIs.
 

docs/environments.md

@@ -1,5 +1,9 @@
 # Environments
 
+Clusterflux disables network access while a task executes. Materializing an
+environment for the first time can still fetch declared inputs; subsequent task
+execution uses the materialized environment with networking disabled.
+
 Declare environments in the bundle under:
 
 ~~~text

docs/getting-started.md

@@ -11,6 +11,9 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Install rootless Podman on each Linux node that will execute container-backed
 environments.
 
@@ -52,7 +55,7 @@ same stored identity:
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <hosted-project-id> \
   --node workstation \
@@ -81,6 +84,21 @@ clusterflux bundle inspect --project .
 clusterflux run --project . build
 ~~~
 
+From this checkout, the complete source-to-task-to-artifact path is:
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The workflow snapshots `examples/hello-build`, starts its `compile` task in the
+network-disabled Linux execution environment, and retains the real static executable returned by
+that task.
+
 Choose another entrypoint by replacing `build`. Clusterflux rejects an oversized
 or invalid bundle before it creates the virtual process.
 
@@ -101,12 +119,23 @@ Restart it as a new attempt under the same logical task identity:
 clusterflux task restart <task-id> --process <process-id> --yes
 ~~~
 
+`examples/recovery-build` demonstrates this without a synthetic trap. It starts
+two instances of `build_lane`; one completes while the other runs a command that
+exits with status 23. Edit that command to produce the recovering output, then
+restart the failed task. Its original join resolves from the replacement
+attempt.
+
 ## 6. Debug in VS Code
 
 Open the project in VS Code and start `Clusterflux: Launch Virtual Process`.
 Set a breakpoint on a generated probe location and use Threads, Stack,
 Variables, Continue, Pause, and Restart.
 
+Breakpoints remain unverified until the coordinator installs them. Thread IDs
+are stable for an exact logical task instance across snapshot updates and retry
+attempts. Observer reconnect diagnostics do not manufacture stopped events, and
+Continue succeeds only after the coordinator acknowledges resume.
+
 A fully frozen Debug Epoch gives a consistent all-participant view. If a
 participant cannot freeze within five seconds, the adapter reports a partial
 epoch. You may inspect frozen participants, but values across running and frozen

docs/nodes.md

@@ -23,7 +23,7 @@ key pair.
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <project-id> \
   --node workstation \

examples/hello-build/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "hello-build"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+
+[lib]
+crate-type = ["rlib", "cdylib"]
+
+[dependencies]
+clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" }

examples/hello-build/README.md

@@ -0,0 +1,18 @@
+# Hello build
+
+The primary Clusterflux example snapshots this project, compiles a real static C
+executable in the declared network-disabled Linux execution environment, and publishes the
+executable as a retained artifact.
+
+Run it with:
+
+~~~bash
+clusterflux bundle inspect --project examples/hello-build
+clusterflux run --project examples/hello-build build
+clusterflux artifact list --process <process-id>
+clusterflux artifact download <artifact-id> --to ./hello-clusterflux
+chmod +x ./hello-clusterflux
+./hello-clusterflux
+~~~
+
+The final command prints hello from a real Clusterflux build.

examples/hello-build/src/lib.rs

@@ -0,0 +1,30 @@
+use clusterflux::prelude::*;
+
+#[clusterflux::task(capabilities = "command")]
+pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
+    let executable = fs::output("hello-clusterflux")?;
+    Command::new("cc")
+        .args([
+            "-Os",
+            "-static",
+            "-s",
+            "fixture/hello-clusterflux.c",
+            "-o",
+            executable.as_str(),
+        ])
+        .cwd(source.mount()?)
+        .env("SOURCE_DATE_EPOCH", "0")
+        .network_disabled()
+        .run()
+        .await?;
+    fs::publish(&executable).await
+}
+
+#[clusterflux::main]
+pub async fn build() -> Result<Artifact> {
+    let source = source::current_project().snapshot().await?;
+    let compile = clusterflux::spawn!(compile(source))
+        .on(clusterflux::env!("linux"))
+        .await?;
+    compile.join().await
+}

examples/launch-build-demo/README.md

@@ -1,14 +0,0 @@
-# Launch Build Demo
-
-This build workflow is expressed as Rust source code. It uses `env!("linux")`,
-recognizes `env!("windows")` when a Windows development node is attached,
-spawns debugger-visible virtual tasks, and returns portable artifact and source
-handles instead of moving large bytes through task arguments.
-
-The Linux environment is defined by `envs/linux/Containerfile`. The Windows environment is a user-attached development contract and does not claim secure managed Windows sandboxing.
-
-Inspect the bundle metadata:
-
-```bash
-clusterflux bundle inspect --project examples/launch-build-demo
-```

examples/launch-build-demo/envs/windows/Dockerfile

@@ -1,2 +0,0 @@
-# User-attached Windows development execution contract.
-# This is not a managed untrusted Windows sandbox.

examples/launch-build-demo/src/bin/sdk-product-runtime.rs

@@ -1,33 +0,0 @@
-use std::error::Error;
-
-use futures_executor::block_on;
-
-fn main() -> Result<(), Box<dyn Error>> {
-    let handle = block_on(
-        clusterflux::spawn::task_with_arg(41_i32, |_| -> i32 {
-            panic!("product-mode spawn invoked its local Rust closure")
-        })
-        .task_id("task_add_one")
-        .name("SDK product Wasmtime task")
-        .env(clusterflux::env!("linux"))
-        .start(),
-    )?;
-    let task_spec = handle
-        .task_spec()
-        .cloned()
-        .ok_or("CLUSTERFLUX_SDK_COORDINATOR did not enable product runtime")?;
-    let virtual_thread_id = handle.virtual_thread_id();
-    let result = block_on(handle.join())?;
-
-    println!(
-        "{}",
-        serde_json::to_string(&serde_json::json!({
-            "kind": "clusterflux-sdk-product-runtime",
-            "virtual_thread_id": virtual_thread_id,
-            "task_spec": task_spec,
-            "remote_result": result,
-            "local_function_invoked_by_spawn": false,
-        }))?
-    );
-    Ok(())
-}

examples/launch-build-demo/src/build.rs

@@ -1,498 +0,0 @@
-use clusterflux::{Artifact, EnvRef, SourceSnapshot};
-use serde::{Deserialize, Serialize};
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)]
-pub struct BuildReport {
-    pub linux_thread: u64,
-    pub linux_parallel_thread: u64,
-    pub package_thread: u64,
-    pub linux_artifact: Artifact,
-    pub package_artifact: Artifact,
-    pub source: SourceSnapshot,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)]
-pub struct PackageInput {
-    pub release_name: String,
-    pub source: SourceSnapshot,
-    pub executable: Option<Artifact>,
-    pub inputs: Vec<Artifact>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)]
-pub struct IdentityProbeInput {
-    pub source: SourceSnapshot,
-    pub label: String,
-    pub delay_seconds: u64,
-}
-
-pub fn linux_env() -> EnvRef {
-    clusterflux::env!("linux")
-}
-
-pub fn windows_env() -> EnvRef {
-    clusterflux::env!("windows")
-}
-
-#[clusterflux::task(capabilities = "source_filesystem")]
-pub async fn prepare_source() -> SourceSnapshot {
-    #[cfg(target_arch = "wasm32")]
-    {
-        return clusterflux::source::snapshot()
-            .await
-            .expect("the source task should snapshot the node checkout it was placed on");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    SourceSnapshot {
-        digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-            .to_owned(),
-    }
-}
-
-#[clusterflux::task(capabilities = "command")]
-pub async fn compile_linux(source: SourceSnapshot) -> Artifact {
-    let _ = &source;
-    #[cfg(target_arch = "wasm32")]
-    {
-        let executable = clusterflux::fs::output_path("hello-clusterflux")
-            .expect("the executable output path should be task-local");
-        let output = clusterflux::command::Command::new("cc")
-            .args([
-                "-Os",
-                "-static",
-                "-s",
-                "fixture/hello-clusterflux.c",
-                "-o",
-                executable.as_str(),
-            ])
-            .current_dir("/workspace")
-            .env("SOURCE_DATE_EPOCH", "0")
-            .timeout(std::time::Duration::from_secs(180))
-            .network_disabled()
-            .output()
-            .await
-            .expect("the declared Linux environment should provide a real C compiler");
-        assert_eq!(output.status_code, Some(0));
-        return clusterflux::fs::flush(&executable)
-            .await
-            .expect("the command-created executable should flush to node artifact storage");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    Artifact {
-        id: "hello-clusterflux-native-test".to_owned(),
-        digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-            .to_owned(),
-        size_bytes: 0,
-    }
-}
-
-#[clusterflux::task]
-#[unsafe(no_mangle)]
-pub extern "C" fn task_add_one(input: i32) -> i32 {
-    input + 1
-}
-
-#[clusterflux::task]
-#[unsafe(no_mangle)]
-pub extern "C" fn task_trap(_input: i32) -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    core::arch::wasm32::unreachable();
-    #[cfg(not(target_arch = "wasm32"))]
-    panic!("intentional task trap")
-}
-
-#[clusterflux::task(capabilities = "command")]
-pub async fn package_release(input: PackageInput) -> Artifact {
-    #[cfg(target_arch = "wasm32")]
-    {
-        assert_eq!(input.release_name, "release.tar");
-        let executable = input
-            .executable
-            .as_ref()
-            .expect("the package input should contain the compiler artifact");
-        assert!(input.inputs.iter().any(|artifact| artifact == executable));
-        let executable = clusterflux::fs::materialize(&executable, "package/hello-clusterflux")
-            .await
-            .expect("the retaining node should materialize the compiler artifact locally");
-        let release = clusterflux::fs::output_path("release.tar")
-            .expect("the release output path should be task-local");
-        let output = clusterflux::command::Command::new("tar")
-            .args([
-                "--sort=name",
-                "--mtime=@0",
-                "--owner=0",
-                "--group=0",
-                "--numeric-owner",
-                "--mode=0755",
-                "-cf",
-                release.as_str(),
-                "-C",
-                "/clusterflux/output/package",
-                "hello-clusterflux",
-            ])
-            .current_dir("/workspace")
-            .env("SOURCE_DATE_EPOCH", "0")
-            .timeout(std::time::Duration::from_secs(180))
-            .network_disabled()
-            .output()
-            .await
-            .expect("the declared Linux environment should package the materialized executable");
-        assert_eq!(output.status_code, Some(0));
-        assert_eq!(
-            executable.as_str(),
-            "/clusterflux/output/package/hello-clusterflux"
-        );
-        return clusterflux::fs::flush(&release)
-            .await
-            .expect("the command-created release archive should flush as the final artifact");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    {
-        let _ = input;
-        Artifact {
-            id: "release-native-test".to_owned(),
-            digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-                .to_owned(),
-            size_bytes: 0,
-        }
-    }
-}
-
-#[clusterflux::task(capabilities = "command")]
-pub async fn abort_probe() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let output = clusterflux::command::Command::new("sh")
-            .args(["-c", "sleep 30"])
-            .output()
-            .await
-            .expect("abort probe command should either finish or be stopped by process abort");
-        return output.status_code.unwrap_or(-1);
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    0
-}
-
-#[clusterflux::task(capabilities = "command")]
-pub async fn long_join_probe(source: SourceSnapshot) -> i32 {
-    let _ = &source;
-    #[cfg(target_arch = "wasm32")]
-    {
-        let output = clusterflux::command::Command::new("sh")
-            .args(["-c", "sleep 125"])
-            .timeout(std::time::Duration::from_secs(180))
-            .network_disabled()
-            .output()
-            .await
-            .expect("the controlled long-running command should complete normally");
-        return output.status_code.unwrap_or(-1);
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    0
-}
-
-#[clusterflux::task(capabilities = "command")]
-pub async fn identity_probe(input: IdentityProbeInput) -> String {
-    let _ = &input.source;
-    #[cfg(target_arch = "wasm32")]
-    {
-        let script = format!(
-            "sleep {}; printf '%s' '{}'",
-            input.delay_seconds, input.label
-        );
-        let output = clusterflux::command::Command::new("sh")
-            .args(["-c", script.as_str()])
-            .timeout(std::time::Duration::from_secs(60))
-            .network_disabled()
-            .output()
-            .await
-            .expect("identity probe command should complete");
-        assert_eq!(output.status_code, Some(0));
-        return input.label;
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    input.label
-}
-
-#[clusterflux::task]
-pub fn cooperative_cancellation_probe() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    loop {
-        if clusterflux::process::cancellation_requested()
-            .expect("task control should remain available while the Wasm task is running")
-        {
-            // Cancellation is a request observed and handled by task code. Returning
-            // normally is intentionally distinct from the runtime's forced abort path.
-            return 17;
-        }
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    17
-}
-
-#[clusterflux::task]
-pub fn debug_child_probe() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    loop {
-        if clusterflux::process::cancellation_requested()
-            .expect("child debug participant should retain task control")
-        {
-            return 23;
-        }
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    23
-}
-
-#[clusterflux::task]
-pub async fn debug_parent_probe() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let child = clusterflux::spawn::task(debug_child_probe)
-            .name("debug child participant")
-            .start()
-            .await
-            .expect("parent debug participant should spawn its child");
-        return child
-            .join()
-            .await
-            .expect("parent debug participant should join its child");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    23
-}
-
-#[clusterflux::main]
-pub async fn build_main() -> BuildReport {
-    run_build_workflow().await
-}
-
-#[clusterflux::main(name = "fail")]
-pub async fn fail_main() -> Result<i32, String> {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let child = clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value))
-            .task_id("task_trap")
-            .name("intentional failing child")
-            .start()
-            .await
-            .map_err(|error| format!("failing entrypoint could not launch its child: {error}"))?;
-        return child
-            .join()
-            .await
-            .map_err(|error| format!("intentional child failure: {error}"));
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    Err("intentional child failure".to_owned())
-}
-
-#[clusterflux::main(name = "restart")]
-pub async fn restart_main() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    {
-        return clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value))
-            .task_id("task_trap")
-            .name("edited restart probe")
-            .failure_policy(clusterflux::spawn::TaskFailurePolicy::AwaitOperator)
-            .start()
-            .await
-            .expect("restart probe should launch its child")
-            .join()
-            .await
-            .expect("restart probe child should complete");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    task_add_one(41)
-}
-
-#[clusterflux::main(name = "park-wake")]
-pub async fn park_wake_main() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let mut value = 0_i32;
-        for _ in 0..16 {
-            let next = clusterflux::spawn::task_with_arg(value, |input| task_add_one(input))
-                .task_id("task_add_one")
-                .name("park/wake fuel refill probe")
-                .start()
-                .await
-                .expect("park/wake probe should launch")
-                .join()
-                .await
-                .expect("park/wake probe should complete");
-            value = i32::try_from(next).expect("park/wake result should remain in i32 range");
-        }
-        return value;
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    16
-}
-
-#[clusterflux::main(name = "long-join")]
-pub async fn long_join_main() -> i32 {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let source = clusterflux::spawn::async_task(prepare_source)
-            .name("prepare source for controlled long-running task")
-            .start()
-            .await
-            .expect("long join source preparation should launch")
-            .join()
-            .await
-            .expect("long join source preparation should complete");
-        return clusterflux::spawn::async_task_with_arg(source, long_join_probe)
-            .name("controlled task longer than two minutes")
-            .env(linux_env())
-            .start()
-            .await
-            .expect("long join probe should launch")
-            .join()
-            .await
-            .expect("long join probe should remain joinable beyond two minutes");
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    0
-}
-
-#[clusterflux::main(name = "identity")]
-pub async fn identity_main() -> Vec<String> {
-    #[cfg(target_arch = "wasm32")]
-    {
-        let source = clusterflux::spawn::async_task(prepare_source)
-            .name("prepare source for identity probes")
-            .start()
-            .await
-            .expect("identity source preparation should launch")
-            .join()
-            .await
-            .expect("identity source preparation should complete");
-        let slow = clusterflux::spawn::async_task_with_arg(
-            IdentityProbeInput {
-                source: source.clone(),
-                label: "slow-first".to_owned(),
-                delay_seconds: 30,
-            },
-            identity_probe,
-        )
-        .name("identity probe slow")
-        .env(linux_env())
-        .start()
-        .await
-        .expect("slow identity probe should launch");
-        let fast = clusterflux::spawn::async_task_with_arg(
-            IdentityProbeInput {
-                source,
-                label: "fast-second".to_owned(),
-                delay_seconds: 20,
-            },
-            identity_probe,
-        )
-        .name("identity probe fast")
-        .env(linux_env())
-        .start()
-        .await
-        .expect("fast identity probe should launch");
-        let fast_result = fast
-            .join()
-            .await
-            .expect("fast identity probe should complete first");
-        let slow_result = slow
-            .join()
-            .await
-            .expect("slow identity probe should remain independently joinable");
-        return vec![fast_result, slow_result];
-    }
-    #[cfg(not(target_arch = "wasm32"))]
-    vec!["fast-second".to_owned(), "slow-first".to_owned()]
-}
-
-pub async fn run_build_workflow() -> BuildReport {
-    let source = clusterflux::spawn::async_task(prepare_source)
-        .name("prepare source snapshot")
-        .start()
-        .await
-        .unwrap()
-        .join()
-        .await
-        .unwrap();
-    let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
-        .name("compile linux")
-        .env(linux_env())
-        .start()
-        .await
-        .unwrap();
-    let linux_parallel = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
-        .name("compile linux in parallel")
-        .env(linux_env())
-        .start()
-        .await
-        .unwrap();
-    let linux_thread = linux.virtual_thread_id();
-    let linux_parallel_thread = linux_parallel.virtual_thread_id();
-    let linux_artifact = linux.join().await.unwrap();
-    let linux_parallel_artifact = linux_parallel.join().await.unwrap();
-    assert_eq!(linux_artifact.digest, linux_parallel_artifact.digest);
-    let package = clusterflux::spawn::async_task_with_arg(
-        PackageInput {
-            release_name: "release.tar".to_owned(),
-            source: source.clone(),
-            executable: Some(linux_artifact.clone()),
-            inputs: vec![linux_artifact.clone()],
-        },
-        package_release,
-    )
-    .name("package artifacts")
-    .env(linux_env())
-    .start()
-    .await
-    .unwrap();
-
-    let package_thread = package.virtual_thread_id();
-    let package_artifact = package.join().await.unwrap();
-
-    BuildReport {
-        linux_thread,
-        linux_parallel_thread,
-        package_thread,
-        linux_artifact,
-        package_artifact,
-        source,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use futures_executor::block_on;
-
-    use super::*;
-
-    #[test]
-    fn flagship_workflow_is_rust_source_with_spawned_virtual_tasks() {
-        let main_report = block_on(build_main());
-        assert_eq!(
-            main_report.linux_artifact.id,
-            "hello-clusterflux-native-test"
-        );
-        assert_eq!(task_add_one(41), 42);
-        assert_eq!(block_on(restart_main()), 42);
-        assert_eq!(block_on(park_wake_main()), 16);
-        assert_eq!(block_on(long_join_main()), 0);
-        assert_eq!(
-            block_on(identity_main()),
-            vec!["fast-second".to_owned(), "slow-first".to_owned()]
-        );
-        assert_eq!(linux_env().name, "linux");
-        assert_eq!(windows_env().name, "windows");
-
-        let report = block_on(run_build_workflow());
-
-        assert_ne!(report.linux_thread, report.package_thread);
-        assert_ne!(report.linux_thread, report.linux_parallel_thread);
-        assert_eq!(report.linux_artifact.id, "hello-clusterflux-native-test");
-        assert_eq!(report.package_artifact.id, "release-native-test");
-        assert_eq!(
-            report.source.digest,
-            "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-        );
-    }
-}

examples/recovery-build/Cargo.toml

@@ -1,18 +1,14 @@
 [package]
-name = "launch-build-demo"
+name = "recovery-build"
 version = "0.1.0"
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
+publish = false
 
 [lib]
-path = "src/build.rs"
 crate-type = ["rlib", "cdylib"]
 
 [dependencies]
 clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" }
-futures-executor.workspace = true
 serde.workspace = true
-serde_json.workspace = true
-
-[dev-dependencies]

examples/recovery-build/README.md

@@ -0,0 +1,7 @@
+# Recovery build
+
+This advanced example starts two instances of the same task definition. The
+stable lane completes while the recovering lane executes a real failing command
+under AwaitOperator. Edit the failing command to write
+/clusterflux/output/recovering.txt, then restart that task from the debugger or
+CLI. The original main join completes from the replacement attempt.