.gitignore

@@ -1,8 +1,5 @@
 /target/
+/web/target/
 /.clusterflux/
 **/.clusterflux/
 /vscode-extension/node_modules/
-/private/*/Cargo.lock
-!/private/hosted-policy/Cargo.lock
-/private/*/target/
-/scripts/containers-home/

CLUSTERFLUX_PUBLIC_TREE.json

@@ -1,22 +0,0 @@
-{
-  "kind": "clusterflux-filtered-public-tree",
-  "source_commit": "fcace1d13a4b4100c5fe95267d06de68ee6675af",
-  "release_name": "release-fcace1d13a4b",
-  "filtered_out": [
-    "private/**",
-    "internal/**",
-    "experiments/**",
-    ".git",
-    "target",
-    "git-ignored source paths",
-    "root/*.md except README.md",
-    "**/.clusterflux/**",
-    ".forgejo/**"
-  ],
-  "public_export": {
-    "host_neutral": true,
-    "include_forgejo_workflows": false
-  },
-  "forgejo_host": "git.michelpaulissen.com",
-  "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
-}

Cargo.lock

@@ -290,6 +290,20 @@ dependencies = [
  "wasmparser 0.245.1",
 ]
 
+[[package]]
+name = "clusterflux-client"
+version = "0.1.0"
+dependencies = [
+ "base64",
+ "clusterflux-control",
+ "clusterflux-coordinator",
+ "clusterflux-core",
+ "serde",
+ "serde_json",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
 [[package]]
 name = "clusterflux-control"
 version = "0.1.0"
@@ -1712,16 +1726,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "runtime-conformance"
-version = "0.1.0"
-dependencies = [
- "clusterflux-sdk",
- "futures-executor",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 resolver = "2"
+exclude = ["web"]
 members = [
   "crates/clusterflux-cli",
+  "crates/clusterflux-client",
   "crates/clusterflux-control",
   "crates/clusterflux-coordinator",
   "crates/clusterflux-core",
@@ -12,13 +14,12 @@ members = [
   "crates/clusterflux-wasm-runtime",
     "examples/hello-build",
     "examples/recovery-build",
-    "tests/fixtures/runtime-conformance",
 ]
 
 [workspace.package]
 edition = "2021"
 license = "Apache-2.0 OR MIT"
-repository = "https://git.michelpaulissen.com/michel/clusterflux-public"
+repository = "https://clusterflux.lesstuff.com"
 
 [workspace.dependencies]
 anyhow = "1.0"

README.md

@@ -1,9 +1,57 @@
 # Clusterflux
 
-Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
-coordinator hosts the async main, while attached nodes execute Wasm tasks,
-rootless containers, and native commands. Tasks exchange canonical values and
-portable typed handles instead of sharing host memory.
+Clusterflux runs a Rust-defined workflow as one distributed virtual process. The
+async main runs serverless, provisioning nodes to run tasks through rootless
+containers. Tasks on nodes exchange data simply and efficiently.
+
+The user experience is built to be as much as possible like building a regular
+program. The processes are debuggable as normal through a debugger adapter.
+
+The primary use case is to consolidate build processes into a single streamlined
+developer experience. An example program can be seen below:
+
+~~~rust
+use clusterflux::prelude::*;
+
+#[clusterflux::task(capabilities = "command")]
+pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
+    let executable = fs::output("hello-clusterflux")?;
+    Command::new("cc")
+        .args([
+            "-Os",
+            "-static",
+            "-s",
+            "fixture/hello-clusterflux.c",
+            "-o",
+            executable.as_str(),
+        ])
+        .cwd(source.mount()?)
+        .env("SOURCE_DATE_EPOCH", "0")
+        .network_disabled()
+        .run()
+        .await?;
+    fs::publish(&executable).await
+}
+
+#[clusterflux::main]
+pub async fn build() -> Result<Artifact> {
+    let source = source::current_project().snapshot().await?;
+    let compile = clusterflux::spawn!(compile(source))
+        .on(clusterflux::env!("linux"))
+        .await?;
+    compile.join().await
+}
+~~~
+
+After setup, this build pipeline could be deployed as easily as launching it
+through your IDE. This repository includes a VS Code extension to make
+development as straightforward as possible. A full collection of CLI tools is
+included for advanced usage.
+
+Clusterflux is explicitly local-first. It is trivial to provision existing
+hardware as resources. Bulk data will typically not leave the local network,
+allowing maximum throughput. The same capability, however, also makes it
+possible to leverage cloud resources easily.
 
 Start with [Getting started](docs/getting-started.md). It takes you through
 authentication, project setup, node enrollment, a run, debugging, task restart,
@@ -49,6 +97,9 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Rootless Podman is required on Linux nodes that build or run a declared
 Containerfile environment. Install VS Code when you want the graphical debug
 workflow.

SECURITY.md

@@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained.
 
 ## Report a vulnerability
 
-Email security@michelpaulissen.com with a concise description, affected version
+Email security@clusterflux.lesstuff.com with a concise description, affected version
 or source revision, reproduction steps, and impact. Do not open a public issue
 for an unpatched vulnerability or include credentials, session tokens, private
 keys, provider tokens, customer data, or operator secrets in a public report.

crates/clusterflux-cli/src/admin.rs

@@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
                 .unwrap_or(json!(false)),
             "response": response,
             "safe_default": "read_only",
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
         "command": "admin status",
         "mode": "self_hosted_local",
         "safe_default": "read_only",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
         "project": project.clone(),
         "user": user,
         "coordinator": scope.coordinator,
-        "private_website_required": false,
+        "external_website_required": false,
         "self_hosted_cli_only": true,
         "project_config_written": project_init
             .get("project_config_written")
@@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
             {
                 "step": "start_self_hosted_coordinator",
                 "command": "clusterflux-coordinator --listen 127.0.0.1:0",
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_or_link_project",
                 "command": "clusterflux project init --yes",
                 "completed": true,
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "create_node_enrollment_grant",
@@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "attach_worker_node",
@@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "run_process",
@@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
                     tenant, project
                 ),
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "inspect_status_logs_artifacts",
@@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
                     "clusterflux artifact list",
                     "clusterflux quota status",
                 ],
-                "private_website_required": false,
+                "external_website_required": false,
             },
             {
                 "step": "revoke_access",
                 "command": "clusterflux admin revoke-node --node <node-id> --yes",
-                "private_website_required": false,
+                "external_website_required": false,
             }
         ],
     }))
@@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
             "actor_tenant": actor_tenant,
             "actor_user": actor_user,
             "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
         "status": "requires_coordinator",
         "requires_confirmation": !args.yes,
         "tenant": tenant,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 

crates/clusterflux-cli/src/artifact.rs

@@ -12,6 +12,7 @@ use crate::client::{
 };
 use crate::config::StoredCliSession;
 use crate::errors::cli_error_summary_for_category;
+use crate::process::hydrate_process_scope;
 use crate::process_events::{
     artifact_download_grant_disclosures, artifact_download_session_summary,
     artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
@@ -27,9 +28,10 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
 }
 
 pub(crate) fn artifact_list_report_with_session(
-    args: ArtifactListArgs,
+    mut args: ArtifactListArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,
@@ -53,9 +55,10 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Val
 }
 
 pub(crate) fn artifact_download_report_with_session(
-    args: ArtifactDownloadArgs,
+    mut args: ArtifactDownloadArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(
@@ -143,9 +146,10 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value>
 }
 
 pub(crate) fn artifact_export_report_with_session(
-    args: ArtifactExportArgs,
+    mut args: ArtifactExportArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     if let Some(coordinator) = &args.scope.coordinator {
         let mut session = JsonLineSession::connect(coordinator)?;
         let response = session.request(authenticated_or_local_trusted_request(

crates/clusterflux-cli/src/auth.rs

@@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<V
                 "suspension_known": false,
                 "account_state_known": false,
                 "account_status": "unknown",
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
             })
         });
@@ -263,7 +263,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -296,7 +296,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -316,7 +316,7 @@ fn coordinator_auth_status_summary(
                 "account_status": "unknown",
                 "suspension_known": false,
                 "account_state_known": false,
-                "private_moderation_details_exposed": false,
+                "sensitive_moderation_details_exposed": false,
                 "signup_failure_details_exposed": false,
                 "machine_error": cli_error_summary(&message),
                 "error": message,
@@ -338,7 +338,7 @@ fn coordinator_auth_status_summary(
             "account_status": "unknown",
             "suspension_known": false,
             "account_state_known": false,
-            "private_moderation_details_exposed": false,
+            "sensitive_moderation_details_exposed": false,
             "signup_failure_details_exposed": false,
             "machine_error": cli_error_summary(message),
             "coordinator_response_type": "error",
@@ -403,7 +403,7 @@ fn coordinator_auth_status_summary(
         "manual_review": manual_review,
         "sanitized_reason": sanitized_reason,
         "next_actions": next_actions,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
         "coordinator_session_requests": coordinator_session_requests,

crates/clusterflux-cli/src/auth_scope.rs

@@ -60,7 +60,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
         "account_status": "unknown",
         "suspension_known": false,
         "account_state_known": false,
-        "private_moderation_details_exposed": false,
+        "sensitive_moderation_details_exposed": false,
         "signup_failure_details_exposed": false,
         "next_actions": ["clusterflux login --browser"],
     })

crates/clusterflux-cli/src/config.rs

@@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::CliScopeArgs;
 
-pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
-    "https://clusterflux.michelpaulissen.com";
+pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com";
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub(crate) struct ProjectConfig {

crates/clusterflux-cli/src/debug.rs

@@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
         "command": "dap",
         "adapter": dap_binary_path()?.display().to_string(),
         "args": args.args,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
                 .cloned()
                 .unwrap_or_else(|| json!(0)),
             "debug_reads_quota_limited": true,
-            "private_website_required": false,
+            "external_website_required": false,
             "coordinator_session_requests": session.requests(),
         }));
     }
@@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
         "dap": dap,
         "authorized": "unknown_without_coordinator",
         "debug_reads_quota_limited": "unknown_without_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }

crates/clusterflux-cli/src/errors.rs

@@ -38,7 +38,10 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s
             );
             object.insert("community_tier_language".to_owned(), json!(true));
             object.insert("community_tier_label".to_owned(), json!("community tier"));
-            object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
+            object.insert(
+                "sensitive_abuse_heuristics_exposed".to_owned(),
+                json!(false),
+            );
         }
     }
     summary

crates/clusterflux-cli/src/logs.rs

@@ -3,6 +3,7 @@ use serde_json::{json, Value};
 
 use crate::client::list_task_events_if_available_with_session;
 use crate::config::StoredCliSession;
+use crate::process::hydrate_process_scope;
 use crate::process_events::log_entries;
 use crate::LogsArgs;
 
@@ -12,9 +13,10 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
 }
 
 pub(crate) fn logs_report_with_session(
-    args: LogsArgs,
+    mut args: LogsArgs,
     stored_session: Option<&StoredCliSession>,
 ) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
     let events = list_task_events_if_available_with_session(
         args.scope.coordinator.as_deref(),
         &args.scope,

crates/clusterflux-cli/src/node.rs

@@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
             "tenant": tenant,
             "project": project,
             "user": user,
-            "private_website_required": false,
+            "external_website_required": false,
             "enrollment_grant": enrollment_grant,
             "response": response,
             "coordinator_session_requests": session.requests(),
@@ -161,7 +161,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
     Ok(json!({
         "command": "node enroll",
         "status": "requires_coordinator",
-        "private_website_required": false,
+        "external_website_required": false,
         "requested_ttl_seconds": args.ttl_seconds,
         "enrollment_grant": null,
         "reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
@@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
     };
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &tenant,
+        "project": &project,
         "node": &plan.node,
     });
     let heartbeat_signature = sign_node_request(

crates/clusterflux-cli/src/output.rs

@@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         }
         push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
         if let Some(exposed) = account
-            .get("private_moderation_details_exposed")
+            .get("sensitive_moderation_details_exposed")
             .and_then(Value::as_bool)
         {
-            lines.push(format!("private moderation details exposed: {exposed}"));
+            lines.push(format!("sensitive moderation details exposed: {exposed}"));
         }
     }
     if let Some(coordinator_selection) = value.get("coordinator") {
@@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String {
         ));
     }
     if let Some(flag) = value
-        .get("private_website_required")
+        .get("external_website_required")
         .and_then(Value::as_bool)
     {
-        lines.push(format!("private website required: {flag}"));
+        lines.push(format!("external website required: {flag}"));
     }
     if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
         let actions = next_actions

crates/clusterflux-cli/src/process.rs

@@ -15,7 +15,10 @@ use crate::{
     ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
 };
 
-fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
+pub(crate) fn hydrate_process_scope(
+    scope: &mut CliScopeArgs,
+    stored_session: Option<&StoredCliSession>,
+) {
     if scope.coordinator.is_none() {
         scope.coordinator = stored_session
             .filter(|session| session.session_secret.is_some())

crates/clusterflux-cli/src/process_events.rs

@@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
         "cross_tenant_reuse_allowed": false,
         "unauthorized_project_reuse_allowed": false,
         "default_durable_store_assumed": false,
-        "private_website_required": false,
+        "external_website_required": false,
     }])
 }
 
@@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<
         "current_usage": current_usage,
         "limits": quota_limits_value(),
         "next_blocked_action": next_blocked_action,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
     })
 }
 

crates/clusterflux-cli/src/project.rs

@@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
     Ok(json!({
         "command": "project init",
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config_written": true,
         "project_config_write_after_coordinator_acceptance": coordinator.is_some(),
         "coordinator_create_before_local_write": coordinator.is_some(),
@@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "config_format": "clusterflux_project_config_v1",
             "links_current_directory": true,
             "writes_current_directory_only": true,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "safe_defaults": {
             "tenant": config.tenant.clone(),
@@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
             "default_project_id_used": args.new_project == "project",
             "default_project_name_used": args.name == "Clusterflux Project",
             "browser_interaction_required": false,
-            "private_website_required": false,
+            "external_website_required": false,
         },
         "project_config": config,
         "config_file": project_config_file(&cwd),
@@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
             "user": user,
             "projects": projects,
             "project_count": project_count,
-            "private_website_required": false,
+            "external_website_required": false,
             "response": response,
             "coordinator_session_requests": session.requests(),
         }));
@@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
         "source": "local_project_config",
         "projects": projects,
         "project_count": project_count,
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 
@@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re
         "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
         "selected_project": selected_project,
         "project_config_written": true,
-        "private_website_required": false,
+        "external_website_required": false,
         "project_config": config,
         "coordinator_response": coordinator_response,
     }))

crates/clusterflux-cli/src/quota.rs

@@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "user": effective_scope.user,
         "coordinator": coordinator,
         "project_config": config,
-        "policy_surface": "generic public quota categories; hosted tuning remains private policy",
+        "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined",
         "limits": limits,
         "window_seconds": window_seconds,
         "current_usage": current_usage,
@@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
         "next_blocked_action": quota_next_blocked_action(&current_usage),
         "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
         "quota_tier": quota_tier,
-        "private_abuse_heuristics_exposed": false,
+        "sensitive_abuse_heuristics_exposed": false,
         "quota_response": quota_status,
     }))
 }

crates/clusterflux-cli/src/run.rs

@@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
         "safe_failure": true,
         "message": message,
         "next_actions": next_actions,
-        "private_website_required": false,
+        "external_website_required": false,
         "machine_error": machine_error,
     })
 }
@@ -329,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
         "task_launch": launch_task_response,
         "coordinator_response": response,
         "coordinator_session_requests": session.requests(),
-        "private_website_required": false,
+        "external_website_required": false,
     }))
 }
 

crates/clusterflux-cli/src/tests.rs

@@ -124,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
                 assert_eq!(summary["resource_category"], "api_calls");
                 assert_eq!(summary["community_tier_language"], true);
                 assert_eq!(summary["community_tier_label"], "community tier");
-                assert_eq!(summary["private_abuse_heuristics_exposed"], false);
+                assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false);
                 let rendered = human_report(&json!({
                     "command": "run",
                     "machine_error": summary,
@@ -207,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
     assert_eq!(report["status"], "authentication_required");
     assert_eq!(report["non_interactive"], true);
     assert_eq!(report["browser_opened"], false);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["machine_error"]["category"], "authentication");
     assert_eq!(report["machine_error"]["stable_exit_code"], 20);
     assert_eq!(report["machine_error"]["browser_opened"], false);
@@ -827,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() {
     assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
     assert_eq!(rejected["machine_error"]["community_tier_language"], true);
     assert_eq!(
-        rejected["machine_error"]["private_abuse_heuristics_exposed"],
+        rejected["machine_error"]["sensitive_abuse_heuristics_exposed"],
         false
     );
     assert!(rejected["machine_error"]["next_actions"]
@@ -1665,12 +1665,11 @@ fn node_attach_refuses_a_symlink_credential_target() {
 fn hosted_coordinator_remains_a_real_https_control_endpoint() {
     assert_eq!(
         control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
+        "https://clusterflux.lesstuff.com/api/v1/control"
     );
     assert_eq!(
-        control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
-            .unwrap(),
-        "https://clusterflux.michelpaulissen.com/api/v1/control"
+        control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
+        "https://clusterflux.lesstuff.com/api/v1/control"
     );
     assert!(control_endpoint_identity("http://operator.example.test").is_err());
     assert_eq!(
@@ -1788,7 +1787,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         assert!(!line.contains(r#""actor_user":"user-session""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1851,7 +1850,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
         "active"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
 }
@@ -1933,7 +1932,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
 }
 
 #[test]
-fn auth_status_queries_coordinator_account_state_without_private_moderation_details() {
+fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() {
     let listener = TcpListener::bind("127.0.0.1:0").unwrap();
     let addr = listener.local_addr().unwrap().to_string();
     let server = std::thread::spawn(move || {
@@ -1947,7 +1946,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         assert!(line.contains(r#""actor_user":"user-live""#));
         stream
                 .write_all(
-                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#,
+                    br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#,
                 )
                 .unwrap();
         stream.write_all(b"\n").unwrap();
@@ -1987,7 +1986,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
         "account or tenant is suspended by hosted policy"
     );
     assert_eq!(
-        report["coordinator_account_status"]["private_moderation_details_exposed"],
+        report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
         false
     );
     assert_eq!(
@@ -2005,13 +2004,13 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
     let serialized = serde_json::to_string(&report).unwrap();
     assert!(!serialized.contains("abuse_score"));
     assert!(!serialized.contains("moderation_notes"));
-    assert!(!serialized.contains("private moderation note"));
+    assert!(!serialized.contains("sensitive moderation note"));
 
     let rendered = human_report(&report);
     assert!(rendered.contains("account status: suspended"));
     assert!(rendered.contains("account suspended: true"));
-    assert!(rendered.contains("private moderation details exposed: false"));
-    assert!(!rendered.contains("private moderation note"));
+    assert!(rendered.contains("sensitive moderation details exposed: false"));
+    assert!(!rendered.contains("sensitive moderation note"));
 }
 
 #[test]
@@ -2062,11 +2061,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
                     "manual_review": manual_review,
                     "sanitized_reason": reason,
                     "next_actions": ["contact the hosted operator"],
-                    "private_moderation_details_exposed": false,
+                    "sensitive_moderation_details_exposed": false,
                     "signup_failure_details_exposed": false,
                     "abuse_score": 99,
-                    "moderation_notes": "private moderation note",
-                    "signup_policy_trace": "private signup trace",
+                    "moderation_notes": "sensitive moderation note",
+                    "signup_policy_trace": "sensitive signup trace",
                 })
             )
             .unwrap();
@@ -2105,7 +2104,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
             true
         );
         assert_eq!(
-            report["coordinator_account_status"]["private_moderation_details_exposed"],
+            report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
             false
         );
         assert_eq!(
@@ -2116,11 +2115,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
         assert!(!serialized.contains("abuse_score"));
         assert!(!serialized.contains("moderation_notes"));
         assert!(!serialized.contains("signup_policy_trace"));
-        assert!(!serialized.contains("private moderation note"));
+        assert!(!serialized.contains("sensitive moderation note"));
         let rendered = human_report(&report);
         assert!(rendered.contains(&format!("account status: {status}")));
         assert!(rendered.contains(rendered_marker));
-        assert!(!rendered.contains("private moderation note"));
+        assert!(!rendered.contains("sensitive moderation note"));
     }
     server.join().unwrap();
 }
@@ -2235,11 +2234,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
 
     assert_eq!(report["command"], "admin bootstrap");
     assert_eq!(report["mode"], "self_hosted_local");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["self_hosted_cli_only"], true);
     assert_eq!(report["project_config_written"], true);
     assert_eq!(report["project_init"]["command"], "project init");
-    assert_eq!(report["project_init"]["private_website_required"], false);
+    assert_eq!(report["project_init"]["external_website_required"], false);
     assert_eq!(
         report["project_init"]["project_config"]["project"],
         "self-hosted"
@@ -2265,7 +2264,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
     }
     assert!(steps.iter().all(|step| {
         !step
-            .get("private_website_required")
+            .get("external_website_required")
             .and_then(Value::as_bool)
             .unwrap_or(false)
     }));
@@ -2893,13 +2892,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
 
     assert_eq!(status["command"], "admin status");
     assert_eq!(status["safe_default"], "read_only");
-    assert_eq!(status["private_website_required"], false);
+    assert_eq!(status["external_website_required"], false);
     assert_eq!(status["suspended"], false);
     assert_eq!(suspended["command"], "admin suspend-tenant");
     assert_eq!(suspended["tenant"], "tenant");
     assert_eq!(suspended["actor_tenant"], "admin-tenant");
     assert_eq!(suspended["suspended"], true);
-    assert_eq!(suspended["private_website_required"], false);
+    assert_eq!(suspended["external_website_required"], false);
 }
 
 #[test]
@@ -2968,7 +2967,7 @@ fn debug_attach_reports_public_authorization() {
     assert_eq!(report["charged_debug_read_bytes"], 1024);
     assert_eq!(report["used_debug_read_bytes"], 1024);
     assert_eq!(report["debug_reads_quota_limited"], true);
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
 }
 
 #[test]
@@ -3005,6 +3004,14 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
                 "restart_task",
                 br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
             ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(),
+            ),
+            (
+                "list_task_events",
+                br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(),
+            ),
             (
                 "create_artifact_download_link",
                 br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
@@ -3109,9 +3116,26 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
-    artifact_download_report_with_session(
+    let logs = logs_report_with_session(
+        LogsArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+            task: Some("task-a".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let artifacts = artifact_list_report_with_session(
+        ArtifactListArgs {
+            scope: scope.clone(),
+            process: Some("vp".to_owned()),
+        },
+        Some(&session),
+    )
+    .unwrap();
+    let download = artifact_download_report_with_session(
         ArtifactDownloadArgs {
-            scope: coordinator_scope.clone(),
+            scope,
             artifact: "app.txt".to_owned(),
             to: None,
             max_bytes: 2048,
@@ -3119,6 +3143,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
         Some(&session),
     )
     .unwrap();
+    assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n");
+    assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt");
+    assert_eq!(download["coordinator"], session.coordinator);
     debug_attach_report_with_dap_and_session(
         DebugAttachArgs {
             scope: coordinator_scope,
@@ -3311,7 +3338,7 @@ fn project_init_uses_public_create_before_writing_local_config() {
         created["safe_defaults"]["browser_interaction_required"],
         false
     );
-    assert_eq!(created["private_website_required"], false);
+    assert_eq!(created["external_website_required"], false);
     assert_eq!(
         read_project_config(temp_success.path())
             .unwrap()
@@ -3411,7 +3438,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(list["source"], "public_coordinator_api");
     assert_eq!(list["project_count"], 1);
     assert_eq!(list["projects"][0]["id"], "project-a");
-    assert_eq!(list["private_website_required"], false);
+    assert_eq!(list["external_website_required"], false);
     assert_eq!(list["coordinator_session_requests"], 1);
 
     let selected = project_select_report(
@@ -3426,7 +3453,7 @@ fn project_list_and_select_use_public_api_without_website() {
     assert_eq!(selected["source"], "public_coordinator_api");
     assert_eq!(selected["selected_project"]["id"], "project-a");
     assert_eq!(selected["project_config_written"], true);
-    assert_eq!(selected["private_website_required"], false);
+    assert_eq!(selected["external_website_required"], false);
     assert_eq!(
         read_project_config(temp.path()).unwrap().unwrap().project,
         "project-a"
@@ -4455,7 +4482,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     let temp = tempfile::tempdir().unwrap();
     let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .join("../..")
-        .join("tests/fixtures/runtime-conformance");
+        .join("examples/hello-build");
     let output = temp.path().join("bundle");
 
     let report = build_report(
@@ -4473,8 +4500,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
     assert_eq!(report["command"], "build");
     assert_eq!(report["content_addressed"], true);
     assert_eq!(report["contains_full_repository_upload"], false);
-    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 12);
-    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 6);
+    assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2);
+    assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1);
     assert!(output.join("module.wasm").is_file());
     assert!(output.join("manifest.json").is_file());
     assert!(output.join("task-descriptors.json").is_file());
@@ -4485,9 +4512,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
         .unwrap()
         .iter()
         .any(|descriptor| {
-            descriptor["name"] == "task_add_one"
-                && descriptor["argument_schema"] == "input : i32"
-                && descriptor["result_schema"] == "i32"
+            descriptor["name"] == "compile"
+                && descriptor["argument_schema"] == "source : SourceSnapshot"
+                && descriptor["result_schema"] == "Result < Artifact >"
                 && descriptor["restart_compatibility_hash"]
                     .as_str()
                     .unwrap()
@@ -4648,7 +4675,7 @@ fn node_enroll_reports_short_lived_public_api_grant() {
 
     assert_eq!(report["command"], "node enroll");
     assert_eq!(report["status"], "created");
-    assert_eq!(report["private_website_required"], false);
+    assert_eq!(report["external_website_required"], false);
     assert_eq!(report["tenant"], "tenant");
     assert_eq!(report["project"], "project");
     assert_eq!(report["user"], "user");
@@ -4791,7 +4818,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() {
     )
     .unwrap();
     assert_eq!(enroll["status"], "requires_coordinator");
-    assert_eq!(enroll["private_website_required"], false);
+    assert_eq!(enroll["external_website_required"], false);
     assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
     assert_eq!(enroll["requested_ttl_seconds"], 60);
 

crates/clusterflux-client/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "clusterflux-client"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+clusterflux-control = { path = "../clusterflux-control" }
+clusterflux-core = { path = "../clusterflux-core" }
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["sync", "time"] }
+
+[dev-dependencies]
+clusterflux-coordinator = { path = "../clusterflux-coordinator" }

crates/clusterflux-client/README.md

@@ -0,0 +1,68 @@
+# clusterflux-client
+
+`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web
+backend. It talks to the same versioned control and login endpoints as the CLI.
+Callers do not construct JSON envelopes or place session secrets into request
+objects.
+
+The crate intentionally contains no coordinator implementation, persistence
+types, scheduler policy, or website-specific business rules. A web backend only
+needs this crate to authenticate and work with account status, projects, Agent
+keys, node enrollment and liveness, current/recent processes, task attempts,
+recent logs, artifacts, quota state, process control, task recovery, and Debug
+Epoch state. It does not expose workflow launch or whole-process replay.
+
+```rust,no_run
+use clusterflux_client::{ClusterfluxClient, SessionCredential};
+
+# async fn example() -> Result<(), clusterflux_client::ClientError> {
+let credential = SessionCredential::from_secret(
+    std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"),
+);
+let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")?
+    .with_session_credential(&credential);
+
+let account = client.account_status().await?;
+let nodes = client.list_nodes().await?;
+let processes = client.list_processes(None, 20).await?;
+# let _ = (account, nodes, processes);
+# Ok(())
+# }
+```
+
+## Login boundary
+
+`begin_browser_login` starts the hosted Authentik flow. The identity callback
+returns only to the fixed configured website URL with a short-lived handoff.
+The website backend calls `exchange_browser_login_handoff` once and stores the
+returned `SessionCredential` only in encrypted server-side session storage.
+The credential has redacted `Debug` output and deliberately does not implement
+Serde serialization. Logout calls the existing session-revocation operation.
+
+The browser must not receive or persist the Clusterflux session credential,
+provider tokens, authorization code, PKCE verifier, or CLI polling secret.
+
+## Errors, bounds, and transport
+
+API failures are returned as `ClientError::Api(ApiError)`. Decisions should use
+the stable `code`, `category`, `retryable`, and `request_id` fields, while
+`message` is for people. The client rejects an error whose request ID does not
+match the originating request.
+
+Paginated list methods require an explicit bounded page size. Process pages are
+limited to 100 entries; node, artifact, and recent-log pages to 200.
+`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes
+its cursor. Recent logs are ephemeral and can report truncation or sequence
+loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates
+offsets and decodes bounded chunks without materializing the complete artifact.
+
+`ControlTransport` has bounded connect and I/O timeouts and performs blocking
+network work outside the async executor. Dropping a request future cancels the
+caller’s wait; any already-started blocking I/O remains bounded by its timeout.
+`MockTransport` supplies deterministic responses and records exact envelopes
+for application tests.
+
+`CLIENT_API_VERSION` is the one supported control protocol version. The
+checked-in `web_operations.json` contract fixture records every website
+operation and its stable error shape, and the crate’s contract suite checks the
+fixture.

crates/clusterflux-client/src/lib.rs

@@ -0,0 +1,924 @@
+mod protocol;
+mod transport;
+mod types;
+
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION};
+use protocol::{AuthenticatedRequest, LoginRequest, WireResponse};
+use serde_json::json;
+use thiserror::Error;
+use transport::{ClientTransport, TransportRequest};
+
+pub use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind,
+    Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId,
+    ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId,
+    UserId, VfsPath,
+};
+pub use transport::{
+    ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse,
+};
+pub use types::*;
+
+pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION;
+
+#[derive(Debug, Error)]
+pub enum ClientError {
+    #[error("Clusterflux API error: {0}")]
+    Api(ApiError),
+    #[error(transparent)]
+    Transport(#[from] ClientTransportError),
+    #[error("Clusterflux client protocol error: {0}")]
+    Protocol(String),
+}
+
+#[derive(Clone)]
+pub struct ClusterfluxClient {
+    transport: Arc<dyn ClientTransport>,
+    session_secret: Arc<Mutex<Option<String>>>,
+    next_request: Arc<AtomicU64>,
+}
+
+impl ClusterfluxClient {
+    pub fn connect(endpoint: impl Into<String>) -> Result<Self, ClientError> {
+        Ok(Self::with_transport(ControlTransport::new(endpoint)?))
+    }
+
+    pub fn with_transport(transport: impl ClientTransport) -> Self {
+        Self {
+            transport: Arc::new(transport),
+            session_secret: Arc::new(Mutex::new(None)),
+            next_request: Arc::new(AtomicU64::new(1)),
+        }
+    }
+
+    pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self {
+        self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone())));
+        self
+    }
+
+    pub fn is_session_configured(&self) -> bool {
+        self.session_secret
+            .lock()
+            .map(|secret| secret.is_some())
+            .unwrap_or(false)
+    }
+
+    pub async fn begin_browser_login(&self) -> Result<BrowserLoginStart, ClientError> {
+        match self
+            .send_login(LoginRequest::BeginWebBrowserLogin {})
+            .await?
+        {
+            WireResponse::WebBrowserLoginStarted {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            } => Ok(BrowserLoginStart {
+                transaction_id,
+                authorization_url,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_login_started")),
+        }
+    }
+
+    pub async fn exchange_browser_login_handoff(
+        &self,
+        transaction_id: impl Into<String>,
+        handoff_code: impl Into<String>,
+    ) -> Result<BrowserSession, ClientError> {
+        match self
+            .send_login(LoginRequest::ExchangeWebLoginHandoff {
+                transaction_id: transaction_id.into(),
+                handoff_code: handoff_code.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserSession { session } => Ok(BrowserSession {
+                tenant: session.tenant,
+                project: session.project,
+                user: session.user,
+                credential: SessionCredential(session.session_secret),
+                expires_at_epoch_seconds: session.expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("web_browser_session")),
+        }
+    }
+
+    pub async fn cancel_browser_login(
+        &self,
+        transaction_id: impl Into<String>,
+    ) -> Result<(), ClientError> {
+        match self
+            .send_login(LoginRequest::CancelWebBrowserLogin {
+                transaction_id: transaction_id.into(),
+            })
+            .await?
+        {
+            WireResponse::WebBrowserLoginCancelled {} => Ok(()),
+            _ => Err(unexpected_response("web_browser_login_cancelled")),
+        }
+    }
+
+    pub async fn account_status(&self) -> Result<AccountStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AuthStatus)
+            .await?
+        {
+            WireResponse::AuthStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            } => Ok(AccountStatus {
+                tenant,
+                project,
+                actor,
+                authenticated,
+                account_status,
+                suspended,
+                disabled,
+                deleted,
+                manual_review,
+                sanitized_reason,
+                next_actions,
+            }),
+            _ => Err(unexpected_response("auth_status")),
+        }
+    }
+
+    pub async fn logout(&self) -> Result<(), ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeCliSession)
+            .await?
+        {
+            WireResponse::CliSessionRevoked {} => {
+                *self.session_secret.lock().map_err(|_| {
+                    ClientError::Protocol("session credential lock was poisoned".to_owned())
+                })? = None;
+                Ok(())
+            }
+            _ => Err(unexpected_response("cli_session_revoked")),
+        }
+    }
+
+    pub async fn list_projects(&self) -> Result<Vec<Project>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProjects)
+            .await?
+        {
+            WireResponse::Projects { projects } => Ok(projects),
+            _ => Err(unexpected_response("projects")),
+        }
+    }
+
+    pub async fn create_project(
+        &self,
+        project: ProjectId,
+        name: impl Into<String>,
+    ) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateProject {
+                project,
+                name: name.into(),
+            })
+            .await?
+        {
+            WireResponse::ProjectCreated { project } => Ok(project),
+            _ => Err(unexpected_response("project_created")),
+        }
+    }
+
+    pub async fn select_project(&self, project: ProjectId) -> Result<Project, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::SelectProject { project })
+            .await?
+        {
+            WireResponse::ProjectSelected { project } => Ok(project),
+            _ => Err(unexpected_response("project_selected")),
+        }
+    }
+
+    pub async fn register_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn list_agent_public_keys(&self) -> Result<Vec<AgentPublicKey>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys)
+            .await?
+        {
+            WireResponse::AgentPublicKeys { records } => Ok(records),
+            _ => Err(unexpected_response("agent_public_keys")),
+        }
+    }
+
+    pub async fn rotate_agent_public_key(
+        &self,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey {
+            agent,
+            public_key: public_key.into(),
+        })
+        .await
+    }
+
+    pub async fn revoke_agent_public_key(
+        &self,
+        agent: AgentId,
+    ) -> Result<AgentPublicKey, ClientError> {
+        self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent })
+            .await
+    }
+
+    async fn agent_key_mutation(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<AgentPublicKey, ClientError> {
+        match self.send_authenticated(request).await? {
+            WireResponse::AgentPublicKey { record } => Ok(record),
+            _ => Err(unexpected_response("agent_public_key")),
+        }
+    }
+
+    pub async fn create_node_enrollment_grant(
+        &self,
+        ttl_seconds: u64,
+    ) -> Result<NodeEnrollmentGrant, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds })
+            .await?
+        {
+            WireResponse::NodeEnrollmentGrantCreated {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            } => Ok(NodeEnrollmentGrant {
+                tenant,
+                project,
+                grant,
+                scope,
+                expires_at_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("node_enrollment_grant_created")),
+        }
+    }
+
+    pub async fn list_nodes(&self) -> Result<Vec<NodeSummary>, ClientError> {
+        Ok(self.list_nodes_page(None, 200).await?.nodes)
+    }
+
+    pub async fn list_nodes_page(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<NodePage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::NodeSummaries { nodes, next_cursor } => {
+                Ok(NodePage { nodes, next_cursor })
+            }
+            _ => Err(unexpected_response("node_summaries")),
+        }
+    }
+
+    pub async fn revoke_node(&self, node: NodeId) -> Result<NodeRevocation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node })
+            .await?
+        {
+            WireResponse::NodeCredentialRevoked {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            } => Ok(NodeRevocation {
+                node,
+                tenant,
+                project,
+                actor,
+                descriptor_removed,
+                queued_assignments_removed,
+            }),
+            _ => Err(unexpected_response("node_credential_revoked")),
+        }
+    }
+
+    pub async fn list_processes(
+        &self,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ProcessPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit })
+            .await?
+        {
+            WireResponse::ProcessSummaries {
+                processes,
+                next_cursor,
+            } => Ok(ProcessPage {
+                processes,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("process_summaries")),
+        }
+    }
+
+    pub async fn cancel_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CancelProcess { process })
+            .await?
+        {
+            WireResponse::ProcessCancellationRequested {
+                process,
+                cancelled_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: cancelled_tasks,
+                affected_nodes,
+                aborted: false,
+            }),
+            _ => Err(unexpected_response("process_cancellation_requested")),
+        }
+    }
+
+    pub async fn abort_process(
+        &self,
+        process: ProcessId,
+    ) -> Result<ProcessCancellation, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::AbortProcess {
+                process,
+                launch_attempt: None,
+            })
+            .await?
+        {
+            WireResponse::ProcessAborted {
+                process,
+                aborted_tasks,
+                affected_nodes,
+            } => Ok(ProcessCancellation {
+                process,
+                affected_tasks: aborted_tasks,
+                affected_nodes,
+                aborted: true,
+            }),
+            _ => Err(unexpected_response("process_aborted")),
+        }
+    }
+
+    pub async fn quota_status(&self) -> Result<QuotaStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::QuotaStatus)
+            .await?
+        {
+            WireResponse::QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            } => Ok(QuotaStatus {
+                tenant,
+                project,
+                actor,
+                policy_label,
+                limits,
+                window_seconds,
+                usage,
+                window_started_epoch_seconds,
+            }),
+            _ => Err(unexpected_response("quota_status")),
+        }
+    }
+
+    pub async fn list_task_events(
+        &self,
+        process: Option<ProcessId>,
+    ) -> Result<Vec<TaskCompletionEvent>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskEvents { process })
+            .await?
+        {
+            WireResponse::TaskEvents { events } => Ok(events),
+            _ => Err(unexpected_response("task_events")),
+        }
+    }
+
+    pub async fn list_task_snapshots(
+        &self,
+        process: ProcessId,
+    ) -> Result<Vec<TaskAttemptSnapshot>, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process })
+            .await?
+        {
+            WireResponse::TaskSnapshots { snapshots } => Ok(snapshots),
+            _ => Err(unexpected_response("task_snapshots")),
+        }
+    }
+
+    pub async fn list_recent_logs(
+        &self,
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<RecentLogPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::RecentLogs {
+                entries,
+                next_sequence,
+                history_truncated,
+            } => Ok(RecentLogPage {
+                entries,
+                next_sequence,
+                history_truncated,
+            }),
+            _ => Err(unexpected_response("recent_logs")),
+        }
+    }
+
+    pub async fn restart_task(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    ) -> Result<TaskRestart, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::RestartTask {
+                process,
+                task,
+                replacement_bundle,
+            })
+            .await?
+        {
+            WireResponse::TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            } => Ok(TaskRestart {
+                process,
+                task,
+                restarted_task_instance,
+                restarted_attempt_id,
+                actor,
+                accepted,
+                clean_boundary_available,
+                active_task,
+                completed_event_observed,
+                requires_whole_process_restart,
+                message,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("task_restart")),
+        }
+    }
+
+    pub async fn resolve_task_failure(
+        &self,
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    ) -> Result<TaskFailureResolutionResult, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResolveTaskFailure {
+                process,
+                task,
+                resolution,
+            })
+            .await?
+        {
+            WireResponse::TaskFailureResolved {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            } => Ok(TaskFailureResolutionResult {
+                process,
+                task,
+                attempt_id,
+                resolution,
+            }),
+            _ => Err(unexpected_response("task_failure_resolved")),
+        }
+    }
+
+    pub async fn debug_attach(&self, process: ProcessId) -> Result<DebugAttach, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::DebugAttach { process })
+            .await?
+        {
+            WireResponse::DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            } => Ok(DebugAttach {
+                process,
+                actor,
+                authorization,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_attach")),
+        }
+    }
+
+    pub async fn create_debug_epoch(
+        &self,
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: impl Into<String>,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateDebugEpoch {
+                process,
+                stopped_task,
+                reason: reason.into(),
+            })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn resume_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochControl, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpoch {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            } => Ok(DebugEpochControl {
+                process,
+                actor,
+                epoch,
+                command,
+                affected_tasks,
+                all_stop_requested,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch")),
+        }
+    }
+
+    pub async fn inspect_debug_epoch(
+        &self,
+        process: ProcessId,
+        epoch: u64,
+    ) -> Result<DebugEpochStatus, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch })
+            .await?
+        {
+            WireResponse::DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            } => Ok(DebugEpochStatus {
+                process,
+                actor,
+                epoch,
+                command,
+                expected_tasks,
+                acknowledgements,
+                fully_frozen,
+                partially_frozen,
+                fully_resumed,
+                failed,
+                failure_messages,
+                audit_event,
+            }),
+            _ => Err(unexpected_response("debug_epoch_status")),
+        }
+    }
+
+    pub async fn list_artifacts(
+        &self,
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<ArtifactPage, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            })
+            .await?
+        {
+            WireResponse::Artifacts {
+                artifacts,
+                next_cursor,
+            } => Ok(ArtifactPage {
+                artifacts,
+                next_cursor,
+            }),
+            _ => Err(unexpected_response("artifacts")),
+        }
+    }
+
+    pub async fn get_artifact(&self, artifact: ArtifactId) -> Result<ArtifactSummary, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::GetArtifact { artifact })
+            .await?
+        {
+            WireResponse::Artifact { artifact } => Ok(artifact),
+            _ => Err(unexpected_response("artifact")),
+        }
+    }
+
+    pub async fn begin_artifact_download(
+        &self,
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+        chunk_bytes: u64,
+    ) -> Result<ArtifactDownload, ClientError> {
+        match self
+            .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink {
+                artifact: artifact.clone(),
+                max_bytes,
+                ttl_seconds,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload {
+                client: self.clone(),
+                artifact,
+                max_bytes,
+                chunk_bytes,
+                token_digest: link.scoped_token_digest.clone(),
+                link,
+                expected_offset: 0,
+                finished: false,
+            }),
+            _ => Err(unexpected_response("artifact_download_link")),
+        }
+    }
+
+    async fn send_authenticated(
+        &self,
+        request: AuthenticatedRequest,
+    ) -> Result<WireResponse, ClientError> {
+        let session_secret = self
+            .session_secret
+            .lock()
+            .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))?
+            .clone()
+            .ok_or_else(|| {
+                ClientError::Api(ApiError::from_message(
+                    "client",
+                    "no authenticated Clusterflux session is configured",
+                ))
+            })?;
+        self.send(
+            CONTROL_API_PATH,
+            json!({
+                "type": "authenticated",
+                "session_secret": session_secret,
+                "request": request,
+            }),
+        )
+        .await
+    }
+
+    async fn send_login(&self, request: LoginRequest) -> Result<WireResponse, ClientError> {
+        let payload = serde_json::to_value(request)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        self.send(LOGIN_API_PATH, payload).await
+    }
+
+    async fn send(
+        &self,
+        api_path: &str,
+        payload: serde_json::Value,
+    ) -> Result<WireResponse, ClientError> {
+        let request_number = self.next_request.fetch_add(1, Ordering::Relaxed);
+        let request_id = format!("client-{request_number}");
+        let envelope = coordinator_wire_request(&request_id, payload);
+        let body = serde_json::to_vec(&envelope)
+            .map_err(|error| ClientError::Protocol(error.to_string()))?;
+        let response = self
+            .transport
+            .send(TransportRequest {
+                api_path: api_path.to_owned(),
+                body,
+            })
+            .await?;
+        let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| {
+            ClientError::Protocol(format!("decode typed API response: {error}"))
+        })?;
+        match response {
+            WireResponse::Error {
+                code,
+                category,
+                message,
+                retryable,
+                request_id: response_request_id,
+            } => {
+                if response_request_id != request_id {
+                    return Err(ClientError::Protocol(format!(
+                        "error response request_id {response_request_id} does not match {request_id}"
+                    )));
+                }
+                Err(ClientError::Api(ApiError::new(
+                    code,
+                    category,
+                    message,
+                    retryable,
+                    response_request_id,
+                )))
+            }
+            response => Ok(response),
+        }
+    }
+}
+
+pub struct ArtifactDownload {
+    client: ClusterfluxClient,
+    artifact: ArtifactId,
+    max_bytes: u64,
+    chunk_bytes: u64,
+    token_digest: Digest,
+    link: clusterflux_core::DownloadLink,
+    expected_offset: u64,
+    finished: bool,
+}
+
+impl ArtifactDownload {
+    pub fn link(&self) -> &clusterflux_core::DownloadLink {
+        &self.link
+    }
+
+    pub async fn next_chunk(&mut self) -> Result<ArtifactDownloadPoll, ClientError> {
+        if self.finished {
+            return Ok(ArtifactDownloadPoll::Chunk {
+                offset: self.expected_offset,
+                bytes: Vec::new(),
+                eof: true,
+            });
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream {
+                artifact: self.artifact.clone(),
+                max_bytes: self.max_bytes,
+                token_digest: self.token_digest.clone(),
+                chunk_bytes: self.chunk_bytes,
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadStream {
+                content_bytes_available,
+                content_offset,
+                content_eof,
+                content_base64,
+                ..
+            } => {
+                if !content_bytes_available {
+                    return Ok(ArtifactDownloadPoll::Pending);
+                }
+                let offset = content_offset.ok_or_else(|| {
+                    ClientError::Protocol(
+                        "artifact response contained bytes without an offset".to_owned(),
+                    )
+                })?;
+                if offset != self.expected_offset {
+                    return Err(ClientError::Protocol(format!(
+                        "artifact chunk offset {offset} does not match expected {}",
+                        self.expected_offset
+                    )));
+                }
+                let bytes = BASE64_STANDARD
+                    .decode(content_base64.unwrap_or_default())
+                    .map_err(|error| {
+                        ClientError::Protocol(format!(
+                            "artifact response contains invalid base64: {error}"
+                        ))
+                    })?;
+                self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64);
+                self.finished = content_eof;
+                Ok(ArtifactDownloadPoll::Chunk {
+                    offset,
+                    bytes,
+                    eof: content_eof,
+                })
+            }
+            _ => Err(unexpected_response("artifact_download_stream")),
+        }
+    }
+
+    pub async fn cancel(mut self) -> Result<(), ClientError> {
+        if self.finished {
+            return Ok(());
+        }
+        match self
+            .client
+            .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink {
+                artifact: self.artifact.clone(),
+                token_digest: self.token_digest.clone(),
+            })
+            .await?
+        {
+            WireResponse::ArtifactDownloadLinkRevoked {} => {
+                self.finished = true;
+                Ok(())
+            }
+            _ => Err(unexpected_response("artifact_download_link_revoked")),
+        }
+    }
+}
+
+fn unexpected_response(expected: &str) -> ClientError {
+    ClientError::Protocol(format!(
+        "expected typed response {expected}, received another response variant"
+    ))
+}

crates/clusterflux-client/src/protocol.rs

@@ -0,0 +1,403 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink,
+    LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId,
+};
+use serde::{Deserialize, Serialize};
+
+use crate::types::*;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum AuthenticatedRequest {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject {
+        project: ProjectId,
+        name: String,
+    },
+    SelectProject {
+        project: ProjectId,
+    },
+    ListProjects,
+    RegisterAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    ListAgentPublicKeys,
+    RotateAgentPublicKey {
+        agent: AgentId,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        agent: AgentId,
+    },
+    CreateNodeEnrollmentGrant {
+        ttl_seconds: u64,
+    },
+    ListNodeSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    RevokeNodeCredential {
+        node: NodeId,
+    },
+    ListProcessSummaries {
+        cursor: Option<String>,
+        limit: u32,
+    },
+    CancelProcess {
+        process: ProcessId,
+    },
+    AbortProcess {
+        process: ProcessId,
+        launch_attempt: Option<String>,
+    },
+    QuotaStatus,
+    ListTaskEvents {
+        process: Option<ProcessId>,
+    },
+    ListTaskSnapshots {
+        process: ProcessId,
+    },
+    ListRecentLogs {
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    },
+    RestartTask {
+        process: ProcessId,
+        task: TaskInstanceId,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        process: ProcessId,
+        task: TaskInstanceId,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+    },
+    CreateDebugEpoch {
+        process: ProcessId,
+        stopped_task: TaskInstanceId,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        process: ProcessId,
+        epoch: u64,
+    },
+    ListArtifacts {
+        process: Option<ProcessId>,
+        cursor: Option<String>,
+        limit: u32,
+    },
+    GetArtifact {
+        artifact: ArtifactId,
+    },
+    CreateArtifactDownloadLink {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        artifact: ArtifactId,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        artifact: ArtifactId,
+        token_digest: Digest,
+    },
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum LoginRequest {
+    BeginWebBrowserLogin {},
+    CancelWebBrowserLogin {
+        transaction_id: String,
+    },
+    ExchangeWebLoginHandoff {
+        transaction_id: String,
+        handoff_code: String,
+    },
+}
+
+#[derive(Clone, Deserialize)]
+pub(crate) struct WireBrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub session_secret: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub(crate) enum WireResponse {
+    AuthStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        authenticated: bool,
+        account_status: String,
+        suspended: bool,
+        disabled: bool,
+        deleted: bool,
+        manual_review: bool,
+        sanitized_reason: Option<String>,
+        next_actions: Vec<String>,
+    },
+    CliSessionRevoked {},
+    ProjectCreated {
+        project: Project,
+    },
+    ProjectSelected {
+        project: Project,
+    },
+    Projects {
+        projects: Vec<Project>,
+    },
+    AgentPublicKey {
+        record: AgentPublicKey,
+    },
+    AgentPublicKeys {
+        records: Vec<AgentPublicKey>,
+    },
+    NodeEnrollmentGrantCreated {
+        tenant: TenantId,
+        project: ProjectId,
+        grant: String,
+        scope: String,
+        expires_at_epoch_seconds: u64,
+    },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+    },
+    NodeCredentialRevoked {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        descriptor_removed: bool,
+        queued_assignments_removed: usize,
+    },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+    },
+    ProcessCancellationRequested {
+        process: ProcessId,
+        cancelled_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessAborted {
+        process: ProcessId,
+        aborted_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    QuotaStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        policy_label: Option<String>,
+        limits: ResourceLimits,
+        window_seconds: BTreeMap<LimitKind, u64>,
+        usage: BTreeMap<LimitKind, u64>,
+        window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+    },
+    TaskEvents {
+        events: Vec<TaskCompletionEvent>,
+    },
+    TaskSnapshots {
+        snapshots: Vec<TaskAttemptSnapshot>,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
+    TaskRestart {
+        process: ProcessId,
+        task: TaskInstanceId,
+        restarted_task_instance: Option<TaskInstanceId>,
+        restarted_attempt_id: Option<String>,
+        actor: UserId,
+        accepted: bool,
+        clean_boundary_available: bool,
+        active_task: bool,
+        completed_event_observed: bool,
+        requires_whole_process_restart: bool,
+        message: String,
+        audit_event: DebugAuditEvent,
+    },
+    TaskFailureResolved {
+        process: ProcessId,
+        task: TaskInstanceId,
+        attempt_id: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: ProcessId,
+        actor: UserId,
+        authorization: Authorization,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpoch {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        affected_tasks: Vec<TaskCancellationTarget>,
+        all_stop_requested: bool,
+        audit_event: DebugAuditEvent,
+    },
+    DebugEpochStatus {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        expected_tasks: Vec<TaskCancellationTarget>,
+        acknowledgements: Vec<DebugParticipantAcknowledgement>,
+        fully_frozen: bool,
+        partially_frozen: bool,
+        fully_resumed: bool,
+        failed: bool,
+        failure_messages: Vec<String>,
+        audit_event: DebugAuditEvent,
+    },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
+    ArtifactDownloadLink {
+        link: DownloadLink,
+    },
+    ArtifactDownloadLinkRevoked {},
+    ArtifactDownloadStream {
+        #[serde(rename = "link")]
+        _link: DownloadLink,
+        content_bytes_available: bool,
+        content_offset: Option<u64>,
+        content_eof: bool,
+        content_base64: Option<String>,
+    },
+    WebBrowserLoginStarted {
+        transaction_id: String,
+        authorization_url: String,
+        expires_at_epoch_seconds: u64,
+    },
+    WebBrowserLoginCancelled {},
+    WebBrowserSession {
+        session: WireBrowserSession,
+    },
+    Error {
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: String,
+        retryable: bool,
+        request_id: String,
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use serde::Deserialize;
+    use serde_json::Value;
+
+    use super::*;
+
+    #[derive(Deserialize)]
+    struct OperationFixture {
+        operation: String,
+        boundary: String,
+        request: Value,
+        response: Value,
+    }
+
+    #[test]
+    fn website_operation_contract_fixtures_cover_every_typed_request() {
+        let fixtures: Vec<OperationFixture> =
+            serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap();
+        let expected = BTreeSet::from([
+            "abort_process",
+            "auth_status",
+            "begin_web_browser_login",
+            "cancel_web_browser_login",
+            "cancel_process",
+            "create_artifact_download_link",
+            "create_debug_epoch",
+            "create_node_enrollment_grant",
+            "create_project",
+            "debug_attach",
+            "exchange_web_login_handoff",
+            "get_artifact",
+            "inspect_debug_epoch",
+            "list_agent_public_keys",
+            "list_artifacts",
+            "list_node_summaries",
+            "list_process_summaries",
+            "list_projects",
+            "list_recent_logs",
+            "list_task_events",
+            "list_task_snapshots",
+            "open_artifact_download_stream",
+            "quota_status",
+            "register_agent_public_key",
+            "resolve_task_failure",
+            "restart_task",
+            "resume_debug_epoch",
+            "revoke_agent_public_key",
+            "revoke_artifact_download_link",
+            "revoke_cli_session",
+            "revoke_node_credential",
+            "rotate_agent_public_key",
+            "select_project",
+        ])
+        .into_iter()
+        .map(str::to_owned)
+        .collect();
+        let mut observed = BTreeSet::new();
+
+        for fixture in fixtures {
+            assert_eq!(fixture.request["type"], fixture.operation);
+            let round_trip = match fixture.boundary.as_str() {
+                "control" => {
+                    let typed: AuthenticatedRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                "login" => {
+                    let typed: LoginRequest =
+                        serde_json::from_value(fixture.request.clone()).unwrap();
+                    serde_json::to_value(typed).unwrap()
+                }
+                boundary => panic!("unknown fixture boundary {boundary}"),
+            };
+            assert_eq!(round_trip, fixture.request);
+            let response: WireResponse = serde_json::from_value(fixture.response).unwrap();
+            assert!(
+                !matches!(response, WireResponse::Error { .. }),
+                "{} must carry its operation-specific success response fixture",
+                fixture.operation
+            );
+            assert!(observed.insert(fixture.operation));
+        }
+        assert_eq!(observed, expected);
+    }
+}

crates/clusterflux-client/src/transport.rs

@@ -0,0 +1,250 @@
+use std::collections::{BTreeMap, VecDeque};
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use clusterflux_control::{endpoint_identity, ControlSession};
+use thiserror::Error;
+
+pub type TransportFuture =
+    Pin<Box<dyn Future<Output = Result<TransportResponse, ClientTransportError>> + Send + 'static>>;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportRequest {
+    pub api_path: String,
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct TransportResponse {
+    pub body: Vec<u8>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ClientTransportError {
+    #[error("client transport failed: {0}")]
+    Failed(String),
+    #[error("client transport task failed: {0}")]
+    Task(String),
+}
+
+pub trait ClientTransport: Send + Sync + 'static {
+    fn send(&self, request: TransportRequest) -> TransportFuture;
+}
+
+type ControlSessionPool = Arc<Vec<Mutex<Option<ControlSession>>>>;
+
+pub struct ControlTransport {
+    endpoint: String,
+    connect_timeout: Duration,
+    io_timeout: Duration,
+    sessions: Arc<Mutex<BTreeMap<String, ControlSessionPool>>>,
+    next_session: Arc<AtomicU64>,
+}
+
+// A small pool allows independent HTMX reads to progress concurrently while
+// keeping connection and blocking-worker use strictly bounded.
+const SESSIONS_PER_API_PATH: usize = 4;
+
+impl ControlTransport {
+    pub fn new(endpoint: impl Into<String>) -> Result<Self, ClientTransportError> {
+        Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
+    }
+
+    pub fn with_timeouts(
+        endpoint: impl Into<String>,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ClientTransportError> {
+        let endpoint = endpoint.into();
+        endpoint_identity(&endpoint)
+            .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+        Ok(Self {
+            endpoint,
+            connect_timeout,
+            io_timeout,
+            sessions: Arc::new(Mutex::new(BTreeMap::new())),
+            next_session: Arc::new(AtomicU64::new(0)),
+        })
+    }
+}
+
+impl ClientTransport for ControlTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let endpoint = self.endpoint.clone();
+        let connect_timeout = self.connect_timeout;
+        let io_timeout = self.io_timeout;
+        let sessions = Arc::clone(&self.sessions);
+        let next_session = Arc::clone(&self.next_session);
+        Box::pin(async move {
+            let pool = {
+                let mut sessions = sessions.lock().map_err(|_| {
+                    ClientTransportError::Failed(
+                        "client transport pool lock was poisoned".to_owned(),
+                    )
+                })?;
+                Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| {
+                    Arc::new(
+                        (0..SESSIONS_PER_API_PATH)
+                            .map(|_| Mutex::new(None))
+                            .collect(),
+                    )
+                }))
+            };
+            let slot_index =
+                next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH;
+            tokio::task::spawn_blocking(move || {
+                let value = serde_json::from_slice(&request.body)
+                    .map_err(|error| ClientTransportError::Failed(error.to_string()))?;
+                let mut selected = None;
+                for offset in 0..SESSIONS_PER_API_PATH {
+                    let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                    match pool[index].try_lock() {
+                        Ok(session) if session.is_some() => {
+                            selected = Some(session);
+                            break;
+                        }
+                        Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {}
+                        Err(std::sync::TryLockError::Poisoned(_)) => {
+                            return Err(ClientTransportError::Failed(
+                                "client transport session lock was poisoned".to_owned(),
+                            ));
+                        }
+                    }
+                }
+                if selected.is_none() {
+                    for offset in 0..SESSIONS_PER_API_PATH {
+                        let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
+                        match pool[index].try_lock() {
+                            Ok(session) => {
+                                selected = Some(session);
+                                break;
+                            }
+                            Err(std::sync::TryLockError::WouldBlock) => {}
+                            Err(std::sync::TryLockError::Poisoned(_)) => {
+                                return Err(ClientTransportError::Failed(
+                                    "client transport session lock was poisoned".to_owned(),
+                                ));
+                            }
+                        }
+                    }
+                }
+                let mut session = match selected {
+                    Some(session) => session,
+                    None => pool[slot_index].lock().map_err(|_| {
+                        ClientTransportError::Failed(
+                            "client transport session lock was poisoned".to_owned(),
+                        )
+                    })?,
+                };
+                if session.is_none() {
+                    *session = Some(
+                        ControlSession::connect_to_api_path_with_timeouts(
+                            &endpoint,
+                            &request.api_path,
+                            connect_timeout,
+                            io_timeout,
+                        )
+                        .map_err(|error| ClientTransportError::Failed(error.to_string()))?,
+                    );
+                }
+                let response = session
+                    .as_mut()
+                    .expect("session was initialized for the requested API path")
+                    .request(&value);
+                match response {
+                    Ok(response) => serde_json::to_vec(&response)
+                        .map(|body| TransportResponse { body })
+                        .map_err(|error| ClientTransportError::Failed(error.to_string())),
+                    Err(error) => {
+                        *session = None;
+                        Err(ClientTransportError::Failed(error.to_string()))
+                    }
+                }
+            })
+            .await
+            .map_err(|error| ClientTransportError::Task(error.to_string()))?
+        })
+    }
+}
+
+#[derive(Clone, Default)]
+pub struct MockTransport {
+    state: Arc<Mutex<MockTransportState>>,
+}
+
+#[derive(Default)]
+struct MockTransportState {
+    responses: VecDeque<Result<Vec<u8>, ClientTransportError>>,
+    requests: Vec<TransportRequest>,
+}
+
+impl MockTransport {
+    pub fn from_json_responses(responses: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        let responses = responses
+            .into_iter()
+            .map(|response| Ok(response.into().into_bytes()))
+            .collect();
+        Self {
+            state: Arc::new(Mutex::new(MockTransportState {
+                responses,
+                requests: Vec::new(),
+            })),
+        }
+    }
+
+    pub fn push_json_response(&self, response: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Ok(response.into().into_bytes()));
+    }
+
+    pub fn push_error(&self, message: impl Into<String>) {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .responses
+            .push_back(Err(ClientTransportError::Failed(message.into())));
+    }
+
+    pub fn request_bodies(&self) -> Vec<String> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .iter()
+            .map(|request| String::from_utf8_lossy(&request.body).into_owned())
+            .collect()
+    }
+
+    pub fn requests(&self) -> Vec<TransportRequest> {
+        self.state
+            .lock()
+            .expect("mock transport lock is not poisoned")
+            .requests
+            .clone()
+    }
+}
+
+impl ClientTransport for MockTransport {
+    fn send(&self, request: TransportRequest) -> TransportFuture {
+        let state = Arc::clone(&self.state);
+        Box::pin(async move {
+            let mut state = state.lock().map_err(|_| {
+                ClientTransportError::Failed("mock transport lock was poisoned".to_owned())
+            })?;
+            state.requests.push(request);
+            state
+                .responses
+                .pop_front()
+                .ok_or_else(|| {
+                    ClientTransportError::Failed("mock transport has no queued response".to_owned())
+                })?
+                .map(|body| TransportResponse { body })
+        })
+    }
+}

crates/clusterflux-client/src/types.rs

@@ -0,0 +1,468 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement,
+    ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy,
+    TaskInstanceId, TenantId, UserId, VfsPath,
+};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AccountStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub authenticated: bool,
+    pub account_status: String,
+    pub suspended: bool,
+    pub disabled: bool,
+    pub deleted: bool,
+    pub manual_review: bool,
+    pub sanitized_reason: Option<String>,
+    pub next_actions: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Project {
+    pub id: ProjectId,
+    pub tenant: TenantId,
+    pub name: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AgentPublicKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub agent: AgentId,
+    pub public_key: String,
+    pub public_key_fingerprint: Digest,
+    pub version: u64,
+    pub revoked: bool,
+    pub scopes: Vec<String>,
+    pub human_account_creation_privilege: bool,
+    pub browser_interaction_required_each_run: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeEnrollmentGrant {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub grant: String,
+    pub scope: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodePage {
+    pub nodes: Vec<NodeSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeRevocation {
+    pub node: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub descriptor_removed: bool,
+    pub queued_assignments_removed: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessPage {
+    pub processes: Vec<ProcessSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskTerminalState {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskExecutor {
+    CoordinatorMain,
+    Node,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCompletionEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub node: NodeId,
+    pub executor: TaskExecutor,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub attempt_id: Option<String>,
+    pub placement: Option<Placement>,
+    pub terminal_state: TaskTerminalState,
+    pub status_code: Option<i32>,
+    pub stdout_bytes: u64,
+    pub stderr_bytes: u64,
+    pub stdout_tail: String,
+    pub stderr_tail: String,
+    pub stdout_truncated: bool,
+    pub stderr_truncated: bool,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub result: Option<TaskBoundaryValue>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskAttemptState {
+    Queued,
+    Running,
+    FailedAwaitingAction,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAttemptSnapshot {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub attempt_number: u32,
+    pub task_definition: TaskDefinitionId,
+    pub display_name: String,
+    pub state: TaskAttemptState,
+    pub current: bool,
+    pub node: Option<NodeId>,
+    pub environment_id: Option<String>,
+    pub environment_digest: Option<Digest>,
+    pub argument_summary: Vec<String>,
+    pub handle_summary: Vec<String>,
+    pub command_state: Option<String>,
+    pub vfs_checkpoint: String,
+    pub probe_symbol: Option<String>,
+    pub source_path: Option<String>,
+    pub source_line: Option<u32>,
+    pub restart_compatible: bool,
+    pub failure_policy: TaskFailurePolicy,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub status_code: Option<i32>,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogPage {
+    pub entries: Vec<RecentLogEntry>,
+    pub next_sequence: Option<u64>,
+    pub history_truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactPage {
+    pub artifacts: Vec<ArtifactSummary>,
+    pub next_cursor: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct QuotaStatus {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: UserId,
+    pub policy_label: Option<String>,
+    pub limits: ResourceLimits,
+    pub window_seconds: BTreeMap<LimitKind, u64>,
+    pub usage: BTreeMap<LimitKind, u64>,
+    pub window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCancellationTarget {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessCancellation {
+    pub process: ProcessId,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub affected_nodes: Vec<NodeId>,
+    pub aborted: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAuditEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+    pub actor: UserId,
+    pub operation: String,
+    pub allowed: bool,
+    pub reason: String,
+    pub charged_debug_read_bytes: u64,
+    pub used_debug_read_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAttach {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub authorization: Authorization,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochControl {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub affected_tasks: Vec<TaskCancellationTarget>,
+    pub all_stop_requested: bool,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DebugAcknowledgementState {
+    Frozen,
+    Running,
+    Failed,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipantAcknowledgement {
+    pub node: NodeId,
+    pub task_definition: TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub epoch: u64,
+    pub state: DebugAcknowledgementState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochStatus {
+    pub process: ProcessId,
+    pub actor: UserId,
+    pub epoch: u64,
+    pub command: String,
+    pub expected_tasks: Vec<TaskCancellationTarget>,
+    pub acknowledgements: Vec<DebugParticipantAcknowledgement>,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+    pub failure_messages: Vec<String>,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskRestart {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub restarted_task_instance: Option<TaskInstanceId>,
+    pub restarted_attempt_id: Option<String>,
+    pub actor: UserId,
+    pub accepted: bool,
+    pub clean_boundary_available: bool,
+    pub active_task: bool,
+    pub completed_event_observed: bool,
+    pub requires_whole_process_restart: bool,
+    pub message: String,
+    pub audit_event: DebugAuditEvent,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskFailureResolutionResult {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub resolution: TaskFailureResolution,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskFailureResolution {
+    AcceptFailure,
+    Cancel,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskReplacementBundle {
+    pub bundle_digest: Digest,
+    pub wasm_module_base64: String,
+    pub source_snapshot: Option<Digest>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BrowserLoginStart {
+    pub transaction_id: String,
+    pub authorization_url: String,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub struct SessionCredential(pub(crate) String);
+
+impl SessionCredential {
+    pub fn from_secret(secret: impl Into<String>) -> Self {
+        Self(secret.into())
+    }
+
+    pub fn expose_secret(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Debug for SessionCredential {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("SessionCredential([REDACTED])")
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct BrowserSession {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub credential: SessionCredential,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ArtifactDownloadPoll {
+    Pending,
+    Chunk {
+        offset: u64,
+        bytes: Vec<u8>,
+        eof: bool,
+    },
+}

crates/clusterflux-client/tests/client_contract.rs

@@ -0,0 +1,254 @@
+use std::net::TcpListener;
+use std::time::Duration;
+
+use clusterflux_client::{
+    ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError,
+    ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId,
+    UserId, CLIENT_API_VERSION,
+};
+use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
+use clusterflux_coordinator::service::CoordinatorService;
+use serde_json::{json, Value};
+
+#[tokio::test]
+async fn mock_transport_exercises_typed_envelope_and_session_plumbing() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "projects",
+        "projects": [{
+            "id": "project-one",
+            "tenant": "tenant-one",
+            "name": "Project one"
+        }],
+        "actor": "user-one"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport.clone())
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, CONTROL_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION);
+    assert_eq!(envelope["request_id"], "client-1");
+    assert_eq!(envelope["operation"], "authenticated");
+    assert_eq!(envelope["payload"]["type"], "authenticated");
+    assert_eq!(envelope["payload"]["request"]["type"], "list_projects");
+    assert_eq!(envelope["payload"]["session_secret"], "test-session-secret");
+}
+
+#[tokio::test]
+async fn structured_errors_retain_machine_fields_and_originating_request_id() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "account_suspended",
+        "category": "authorization",
+        "message": "account access is suspended",
+        "retryable": false,
+        "request_id": "client-1"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Api(error) = client.account_status().await.unwrap_err() else {
+        panic!("expected typed API error");
+    };
+    assert_eq!(error.code, ApiErrorCode::AccountSuspended);
+    assert_eq!(error.category, ApiErrorCategory::Authorization);
+    assert_eq!(error.request_id, "client-1");
+    assert!(!error.retryable);
+}
+
+#[tokio::test]
+async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() {
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "web_browser_login_cancelled" }).to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport.clone());
+
+    client
+        .cancel_browser_login("login-transaction")
+        .await
+        .unwrap();
+
+    let requests = transport.requests();
+    assert_eq!(requests.len(), 1);
+    assert_eq!(requests[0].api_path, LOGIN_API_PATH);
+    let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
+    assert_eq!(envelope["operation"], "cancel_web_browser_login");
+    assert_eq!(
+        envelope["payload"],
+        json!({
+            "type": "cancel_web_browser_login",
+            "transaction_id": "login-transaction"
+        })
+    );
+}
+
+#[tokio::test]
+async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() {
+    let transport = MockTransport::from_json_responses([json!({
+        "type": "error",
+        "code": "validation_error",
+        "category": "validation",
+        "message": "bad request",
+        "retryable": false,
+        "request_id": "another-request"
+    })
+    .to_string()]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+
+    let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else {
+        panic!("expected protocol error");
+    };
+    assert!(message.contains("does not match client-1"));
+}
+
+#[test]
+fn session_credentials_are_redacted_from_debug_output() {
+    let credential = SessionCredential::from_secret("must-not-appear");
+    assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])");
+}
+
+#[tokio::test]
+async fn artifact_download_is_a_typed_bounded_chunk_stream() {
+    let link = json!({
+        "artifact": "artifact-one",
+        "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        "artifact_size_bytes": 5,
+        "source": { "RetainedNode": "node-one" },
+        "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one",
+        "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+        "expires_at_epoch_seconds": 1000,
+        "tenant": "tenant-one",
+        "project": "project-one",
+        "process": "process-one",
+        "actor": { "User": "user-one" },
+        "max_bytes": 1024,
+        "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+    });
+    let transport = MockTransport::from_json_responses([
+        json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link.clone(),
+            "streamed_bytes": 0,
+            "charged_download_bytes": 0,
+            "content_bytes_available": false,
+            "content_eof": false
+        })
+        .to_string(),
+        json!({
+            "type": "artifact_download_stream",
+            "link": link,
+            "streamed_bytes": 5,
+            "charged_download_bytes": 5,
+            "content_bytes_available": true,
+            "content_offset": 0,
+            "content_eof": true,
+            "content_base64": "aGVsbG8="
+        })
+        .to_string(),
+    ]);
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("test-session-secret"));
+    let mut download = client
+        .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64)
+        .await
+        .unwrap();
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Pending
+    );
+    assert_eq!(
+        download.next_chunk().await.unwrap(),
+        ArtifactDownloadPoll::Chunk {
+            offset: 0,
+            bytes: b"hello".to_vec(),
+            eof: true,
+        }
+    );
+}
+
+#[tokio::test]
+async fn typed_client_runs_against_the_real_strict_control_endpoint() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let mut service = CoordinatorService::new(7);
+        service
+            .issue_cli_session(
+                TenantId::from("tenant-one"),
+                ProjectId::from("project-one"),
+                UserId::from("user-one"),
+                "real-endpoint-session",
+                None,
+            )
+            .unwrap();
+        let (stream, _) = listener.accept().unwrap();
+        service.handle_stream(stream).unwrap();
+    });
+
+    let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}"))
+        .unwrap()
+        .with_session_credential(&SessionCredential::from_secret("real-endpoint-session"));
+    let projects = client.list_projects().await.unwrap();
+    assert_eq!(projects.len(), 1);
+    assert_eq!(projects[0].id, ProjectId::from("project-one"));
+    let status = client.account_status().await.unwrap();
+    assert!(status.authenticated);
+    assert_eq!(status.actor, UserId::from("user-one"));
+
+    drop(client);
+    server.join().unwrap();
+}
+
+#[tokio::test]
+async fn concurrent_requests_expand_the_bounded_pool_without_serializing() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let address = listener.local_addr().unwrap();
+    let server = std::thread::spawn(move || {
+        let handlers = (0..2)
+            .map(|_| {
+                let (stream, _) = listener.accept().unwrap();
+                std::thread::spawn(move || {
+                    let mut service = CoordinatorService::new(7);
+                    service
+                        .issue_cli_session(
+                            TenantId::from("tenant-one"),
+                            ProjectId::from("project-one"),
+                            UserId::from("user-one"),
+                            "concurrent-session",
+                            None,
+                        )
+                        .unwrap();
+                    service.handle_stream(stream).unwrap();
+                })
+            })
+            .collect::<Vec<_>>();
+        for handler in handlers {
+            handler.join().unwrap();
+        }
+    });
+
+    let transport = ControlTransport::with_timeouts(
+        format!("clusterflux+tcp://{address}"),
+        Duration::from_secs(2),
+        Duration::from_secs(5),
+    )
+    .unwrap();
+    let client = ClusterfluxClient::with_transport(transport)
+        .with_session_credential(&SessionCredential::from_secret("concurrent-session"));
+    let (first, second) = tokio::join!(client.account_status(), client.account_status());
+    assert!(first.unwrap().authenticated);
+    assert!(second.unwrap().authenticated);
+
+    drop(client);
+    server.join().unwrap();
+}

crates/clusterflux-client/tests/fixtures/web_operations.json

@@ -0,0 +1,200 @@
+[
+  {
+    "operation": "begin_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "begin_web_browser_login" },
+    "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "cancel_web_browser_login",
+    "boundary": "login",
+    "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" },
+    "response": { "type": "web_browser_login_cancelled" }
+  },
+  {
+    "operation": "exchange_web_login_handoff",
+    "boundary": "login",
+    "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" },
+    "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } }
+  },
+  {
+    "operation": "auth_status",
+    "boundary": "control",
+    "request": { "type": "auth_status" },
+    "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] }
+  },
+  {
+    "operation": "revoke_cli_session",
+    "boundary": "control",
+    "request": { "type": "revoke_cli_session" },
+    "response": { "type": "cli_session_revoked" }
+  },
+  {
+    "operation": "create_project",
+    "boundary": "control",
+    "request": { "type": "create_project", "project": "project-new", "name": "New project" },
+    "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } }
+  },
+  {
+    "operation": "select_project",
+    "boundary": "control",
+    "request": { "type": "select_project", "project": "project-selected" },
+    "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } }
+  },
+  {
+    "operation": "list_projects",
+    "boundary": "control",
+    "request": { "type": "list_projects" },
+    "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] }
+  },
+  {
+    "operation": "register_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "list_agent_public_keys",
+    "boundary": "control",
+    "request": { "type": "list_agent_public_keys" },
+    "response": { "type": "agent_public_keys", "records": [] }
+  },
+  {
+    "operation": "rotate_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "revoke_agent_public_key",
+    "boundary": "control",
+    "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" },
+    "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
+  },
+  {
+    "operation": "create_node_enrollment_grant",
+    "boundary": "control",
+    "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 },
+    "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 }
+  },
+  {
+    "operation": "list_node_summaries",
+    "boundary": "control",
+    "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 },
+    "response": { "type": "node_summaries", "nodes": [], "next_cursor": null }
+  },
+  {
+    "operation": "revoke_node_credential",
+    "boundary": "control",
+    "request": { "type": "revoke_node_credential", "node": "node-one" },
+    "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 }
+  },
+  {
+    "operation": "list_process_summaries",
+    "boundary": "control",
+    "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 },
+    "response": { "type": "process_summaries", "processes": [], "next_cursor": null }
+  },
+  {
+    "operation": "cancel_process",
+    "boundary": "control",
+    "request": { "type": "cancel_process", "process": "process-one" },
+    "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "abort_process",
+    "boundary": "control",
+    "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null },
+    "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] }
+  },
+  {
+    "operation": "quota_status",
+    "boundary": "control",
+    "request": { "type": "quota_status" },
+    "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } }
+  },
+  {
+    "operation": "list_task_events",
+    "boundary": "control",
+    "request": { "type": "list_task_events", "process": "process-one" },
+    "response": { "type": "task_events", "events": [] }
+  },
+  {
+    "operation": "list_task_snapshots",
+    "boundary": "control",
+    "request": { "type": "list_task_snapshots", "process": "process-one" },
+    "response": { "type": "task_snapshots", "snapshots": [] }
+  },
+  {
+    "operation": "list_recent_logs",
+    "boundary": "control",
+    "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 },
+    "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false }
+  },
+  {
+    "operation": "restart_task",
+    "boundary": "control",
+    "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null },
+    "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resolve_task_failure",
+    "boundary": "control",
+    "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" },
+    "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" }
+  },
+  {
+    "operation": "debug_attach",
+    "boundary": "control",
+    "request": { "type": "debug_attach", "process": "process-one" },
+    "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "create_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "resume_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "inspect_debug_epoch",
+    "boundary": "control",
+    "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 },
+    "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
+  },
+  {
+    "operation": "list_artifacts",
+    "boundary": "control",
+    "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 },
+    "response": { "type": "artifacts", "artifacts": [], "next_cursor": null }
+  },
+  {
+    "operation": "get_artifact",
+    "boundary": "control",
+    "request": { "type": "get_artifact", "artifact": "artifact-one" },
+    "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } }
+  },
+  {
+    "operation": "create_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 },
+    "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } }
+  },
+  {
+    "operation": "open_artifact_download_stream",
+    "boundary": "control",
+    "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 },
+    "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" }
+  },
+  {
+    "operation": "revoke_artifact_download_link",
+    "boundary": "control",
+    "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
+    "response": { "type": "artifact_download_link_revoked" }
+  }
+]

crates/clusterflux-control/src/lib.rs

@@ -61,13 +61,35 @@ impl ControlSession {
         endpoint: &str,
         api_path: &str,
     ) -> Result<Self, ControlTransportError> {
-        let mut session = Self::connect(endpoint)?;
+        Self::connect_to_api_path_with_timeouts(
+            endpoint,
+            api_path,
+            Duration::from_secs(10),
+            Duration::from_secs(30),
+        )
+    }
+
+    pub fn connect_to_api_path_with_timeouts(
+        endpoint: &str,
+        api_path: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ControlTransportError> {
+        let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?;
         #[cfg(not(target_arch = "wasm32"))]
+        {
+            let mut session = session;
             if let ControlTransport::Https { url, .. } = &mut session.transport {
                 *url = endpoint_api_url(endpoint, api_path)?;
             }
             Ok(session)
         }
+        #[cfg(target_arch = "wasm32")]
+        {
+            let _ = api_path;
+            Ok(session)
+        }
+    }
 
     pub fn connect_with_timeouts(
         endpoint: &str,

crates/clusterflux-coordinator/src/durable.rs

@@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
     pub enrollment_scope: String,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct NodeScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub node: NodeId,
+}
+
+impl NodeScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
+        Self {
+            tenant,
+            project,
+            node,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
+        Self::new(tenant.clone(), project.clone(), node.clone())
+    }
+
+    pub fn credential_subject(&self) -> String {
+        format!(
+            "node:{}:{}:{}:{}:{}:{}",
+            self.tenant.as_str().len(),
+            self.tenant,
+            self.project.as_str().len(),
+            self.project,
+            self.node.as_str().len(),
+            self.node
+        )
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct CredentialRecord {
     pub subject: String,
@@ -107,7 +140,7 @@ pub struct DurableState {
     pub tenants: BTreeMap<TenantId, TenantRecord>,
     pub users: BTreeMap<UserId, UserRecord>,
     pub projects: BTreeMap<ProjectId, ProjectRecord>,
-    pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
+    pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
     pub credentials: BTreeMap<String, CredentialRecord>,
     pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
     pub source_provider_configs:

crates/clusterflux-coordinator/src/lib.rs

@@ -13,7 +13,7 @@ pub mod service;
 mod sessions;
 pub use durable::{
     AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
-    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
+    DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
     ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
     TenantRecord, UserRecord,
 };
@@ -128,8 +128,9 @@ impl Coordinator {
         public_key: impl Into<String>,
         enrollment_scope: impl Into<String>,
     ) {
+        let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
         self.durable.node_identities.insert(
-            node.clone(),
+            key,
             NodeIdentityRecord {
                 id: node,
                 tenant,
@@ -181,10 +182,13 @@ impl Coordinator {
             public_key,
             credential.scope.clone(),
         );
+        let subject =
+            NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
+                .credential_subject();
         self.durable.credentials.insert(
-            format!("node:{node}"),
+            subject.clone(),
             CredentialRecord {
-                subject: format!("node:{node}"),
+                subject,
                 tenant: credential.tenant.clone(),
                 project: Some(credential.project.clone()),
                 kind: credential.credential_kind.clone(),
@@ -356,15 +360,12 @@ impl Coordinator {
         project: &ProjectId,
         process: &ProcessId,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
-            .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "node identity is outside the requested tenant/project scope".to_owned(),
-            ));
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
+        {
+            return Err(CoordinatorError::UnknownNode);
         }
         if !self
             .active_processes
@@ -379,14 +380,18 @@ impl Coordinator {
 
     pub fn reconnect_node(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         process: Option<(&ProcessId, u64)>,
     ) -> Result<(), CoordinatorError> {
-        let identity = self
+        if !self
             .durable
             .node_identities
-            .get(node)
-            .ok_or(CoordinatorError::UnknownNode)?;
+            .contains_key(&NodeScopeKey::from_refs(tenant, project, node))
+        {
+            return Err(CoordinatorError::UnknownNode);
+        }
 
         if let Some((process_id, stale_epoch)) = process {
             if stale_epoch != self.coordinator_epoch {
@@ -395,11 +400,7 @@ impl Coordinator {
                     current_epoch: self.coordinator_epoch,
                 });
             }
-            let key = (
-                identity.tenant.clone(),
-                identity.project.clone(),
-                process_id.clone(),
-            );
+            let key = (tenant.clone(), project.clone(), process_id.clone());
             if let Some(active) = self.active_processes.get_mut(&key) {
                 active.connected_nodes.insert(node.clone());
             }
@@ -416,23 +417,27 @@ impl Coordinator {
         let identity = self
             .durable
             .node_identities
-            .get(node)
+            .get(&NodeScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                node,
+            ))
             .ok_or(CoordinatorError::UnknownNode)?
             .clone();
-        if identity.tenant != context.tenant || identity.project != context.project {
-            return Err(CoordinatorError::Unauthorized(
-                "node credential is outside the signed-in tenant/project scope".to_owned(),
-            ));
-        }
         if !matches!(context.actor, Actor::User(_)) {
             return Err(CoordinatorError::Unauthorized(
                 "node credential revocation requires a user identity".to_owned(),
             ));
         }
-        self.durable.node_identities.remove(node);
-        self.durable.credentials.remove(&format!("node:{node}"));
-        for active in self.active_processes.values_mut() {
-            active.connected_nodes.remove(node);
+        let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
+        self.durable.node_identities.remove(&key);
+        self.durable.credentials.remove(&key.credential_subject());
+        for active in self
+            .active_processes
+            .values_mut()
+            .filter(|active| active.tenant == context.tenant && active.project == context.project)
+        {
+            active.connected_nodes.remove(&key.node);
         }
         Ok(identity)
     }
@@ -580,8 +585,31 @@ impl Coordinator {
             .count()
     }
 
-    pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
-        self.durable.node_identities.get(id)
+    pub fn tenant_count(&self) -> usize {
+        self.durable.tenants.len()
+    }
+
+    pub fn user_count(&self) -> usize {
+        self.durable.users.len()
+    }
+
+    pub fn project_count(&self) -> usize {
+        self.durable.projects.len()
+    }
+
+    pub fn node_identity_count(&self) -> usize {
+        self.durable.node_identities.len()
+    }
+
+    pub fn node_identity(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        id: &NodeId,
+    ) -> Option<&NodeIdentityRecord> {
+        self.durable
+            .node_identities
+            .get(&NodeScopeKey::from_refs(tenant, project, id))
     }
 
     pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@@ -683,12 +711,24 @@ mod tests {
             .contains_key(&TenantId::from("tenant")));
         assert!(restarted.durable.users.contains_key(&UserId::from("user")));
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         assert_eq!(
             restarted
                 .durable
                 .credentials
-                .get("node:node")
+                .get(&node_subject)
                 .map(|credential| &credential.kind),
             Some(&CredentialKind::NodeCredential)
         );
@@ -712,7 +752,12 @@ mod tests {
         );
         assert_eq!(rerun.coordinator_epoch, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), Some((&process, 2)))
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                Some((&process, 2)),
+            )
             .unwrap();
         assert!(restarted
             .active_process(
@@ -725,6 +770,68 @@ mod tests {
             .contains(&NodeId::from("node")));
     }
 
+    #[test]
+    fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
+        let store = InMemoryDurableStore::default();
+        let mut coordinator = Coordinator::boot(&store, 1);
+        let node = NodeId::from("shared-node");
+        let scopes = [
+            (TenantId::from("tenant-a"), ProjectId::from("project-a")),
+            (TenantId::from("tenant-b"), ProjectId::from("project-b")),
+            (TenantId::from("tenant-a"), ProjectId::from("project-c")),
+        ];
+        for (index, (tenant, project)) in scopes.iter().enumerate() {
+            let mut grant = coordinator.create_node_enrollment_grant(
+                tenant.clone(),
+                project.clone(),
+                format!("grant-{index}"),
+                "node:attach",
+                100,
+            );
+            coordinator
+                .exchange_node_enrollment_grant(
+                    &mut grant,
+                    node.clone(),
+                    &format!("public-key-{index}"),
+                    "node:attach",
+                    99,
+                )
+                .unwrap();
+        }
+
+        let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
+        let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
+        let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
+        assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
+        assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
+        assert_eq!(
+            coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
+            coordinator.durable.node_identities.get(&scope_a)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
+            coordinator.durable.node_identities.get(&scope_b)
+        );
+        assert_eq!(
+            coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
+            coordinator.durable.node_identities.get(&scope_c)
+        );
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_a.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_b.credential_subject()));
+        assert!(coordinator
+            .durable
+            .credentials
+            .contains_key(&scope_c.credential_subject()));
+        assert_eq!(coordinator.durable.node_identities.len(), 3);
+        assert_eq!(coordinator.durable.credentials.len(), 3);
+    }
+
     #[test]
     fn identical_process_ids_are_isolated_by_tenant_and_project() {
         let store = InMemoryDurableStore::default();
@@ -773,11 +880,18 @@ mod tests {
 
         let mut restarted = Coordinator::boot(&store, 2);
         restarted
-            .reconnect_node(&NodeId::from("node"), None)
+            .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+                None,
+            )
             .unwrap();
 
         let error = restarted
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -809,7 +923,13 @@ mod tests {
             .unwrap();
 
         assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
+        assert!(coordinator
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
     }
 
     #[test]
@@ -823,10 +943,16 @@ mod tests {
             "public-key",
             "node:attach",
         );
+        let node_subject = NodeScopeKey::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+        )
+        .credential_subject();
         coordinator.durable.credentials.insert(
-            "node:node".to_owned(),
+            node_subject.clone(),
             CredentialRecord {
-                subject: "node:node".to_owned(),
+                subject: node_subject.clone(),
                 tenant: TenantId::from("tenant"),
                 project: Some(ProjectId::from("project")),
                 kind: CredentialKind::NodeCredential,
@@ -840,6 +966,8 @@ mod tests {
         );
         coordinator
             .reconnect_node(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
                 &NodeId::from("node"),
                 Some((&ProcessId::from("process"), 1)),
             )
@@ -855,7 +983,7 @@ mod tests {
                 &NodeId::from("node"),
             )
             .unwrap_err();
-        assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(foreign, CoordinatorError::UnknownNode));
 
         let revoked = coordinator
             .revoke_node_credential(
@@ -868,8 +996,14 @@ mod tests {
             )
             .unwrap();
         assert_eq!(revoked.id, NodeId::from("node"));
-        assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
-        assert!(!coordinator.durable.credentials.contains_key("node:node"));
+        assert!(coordinator
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_none());
+        assert!(!coordinator.durable.credentials.contains_key(&node_subject));
         assert!(!coordinator
             .active_process(
                 &TenantId::from("tenant"),
@@ -928,7 +1062,7 @@ mod tests {
     }
 
     #[test]
-    fn account_policy_state_summarizes_private_admin_records_safely() {
+    fn account_policy_state_summarizes_sensitive_admin_records_safely() {
         let tenant = TenantId::from("tenant");
         let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
 
@@ -1084,7 +1218,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, CoordinatorError::Unauthorized(_)));
+        assert!(matches!(error, CoordinatorError::UnknownNode));
     }
 
     #[test]

crates/clusterflux-coordinator/src/main.rs

@@ -5,17 +5,40 @@ use clusterflux_core::{ProjectId, TenantId, UserId};
 use serde_json::json;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux coordinator.\n\n\
+                 Usage: clusterflux-coordinator [OPTIONS]\n\n\
+                 Options:\n  \
+                   --listen <ADDRESS>                 [default: 127.0.0.1:0]\n  \
+                   --allow-local-trusted-loopback\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
+
     let mut listen = "127.0.0.1:0".to_owned();
     let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
         .ok()
         .as_deref()
         == Some("1");
-    let mut args = std::env::args().skip(1);
+    let mut args = raw_args.into_iter();
     while let Some(arg) = args.next() {
         if arg == "--listen" {
             listen = args.next().ok_or("--listen requires an address")?;
         } else if arg == "--allow-local-trusted-loopback" {
             allow_local_trusted = true;
+        } else {
+            return Err(format!("unknown argument: {arg}").into());
         }
     }
 

crates/clusterflux-coordinator/src/postgres_store.rs

@@ -4,8 +4,8 @@ use thiserror::Error;
 
 use crate::{
     AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
-    DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
-    ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
+    DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
+    ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
 };
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
             state.projects.insert(record.id.clone(), record);
         }
         for record in self.query_records::<NodeIdentityRecord>(
-            "SELECT record FROM clusterflux_node_identities ORDER BY node_id",
+            "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
         )? {
-            state.node_identities.insert(record.id.clone(), record);
+            state.node_identities.insert(
+                NodeScopeKey::new(
+                    record.tenant.clone(),
+                    record.project.clone(),
+                    record.id.clone(),
+                ),
+                record,
+            );
         }
         for record in self.query_records::<CredentialRecord>(
             "SELECT record FROM clusterflux_credentials ORDER BY subject",
@@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
 );
 
 CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
-    node_id TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
     project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
-    record JSONB NOT NULL
+    node_id TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, node_id)
 );
 
+DO $clusterflux_node_scope_migration$
+DECLARE
+    current_primary_key TEXT;
+    current_primary_key_columns TEXT[];
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_node_identities
+        WHERE record->>'id' IS DISTINCT FROM node_id
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration refused an internally inconsistent legacy row';
+    END IF;
+
+    SELECT
+        constraint_record.conname,
+        array_agg(attribute.attname ORDER BY key_column.ordinality)
+    INTO current_primary_key, current_primary_key_columns
+    FROM pg_constraint AS constraint_record
+    CROSS JOIN LATERAL unnest(constraint_record.conkey)
+        WITH ORDINALITY AS key_column(attribute_number, ordinality)
+    JOIN pg_attribute AS attribute
+      ON attribute.attrelid = constraint_record.conrelid
+     AND attribute.attnum = key_column.attribute_number
+    WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
+      AND constraint_record.contype = 'p'
+    GROUP BY constraint_record.conname;
+
+    IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
+        EXECUTE format(
+            'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
+            current_primary_key
+        );
+        ALTER TABLE clusterflux_node_identities
+            ADD PRIMARY KEY (tenant_id, project_id, node_id);
+    ELSIF current_primary_key_columns
+        IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
+    THEN
+        RAISE EXCEPTION
+            'clusterflux node identity migration found an unexpected primary key shape';
+    END IF;
+END
+$clusterflux_node_scope_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     subject TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
     record JSONB NOT NULL
 );
 
+DO $clusterflux_node_credential_migration$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials
+        WHERE record->>'subject' IS DISTINCT FROM subject
+           OR record->>'tenant' IS DISTINCT FROM tenant_id
+           OR record->>'project' IS DISTINCT FROM project_id
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration refused an internally inconsistent legacy row';
+    END IF;
+
+    UPDATE clusterflux_credentials AS credential
+    SET subject = format(
+            'node:%s:%s:%s:%s:%s:%s',
+            octet_length(identity.tenant_id),
+            identity.tenant_id,
+            octet_length(identity.project_id),
+            identity.project_id,
+            octet_length(identity.node_id),
+            identity.node_id
+        ),
+        record = jsonb_set(
+            credential.record,
+            '{subject}',
+            to_jsonb(format(
+                'node:%s:%s:%s:%s:%s:%s',
+                octet_length(identity.tenant_id),
+                identity.tenant_id,
+                octet_length(identity.project_id),
+                identity.project_id,
+                octet_length(identity.node_id),
+                identity.node_id
+            )),
+            false
+        )
+    FROM clusterflux_node_identities AS identity
+    WHERE credential.subject = 'node:' || identity.node_id
+      AND credential.tenant_id = identity.tenant_id
+      AND credential.project_id = identity.project_id;
+
+    IF EXISTS (
+        SELECT 1
+        FROM clusterflux_credentials AS credential
+        WHERE credential.subject LIKE 'node:%'
+          AND NOT EXISTS (
+              SELECT 1
+              FROM clusterflux_node_identities AS identity
+              WHERE credential.tenant_id = identity.tenant_id
+                AND credential.project_id = identity.project_id
+                AND credential.subject = format(
+                    'node:%s:%s:%s:%s:%s:%s',
+                    octet_length(identity.tenant_id),
+                    identity.tenant_id,
+                    octet_length(identity.project_id),
+                    identity.project_id,
+                    octet_length(identity.node_id),
+                    identity.node_id
+                )
+          )
+    ) THEN
+        RAISE EXCEPTION
+            'clusterflux node credential migration found an unscoped or orphaned node subject';
+    END IF;
+END
+$clusterflux_node_credential_migration$;
+
 CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
     session_digest TEXT PRIMARY KEY,
     tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@@ -529,7 +651,13 @@ mod tests {
         let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
 
         assert!(restarted.project(&ProjectId::from("project")).is_some());
-        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert!(restarted
+            .node_identity(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &NodeId::from("node"),
+            )
+            .is_some());
         assert_eq!(restarted.active_process_count(), 0);
     }
 

crates/clusterflux-coordinator/src/service.rs

@@ -7,13 +7,14 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use clusterflux_core::{
-    Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
-    NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
-    RateLimit, TenantId, TransportError, UserId,
+    Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
+    CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
+    NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
+    TenantId, TransportError, UserId,
 };
 use thiserror::Error;
 
-use crate::{Coordinator, CoordinatorError};
+use crate::{Coordinator, CoordinatorError, NodeScopeKey};
 
 mod admin;
 mod artifacts;
@@ -34,6 +35,7 @@ mod quota;
 mod relay;
 mod routing;
 mod signed_nodes;
+mod summaries;
 mod tcp;
 mod wire_protocol;
 use authorization::authorize_authenticated_user_operation;
@@ -43,12 +45,14 @@ use keys::{
     ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
 };
 pub use protocol::{
-    ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
-    CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
-    DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
-    TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
-    TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
-    TaskTerminalState, VirtualProcessStatus, WorkflowActor,
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
+    DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
+    RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
+    TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
+    TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
+    VirtualProcessStatus, WorkflowActor,
 };
 pub use quota::CoordinatorQuotaConfiguration;
 pub use relay::{
@@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
 const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
 const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
 const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
-const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
+const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
 const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
 const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
+const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
+const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
+const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
+const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
+const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
+const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
 const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
 fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
     requested.clamp(1, maximum)
@@ -111,6 +121,24 @@ pub struct CoordinatorAdmission {
     pub max_artifact_download_ttl_seconds: u64,
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CoordinatorOperationalMetrics {
+    pub tenants: usize,
+    pub users: usize,
+    pub projects: usize,
+    pub enrolled_nodes: usize,
+    pub reported_nodes: usize,
+    pub live_nodes: usize,
+    pub active_processes: usize,
+    pub active_coordinator_mains: usize,
+    pub max_active_coordinator_mains: usize,
+    pub active_tasks: usize,
+    pub queued_tasks: usize,
+    pub artifacts: usize,
+    pub retained_download_links: usize,
+    pub relay: ArtifactRelayUsage,
+}
+
 impl Default for CoordinatorAdmission {
     fn default() -> Self {
         Self {
@@ -151,16 +179,139 @@ pub enum CoordinatorServiceError {
     Durable(String),
 }
 
+impl CoordinatorServiceError {
+    pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
+        let request_id = request_id.into();
+        let message = self.to_string();
+        let (code, category, retryable) = match self {
+            Self::Io(_) => (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Json(_)
+            | Self::CapabilityReport(_)
+            | Self::InvalidArtifactPath(_)
+            | Self::InvalidTaskLogTail(_) => (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            ),
+            Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
+                return ApiError::from_message(request_id, message);
+            }
+            Self::Coordinator(CoordinatorError::UnknownNode) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Coordinator(CoordinatorError::Enrollment(_)) => (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            ),
+            Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
+                (ApiErrorCode::Conflict, ApiErrorCategory::State, true)
+            }
+            Self::Download(DownloadError::NotFound) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Download(DownloadError::Unavailable)
+            | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Download(DownloadError::LimitExceeded { .. }) => (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            ),
+            Self::Download(DownloadError::Unauthorized(_))
+            | Self::Download(DownloadError::InvalidToken)
+            | Self::Download(DownloadError::Expired)
+            | Self::Download(DownloadError::Revoked) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Scheduler(_) => (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Transport(_) => (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            ),
+            Self::Panel(PanelError::RateLimited) => (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            ),
+            Self::Panel(PanelError::UnknownWidget(_)) => {
+                (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+            }
+            Self::Panel(_) => (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            ),
+            Self::Durable(_) => (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                true,
+            ),
+        };
+        ApiError::new(code, category, message, retryable, request_id)
+    }
+}
+
 pub struct CoordinatorService {
     coordinator: Coordinator,
     store: RuntimeDurableStore,
-    node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
-    node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
+    node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
+    node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
     node_stale_after_seconds: u64,
     debug_freeze_timeout: std::time::Duration,
     enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
     task_events: VecDeque<TaskCompletionEvent>,
     process_scope_history: VecDeque<ProcessControlKey>,
+    process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
+    process_summary_order: VecDeque<ProcessControlKey>,
+    next_process_summary_order: u64,
+    task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
+    recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
+    recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
+    recent_log_accounted_bytes: BTreeMap<
+        (
+            TenantId,
+            ProjectId,
+            ProcessId,
+            clusterflux_core::TaskInstanceId,
+            String,
+        ),
+        u64,
+    >,
+    recent_log_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    recent_log_quota_truncated_streams: BTreeSet<(
+        TenantId,
+        ProjectId,
+        ProcessId,
+        clusterflux_core::TaskInstanceId,
+        String,
+    )>,
+    next_recent_log_sequence: u64,
     debug_audit_events: VecDeque<DebugAuditEvent>,
     debug_epochs: BTreeMap<ProcessControlKey, u64>,
     debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@@ -180,7 +331,7 @@ pub struct CoordinatorService {
     process_cancellations: BTreeSet<ProcessControlKey>,
     process_aborts: BTreeSet<ProcessControlKey>,
     agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
-    node_replay_nonces: BTreeMap<(NodeId, String), u64>,
+    node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
     panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
     stopped_panels: BTreeSet<PanelStopKey>,
     panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@@ -215,6 +366,29 @@ impl CoordinatorService {
         self.artifact_relay.usage()
     }
 
+    pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
+        CoordinatorOperationalMetrics {
+            tenants: self.coordinator.tenant_count(),
+            users: self.coordinator.user_count(),
+            projects: self.coordinator.project_count(),
+            enrolled_nodes: self.coordinator.node_identity_count(),
+            reported_nodes: self.node_descriptors.len(),
+            live_nodes: self
+                .node_descriptors
+                .keys()
+                .filter(|scope| self.node_is_live(scope))
+                .count(),
+            active_processes: self.coordinator.active_process_count(),
+            active_coordinator_mains: self.main_runtime.active_main_count(),
+            max_active_coordinator_mains: self.main_runtime.max_active_mains(),
+            active_tasks: self.active_tasks.len(),
+            queued_tasks: self.pending_task_launches.len(),
+            artifacts: self.artifact_registry.artifact_count(),
+            retained_download_links: self.artifact_registry.retained_download_link_count(),
+            relay: self.artifact_relay.usage(),
+        }
+    }
+
     fn commit_artifact_relay(
         &mut self,
         candidate: relay::ArtifactRelayLedger,
@@ -284,15 +458,10 @@ impl CoordinatorService {
         {
             let identity = self
                 .coordinator
-                .node_identity(node)
+                .node_identity(tenant, project, node)
                 .ok_or(CoordinatorError::UnknownNode)?;
-            if &identity.tenant != tenant || &identity.project != project {
-                return Err(CoordinatorError::Unauthorized(
-                    "node process-control request is outside its enrolled tenant/project scope"
-                        .to_owned(),
-                )
-                .into());
-            }
+            debug_assert_eq!(&identity.tenant, tenant);
+            debug_assert_eq!(&identity.project, project);
             return Ok(());
         }
         self.coordinator
@@ -409,6 +578,16 @@ impl CoordinatorService {
             enrollment_grants: BTreeMap::new(),
             task_events: VecDeque::new(),
             process_scope_history: VecDeque::new(),
+            process_summaries: BTreeMap::new(),
+            process_summary_order: VecDeque::new(),
+            next_process_summary_order: 1,
+            task_terminal_states: BTreeMap::new(),
+            recent_logs: BTreeMap::new(),
+            recent_log_dropped_through: BTreeMap::new(),
+            recent_log_accounted_bytes: BTreeMap::new(),
+            recent_log_truncated_streams: BTreeSet::new(),
+            recent_log_quota_truncated_streams: BTreeSet::new(),
+            next_recent_log_sequence: 1,
             debug_audit_events: VecDeque::new(),
             debug_epochs: BTreeMap::new(),
             debug_epoch_runtime: BTreeMap::new(),

crates/clusterflux-coordinator/src/service/artifacts.rs

@@ -8,7 +8,7 @@ use clusterflux_core::{
 };
 use sha2::{Digest as _, Sha256};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::relay::RelayFinishReason;
 use super::{
@@ -52,7 +52,11 @@ impl CoordinatorService {
         let action = self
             .artifact_registry
             .download_action(&context, &artifact, &policy)?;
-        self.ensure_download_source_connectivity(&action.source)?;
+        self.ensure_download_source_connectivity(
+            &context.tenant,
+            &context.project,
+            &action.source,
+        )?;
         let downloadable_size = self
             .artifact_registry
             .downloadable_size(&context, &artifact, &policy)?;
@@ -163,7 +167,11 @@ impl CoordinatorService {
             },
             &mut validation_meter,
         )?;
-        self.ensure_download_source_connectivity(&stream.link.source)?;
+        self.ensure_download_source_connectivity(
+            &stream.link.tenant,
+            &stream.link.project,
+            &stream.link.source,
+        )?;
         self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
 
         if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@@ -302,7 +310,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let transfer_id = generate_opaque_token("artifact_transfer")
             .map_err(CoordinatorServiceError::Protocol)?;
@@ -410,7 +418,7 @@ impl CoordinatorService {
         };
         let metadata = self
             .artifact_registry
-            .metadata(&artifact)
+            .metadata(&context.tenant, &context.project, &artifact)
             .ok_or(clusterflux_core::DownloadError::NotFound)?;
         let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
         let destination =
@@ -682,15 +690,10 @@ impl CoordinatorService {
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "artifact reverse transfer node is outside its enrolled tenant/project scope"
-                    .to_owned(),
-            )
-            .into());
-        }
+        debug_assert_eq!(&identity.tenant, tenant);
+        debug_assert_eq!(&identity.project, project);
         Ok(())
     }
 
@@ -725,15 +728,12 @@ impl CoordinatorService {
     ) -> Result<NodeEndpoint, CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(tenant, project, node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if &identity.tenant != tenant || &identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "artifact export node is outside the tenant/project scope".to_owned(),
-            )
-            .into());
-        }
-        let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+        debug_assert_eq!(&identity.tenant, tenant);
+        debug_assert_eq!(&identity.project, project);
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
+        let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "node {node} has not reported export connectivity"
             ))
@@ -744,7 +744,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "node {node} is offline for artifact export"
@@ -769,17 +769,20 @@ impl CoordinatorService {
 
     fn ensure_download_source_connectivity(
         &self,
+        tenant: &TenantId,
+        project: &ProjectId,
         source: &StorageLocation,
     ) -> Result<(), clusterflux_core::DownloadError> {
         let StorageLocation::RetainedNode(node) = source else {
             return Ok(());
         };
-        let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+        let node_scope = NodeScopeKey::from_refs(tenant, project, node);
+        let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
             clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                 "retaining node {node} has not reported online status for artifact download"
             ))
         })?;
-        if !self.node_is_live(node) {
+        if !self.node_is_live(&node_scope) {
             return Err(
                 clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
                     "retaining node {node} is offline for artifact download"

crates/clusterflux-coordinator/src/service/authorization.rs

@@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation {
     RevokeAgentPublicKey,
     CreateNodeEnrollmentGrant,
     ListNodeDescriptors,
+    ListNodeSummaries,
     RevokeNodeCredential,
     StartProcess,
     ScheduleTask,
@@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation {
     CancelProcess,
     AbortProcess,
     ListProcesses,
+    ListProcessSummaries,
     QuotaStatus,
     RestartTask,
     ResolveTaskFailure,
@@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation {
     InspectDebugEpoch,
     ListTaskEvents,
     ListTaskSnapshots,
+    ListRecentLogs,
     JoinTask,
+    ListArtifacts,
+    GetArtifact,
     CreateArtifactDownloadLink,
     OpenArtifactDownloadStream,
     RevokeArtifactDownloadLink,
@@ -56,6 +61,7 @@ impl PublicUserOperation {
             Self::RevokeAgentPublicKey => "revoke_agent_public_key",
             Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
             Self::ListNodeDescriptors => "list_node_descriptors",
+            Self::ListNodeSummaries => "list_node_summaries",
             Self::RevokeNodeCredential => "revoke_node_credential",
             Self::StartProcess => "start_process",
             Self::ScheduleTask => "schedule_task",
@@ -63,6 +69,7 @@ impl PublicUserOperation {
             Self::CancelProcess => "cancel_process",
             Self::AbortProcess => "abort_process",
             Self::ListProcesses => "list_processes",
+            Self::ListProcessSummaries => "list_process_summaries",
             Self::QuotaStatus => "quota_status",
             Self::RestartTask => "restart_task",
             Self::ResolveTaskFailure => "resolve_task_failure",
@@ -74,7 +81,10 @@ impl PublicUserOperation {
             Self::InspectDebugEpoch => "inspect_debug_epoch",
             Self::ListTaskEvents => "list_task_events",
             Self::ListTaskSnapshots => "list_task_snapshots",
+            Self::ListRecentLogs => "list_recent_logs",
             Self::JoinTask => "join_task",
+            Self::ListArtifacts => "list_artifacts",
+            Self::GetArtifact => "get_artifact",
             Self::CreateArtifactDownloadLink => "create_artifact_download_link",
             Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
             Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
@@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
                 Self::CreateNodeEnrollmentGrant
             }
             AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries,
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
                 Self::RevokeNodeCredential
             }
@@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
             AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
             AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => {
+                Self::ListProcessSummaries
+            }
             AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
             AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
             AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
@@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
             AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
             AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
             AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
+            AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs,
             AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
+            AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts,
+            AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact,
             AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
                 Self::CreateArtifactDownloadLink
             }

crates/clusterflux-coordinator/src/service/debug.rs

@@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct DebugBreakpointPlan {
     pub(super) actor: UserId,
+    pub(super) revision: u64,
     pub(super) probe_symbols: BTreeSet<String>,
     pub(super) hit_epoch: Option<u64>,
     pub(super) hit_task: Option<TaskInstanceId>,
@@ -85,6 +86,7 @@ impl CoordinatorService {
         project: String,
         actor_user: String,
         process: String,
+        revision: u64,
         probe_symbols: Vec<String>,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let probe_symbols = validate_probe_symbols(probe_symbols)?;
@@ -115,10 +117,28 @@ impl CoordinatorService {
             ))
             .into());
         }
+        let key = process_control_key(&tenant, &project, &process);
+        if let Some(current) = self.debug_breakpoints.get(&key) {
+            if current.actor == actor && revision < current.revision {
+                return Ok(CoordinatorResponse::DebugBreakpoints {
+                    process,
+                    actor,
+                    revision: current.revision,
+                    probe_symbols: current.probe_symbols.iter().cloned().collect(),
+                    hit_epoch: current.hit_epoch,
+                    hit_task: current.hit_task.clone(),
+                    hit_probe_symbol: current.hit_probe_symbol.clone(),
+                    charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
+                    used_debug_read_bytes: audit_event.used_debug_read_bytes,
+                    audit_event,
+                });
+            }
+        }
         self.debug_breakpoints.insert(
-            process_control_key(&tenant, &project, &process),
+            key,
             DebugBreakpointPlan {
                 actor: actor.clone(),
+                revision,
                 probe_symbols: probe_symbols.iter().cloned().collect(),
                 hit_epoch: None,
                 hit_task: None,
@@ -128,6 +148,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision,
             probe_symbols,
             hit_epoch: None,
             hit_task: None,
@@ -190,6 +211,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::DebugBreakpoints {
             process,
             actor,
+            revision: plan.revision,
             probe_symbols: plan.probe_symbols.into_iter().collect(),
             hit_epoch: plan.hit_epoch,
             hit_task: plan.hit_task,

crates/clusterflux-coordinator/src/service/debug_requests.rs

@@ -33,12 +33,14 @@ impl CoordinatorService {
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant,
                 project,
                 actor_user,
                 process,
+                revision,
                 probe_symbols,
             ),
             CoordinatorRequest::InspectDebugBreakpoints {
@@ -96,12 +98,14 @@ impl CoordinatorService {
             ),
             AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
                 process,
+                revision,
                 probe_symbols,
             } => self.handle_set_debug_breakpoints(
                 tenant.as_str().to_owned(),
                 project.as_str().to_owned(),
                 actor.as_str().to_owned(),
                 process,
+                revision,
                 probe_symbols,
             ),
             AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
@@ -222,24 +226,25 @@ impl CoordinatorService {
         } else if !completed_event_observed {
             "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
         } else if let Some(checkpoint) = checkpoint {
-            let vfs_available =
-                checkpoint
-                    .checkpoint
-                    .vfs_manifest
-                    .objects
-                    .iter()
-                    .all(|(path, object)| {
-                        let artifact = super::keys::artifact_id_from_path(path);
-                        self.artifact_registry
-                            .metadata(&artifact)
+            let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
+                true,
+                |available, (path, object)| {
+                    let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
+                        CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                    })?;
+                    Ok::<_, CoordinatorServiceError>(
+                        available
+                            && self
+                                .artifact_registry
+                                .metadata(&tenant, &project, &artifact)
                                 .is_some_and(|metadata| {
-                                metadata.tenant == tenant
-                                    && metadata.project == project
-                                    && metadata.digest == object.digest
+                                    metadata.digest == object.digest
                                         && metadata.size == object.size
                                         && !metadata.retaining_nodes.is_empty()
-                            })
-                    });
+                                }),
+                    )
+                },
+            )?;
             if !vfs_available {
                 "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
             } else {
@@ -475,6 +480,7 @@ impl CoordinatorService {
         }
         self.record_task_completion_event(event.clone());
         self.notify_coordinator_main_waiters(&event);
+        self.maybe_retire_terminal_process(&tenant, &project, &process)?;
         Ok(CoordinatorResponse::TaskFailureResolved {
             process,
             task,

crates/clusterflux-coordinator/src/service/keys.rs

@@ -1,6 +1,7 @@
 use clusterflux_core::{
     ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
 };
+use thiserror::Error;
 
 pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
 pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
     (tenant.clone(), project.clone(), grant.to_owned())
 }
 
-pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub(super) enum ArtifactPathError {
+    #[error("path must start with the exact /vfs/artifacts/ prefix")]
+    WrongPrefix,
+    #[error("path must name an artifact after /vfs/artifacts/")]
+    EmptyArtifact,
+    #[error("mapped artifact identifier is invalid: {0}")]
+    InvalidArtifactId(#[from] clusterflux_core::IdParseError),
+}
+
+pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
     let value = path
         .as_str()
         .strip_prefix("/vfs/artifacts/")
-        .unwrap_or(path.as_str())
-        .replace('/', ":");
-    ArtifactId::new(value)
+        .ok_or(ArtifactPathError::WrongPrefix)?;
+    if value.is_empty() {
+        return Err(ArtifactPathError::EmptyArtifact);
+    }
+    ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn artifact_id_conversion_is_fallible_and_structural() {
+        assert_eq!(
+            artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
+            ArtifactId::from("build:output")
+        );
+        for path in [
+            "/vfs/other/output",
+            "/vfs/artifacts",
+            "/vfs/artifacts/bad artifact!",
+        ] {
+            let path = VfsPath::new(path).unwrap();
+            assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
+        }
+
+        let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
+        assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
+    }
 }

crates/clusterflux-coordinator/src/service/logs.rs

@@ -1,5 +1,5 @@
 use clusterflux_core::{
-    ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
+    ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
     TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
 };
 
@@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key};
 use super::protocol::TaskAttemptState;
 use super::{
     artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
-    TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
+    RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState,
+    MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES,
+    MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT,
+    MAX_TASK_LOG_TAIL_BYTES,
 };
 
 impl CoordinatorService {
@@ -36,23 +39,44 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         validate_task_log_tail("stdout_tail", &stdout_tail)?;
         validate_task_log_tail("stderr_tail", &stderr_tail)?;
-        let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota
-            .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
-        self.quota
-            .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+        let stdout_retained = self.accept_final_log_stream(
+            &tenant,
+            &project,
+            &process,
+            &task,
+            TaskLogStream::Stdout,
+            stdout_bytes,
+            &stdout_tail,
+            stdout_truncated,
+            now_epoch_seconds,
+        )?;
+        let stderr_retained = self.accept_final_log_stream(
+            &tenant,
+            &project,
+            &process,
+            &task,
+            TaskLogStream::Stderr,
+            stderr_bytes,
+            &stderr_tail,
+            stderr_truncated,
+            now_epoch_seconds,
+        )?;
         Ok(CoordinatorResponse::TaskLogRecorded {
             process,
             task,
             stdout_bytes,
             stderr_bytes,
-            stdout_tail: if stdout_truncated {
+            stdout_tail: if !stdout_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stdout_truncated {
                 format!("{stdout_tail}\n... truncated")
             } else {
                 stdout_tail
             },
-            stderr_tail: if stderr_truncated {
+            stderr_tail: if !stderr_retained {
+                "[log output truncated at project log quota]".to_owned()
+            } else if stderr_truncated {
                 format!("{stderr_tail}\n... truncated")
             } else {
                 stderr_tail
@@ -61,6 +85,158 @@ impl CoordinatorService {
         })
     }
 
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_report_task_log_chunk(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stream: TaskLogStream,
+        offset: u64,
+        source_bytes: u64,
+        text: String,
+        truncated: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
+                "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}",
+                text.len()
+            )));
+        }
+        if source_bytes == 0 && !text.is_empty() && !truncated {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes must describe non-empty text".to_owned(),
+            ));
+        }
+        if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) {
+            return Err(CoordinatorServiceError::Protocol(
+                "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(),
+            ));
+        }
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
+        let expected = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let end = offset.checked_add(source_bytes).ok_or_else(|| {
+            CoordinatorServiceError::Protocol(
+                "live log chunk offset exceeds the supported range".to_owned(),
+            )
+        })?;
+        let state_marker = source_bytes == 0 && truncated;
+        if end < expected || (end == expected && !state_marker) {
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: expected,
+            });
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key, end);
+            }
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence: None,
+                next_offset: end.max(expected),
+            });
+        }
+        let newly_accounted = end.saturating_sub(expected);
+        if self
+            .quota
+            .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)
+            .is_err()
+        {
+            if end > expected {
+                self.recent_log_accounted_bytes.insert(key.clone(), end);
+            }
+            let sequence = self.mark_log_quota_truncated(
+                &tenant,
+                &project,
+                &process,
+                &task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(CoordinatorResponse::TaskLogChunkRecorded {
+                process,
+                task,
+                sequence,
+                next_offset: end.max(expected),
+            });
+        }
+        if offset > expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!("[log output lost: {} bytes]", offset - expected),
+                true,
+                now_epoch_seconds,
+            );
+        } else if offset < expected {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream.clone(),
+                format!(
+                    "[log output overlap omitted: {} new source bytes]",
+                    end - expected
+                ),
+                true,
+                now_epoch_seconds,
+            );
+        }
+        let marker_is_new = if truncated {
+            self.recent_log_truncated_streams.insert(key.clone())
+        } else {
+            false
+        };
+        let text = if state_marker && text.is_empty() {
+            "[log output truncated at source]".to_owned()
+        } else {
+            text
+        };
+        let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
+            .then(|| {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream,
+                    text,
+                    truncated,
+                    now_epoch_seconds,
+                )
+            });
+        if end > expected {
+            self.recent_log_accounted_bytes.insert(key, end);
+        }
+        Ok(CoordinatorResponse::TaskLogChunkRecorded {
+            process,
+            task,
+            sequence,
+            next_offset: end,
+        })
+    }
+
     pub(super) fn handle_report_vfs_metadata(
         &mut self,
         tenant: String,
@@ -85,7 +261,9 @@ impl CoordinatorService {
         self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
         if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant,
                 project,
                 process: process.clone(),
@@ -177,20 +355,37 @@ impl CoordinatorService {
             artifact_size_bytes,
             result,
         };
-        let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
         let now_epoch_seconds = self.current_epoch_seconds()?;
-        self.quota.can_charge_log_bytes(
+        let stdout_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stdout,
+            event.stdout_bytes,
+            &event.stdout_tail,
+            event.stdout_truncated,
             now_epoch_seconds,
         )?;
-        self.quota.charge_log_bytes(
+        let stderr_retained = self.accept_final_log_stream(
             &event.tenant,
             &event.project,
-            reported_bytes,
+            &event.process,
+            &event.task,
+            TaskLogStream::Stderr,
+            event.stderr_bytes,
+            &event.stderr_tail,
+            event.stderr_truncated,
             now_epoch_seconds,
         )?;
+        if !stdout_retained {
+            event.stdout_tail = "[log output truncated at project log quota]".to_owned();
+            event.stdout_truncated = true;
+        }
+        if !stderr_retained {
+            event.stderr_tail = "[log output truncated at project log quota]".to_owned();
+            event.stderr_truncated = true;
+        }
         let task_key = task_control_key(
             &event.tenant,
             &event.project,
@@ -203,7 +398,9 @@ impl CoordinatorService {
         event.placement = self.task_placements.remove(&task_key);
         if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
             self.flush_artifact_metadata(ArtifactFlush {
-                id: artifact_id_from_path(path),
+                id: artifact_id_from_path(path).map_err(|error| {
+                    CoordinatorServiceError::InvalidArtifactPath(error.to_string())
+                })?,
                 tenant: event.tenant.clone(),
                 project: event.project.clone(),
                 process: event.process.clone(),
@@ -217,26 +414,22 @@ impl CoordinatorService {
         self.task_aborts.remove(&task_key);
         self.debug_commands.remove(&task_key);
         self.active_tasks.remove(&task_key);
-        let no_active_tasks =
-            !self
-                .active_tasks
-                .iter()
-                .any(|(task_tenant, task_project, task_process, _, _)| {
-                    task_tenant == &event.tenant
-                        && task_project == &event.project
-                        && task_process == &event.process
+        self.clear_recent_log_offsets_for_task(
+            &event.tenant,
+            &event.project,
+            &event.process,
+            &event.task,
+        );
+        self.task_assignments.retain(|_, assignments| {
+            assignments.retain(|assignment| {
+                assignment.tenant != event.tenant
+                    || assignment.project != event.project
+                    || assignment.process != event.process
+                    || assignment.node != event.node
+                    || assignment.task != event.task
+            });
+            !assignments.is_empty()
         });
-        if no_active_tasks {
-            self.process_aborts.remove(&process_key);
-            if self.process_cancellations.remove(&process_key)
-                && !self.main_runtime.controls.contains_key(&process_key)
-            {
-                self.coordinator
-                    .abort_process(&event.tenant, &event.project, &event.process)?;
-                self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
-                self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
-            }
-        }
         if process_was_aborted {
             let checkpoint_key = super::keys::task_restart_key(
                 &event.tenant,
@@ -253,10 +446,20 @@ impl CoordinatorService {
         if !awaiting_operator {
             self.notify_coordinator_main_waiters(&event);
         }
+        self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
+        let events_recorded = self
+            .task_events
+            .iter()
+            .filter(|recorded| {
+                recorded.tenant == event.tenant
+                    && recorded.project == event.project
+                    && recorded.process == event.process
+            })
+            .count();
         Ok(CoordinatorResponse::TaskRecorded {
             process: event.process,
             task: event.task,
-            events_recorded: self.task_events.len(),
+            events_recorded,
         })
     }
 
@@ -316,7 +519,7 @@ impl CoordinatorService {
         Ok(CoordinatorResponse::TaskSnapshots { snapshots })
     }
 
-    fn authorize_task_event_process_scope(
+    pub(super) fn authorize_task_event_process_scope(
         &self,
         tenant: &TenantId,
         project: &ProjectId,
@@ -356,6 +559,47 @@ impl CoordinatorService {
         Ok(())
     }
 
+    pub(super) fn handle_list_recent_logs(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: Option<String>,
+        after_sequence: Option<u64>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = task.map(TaskInstanceId::new);
+        self.authorize_task_event_process_scope(&tenant, &project, &process)?;
+        let after_sequence = after_sequence.unwrap_or(0);
+        let retained = self.recent_logs.get(&(tenant.clone(), project.clone()));
+        let history_truncated = self
+            .recent_log_dropped_through
+            .get(&process_control_key(&tenant, &project, &process))
+            .is_some_and(|dropped_through| *dropped_through > after_sequence);
+        let entries = retained
+            .into_iter()
+            .flatten()
+            .filter(|entry| {
+                entry.process == process
+                    && entry.sequence > after_sequence
+                    && task.as_ref().is_none_or(|task| &entry.task == task)
+            })
+            .take(limit as usize)
+            .cloned()
+            .collect::<Vec<_>>();
+        let next_sequence = entries.last().map(|entry| entry.sequence);
+        Ok(CoordinatorResponse::RecentLogs {
+            entries,
+            next_sequence,
+            history_truncated,
+        })
+    }
+
     pub(super) fn handle_join_task(
         &mut self,
         tenant: String,
@@ -474,6 +718,22 @@ impl CoordinatorService {
     pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
         event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
         event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
+        match event.executor {
+            super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
+                &event.tenant,
+                &event.project,
+                &event.process,
+                event.task_definition.clone(),
+                event.task.clone(),
+                event.terminal_state.clone(),
+            ),
+            super::TaskExecutor::Node => {
+                self.task_terminal_states.insert(
+                    task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
+                    event.terminal_state.clone(),
+                );
+            }
+        }
         let process_scope = (
             event.tenant.clone(),
             event.project.clone(),
@@ -511,6 +771,321 @@ impl CoordinatorService {
         self.task_events.push_back(event);
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn record_recent_log(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        task: TaskInstanceId,
+        stream: TaskLogStream,
+        mut text: String,
+        mut truncated: bool,
+        server_timestamp_epoch_seconds: u64,
+    ) -> u64 {
+        if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
+            let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES;
+            while !text.is_char_boundary(boundary) {
+                boundary -= 1;
+            }
+            text.truncate(boundary);
+            truncated = true;
+        }
+        let sequence = self.next_recent_log_sequence;
+        self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1);
+        let logs = self
+            .recent_logs
+            .entry((tenant.clone(), project.clone()))
+            .or_default();
+        let mut dropped = Vec::new();
+        while logs.iter().filter(|entry| entry.process == process).count()
+            >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS
+        {
+            let Some(index) = logs.iter().position(|entry| entry.process == process) else {
+                break;
+            };
+            if let Some(entry) = logs.remove(index) {
+                dropped.push(entry);
+            }
+        }
+        while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT
+            || logs
+                .iter()
+                .map(|entry| entry.text.len())
+                .sum::<usize>()
+                .saturating_add(text.len())
+                > MAX_RECENT_LOG_BYTES_PER_PROJECT
+        {
+            match logs.pop_front() {
+                Some(entry) => dropped.push(entry),
+                None => break,
+            }
+        }
+        logs.push_back(RecentLogEntry {
+            sequence,
+            process,
+            task,
+            stream,
+            text,
+            server_timestamp_epoch_seconds,
+            truncated,
+        });
+        for entry in dropped {
+            let key = process_control_key(&tenant, &project, &entry.process);
+            self.recent_log_dropped_through
+                .entry(key)
+                .and_modify(|dropped_through| {
+                    *dropped_through = (*dropped_through).max(entry.sequence);
+                })
+                .or_insert(entry.sequence);
+        }
+        sequence
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn accept_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| {
+            let stream_name = match stream {
+                TaskLogStream::Stdout => "stdout",
+                TaskLogStream::Stderr => "stderr",
+            };
+            CoordinatorServiceError::Protocol(format!(
+                "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted"
+            ))
+        })?;
+        if self.recent_log_quota_truncated_streams.contains(&key) {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            return Ok(false);
+        }
+        if self
+            .quota
+            .charge_log_bytes(tenant, project, remaining, now_epoch_seconds)
+            .is_err()
+        {
+            self.recent_log_accounted_bytes
+                .insert(key, total_source_bytes);
+            self.mark_log_quota_truncated(
+                tenant,
+                project,
+                process,
+                task,
+                &stream,
+                now_epoch_seconds,
+            );
+            return Ok(false);
+        }
+        self.reconcile_final_log_stream(
+            tenant,
+            project,
+            process,
+            task,
+            stream,
+            total_source_bytes,
+            final_tail,
+            source_truncated,
+            now_epoch_seconds,
+        );
+        Ok(true)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn mark_log_quota_truncated(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: &TaskLogStream,
+        now_epoch_seconds: u64,
+    ) -> Option<u64> {
+        let key = recent_log_offset_key(tenant, project, process, task, stream);
+        if !self.recent_log_quota_truncated_streams.insert(key.clone()) {
+            return None;
+        }
+        self.recent_log_truncated_streams.insert(key);
+        Some(self.record_recent_log(
+            tenant.clone(),
+            project.clone(),
+            process.clone(),
+            task.clone(),
+            stream.clone(),
+            "[log output truncated at project log quota]".to_owned(),
+            true,
+            now_epoch_seconds,
+        ))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn reconcile_final_log_stream(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+        stream: TaskLogStream,
+        total_source_bytes: u64,
+        final_tail: &str,
+        source_truncated: bool,
+        now_epoch_seconds: u64,
+    ) {
+        let key = recent_log_offset_key(tenant, project, process, task, &stream);
+        let accounted = self
+            .recent_log_accounted_bytes
+            .get(&key)
+            .copied()
+            .unwrap_or(0);
+        let mut visible_truncation = false;
+        if total_source_bytes > accounted {
+            let missing = total_source_bytes - accounted;
+            if final_tail.is_empty() {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!("[log output unavailable: {missing} source bytes]"),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            } else if (final_tail.len() as u64) <= total_source_bytes {
+                let tail_source_start = total_source_bytes - final_tail.len() as u64;
+                if accounted < tail_source_start {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        format!(
+                            "[log output lost before final tail: {} source bytes]",
+                            tail_source_start - accounted
+                        ),
+                        true,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation = true;
+                }
+                let source_start = accounted.max(tail_source_start) - tail_source_start;
+                let mut byte_start = usize::try_from(source_start)
+                    .unwrap_or(final_tail.len())
+                    .min(final_tail.len());
+                while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
+                    byte_start += 1;
+                }
+                let suffix = &final_tail[byte_start..];
+                if !suffix.is_empty() {
+                    self.record_recent_log(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        task.clone(),
+                        stream.clone(),
+                        suffix.to_owned(),
+                        source_truncated || visible_truncation,
+                        now_epoch_seconds,
+                    );
+                    visible_truncation |= source_truncated;
+                }
+            } else if accounted == 0 {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    final_tail.to_owned(),
+                    source_truncated,
+                    now_epoch_seconds,
+                );
+                visible_truncation |= source_truncated;
+            } else {
+                self.record_recent_log(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    task.clone(),
+                    stream.clone(),
+                    format!(
+                        "[{missing} additional source bytes could not be merged without duplicating redacted output]"
+                    ),
+                    true,
+                    now_epoch_seconds,
+                );
+                visible_truncation = true;
+            }
+            self.recent_log_accounted_bytes
+                .insert(key.clone(), total_source_bytes);
+        }
+
+        let marker_is_new = (source_truncated || visible_truncation)
+            && self.recent_log_truncated_streams.insert(key);
+        if marker_is_new && !visible_truncation {
+            self.record_recent_log(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                task.clone(),
+                stream,
+                "[log output truncated at source]".to_owned(),
+                true,
+                now_epoch_seconds,
+            );
+        }
+    }
+
+    pub(super) fn clear_recent_log_offsets_for_task(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+    ) {
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _), _| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, entry_task, _)| {
+                entry_tenant != tenant
+                    || entry_project != project
+                    || entry_process != process
+                    || entry_task != task
+            },
+        );
+    }
+
     fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
         let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
         let Some(attempt) = self
@@ -545,6 +1120,118 @@ impl CoordinatorService {
         awaiting_operator
     }
 
+    pub(super) fn maybe_retire_terminal_process(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<bool, CoordinatorServiceError> {
+        let process_key = process_control_key(tenant, project, process);
+        if self.main_runtime.controls.contains_key(&process_key) {
+            return Ok(false);
+        }
+
+        let has_runnable_remote_work =
+            self.active_tasks
+                .iter()
+                .any(|(task_tenant, task_project, task_process, _, _)| {
+                    task_tenant == tenant && task_project == project && task_process == process
+                })
+                || self.pending_task_launches.iter().any(|pending| {
+                    &pending.tenant == tenant
+                        && &pending.project == project
+                        && &pending.process == process
+                })
+                || self.task_assignments.values().any(|assignments| {
+                    assignments.iter().any(|assignment| {
+                        &assignment.tenant == tenant
+                            && &assignment.project == project
+                            && &assignment.process == process
+                    })
+                })
+                || self.task_attempts.iter().any(
+                    |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                        attempt_tenant == tenant
+                            && attempt_project == project
+                            && attempt_process == process
+                            && attempts.iter().rev().any(|attempt| {
+                                attempt.current
+                                    && matches!(
+                                        attempt.state,
+                                        TaskAttemptState::Queued
+                                            | TaskAttemptState::Running
+                                            | TaskAttemptState::FailedAwaitingAction
+                                    )
+                            })
+                    },
+                );
+        if has_runnable_remote_work {
+            return Ok(false);
+        }
+
+        let main_completed = self
+            .process_summaries
+            .get(&process_key)
+            .and_then(|summary| summary.main_terminal_state.as_ref())
+            .is_some_and(|state| matches!(state, TaskTerminalState::Completed));
+        let cancellation_completed = self.process_cancellations.contains(&process_key);
+        if !main_completed && !cancellation_completed {
+            return Ok(false);
+        }
+
+        self.process_aborts.remove(&process_key);
+        self.process_cancellations.remove(&process_key);
+        if self
+            .coordinator
+            .active_process(tenant, project, process)
+            .is_none()
+        {
+            return Ok(false);
+        }
+        let final_result = if cancellation_completed {
+            super::ProcessFinalResult::Cancelled
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Failed
+            },
+        ) {
+            super::ProcessFinalResult::Failed
+        } else if self.task_terminal_states.iter().any(
+            |((task_tenant, task_project, task_process, _), terminal_state)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && terminal_state == &TaskTerminalState::Cancelled
+            },
+        ) {
+            super::ProcessFinalResult::Cancelled
+        } else {
+            super::ProcessFinalResult::Completed
+        };
+        self.record_process_terminal(
+            tenant,
+            project,
+            process,
+            final_result,
+            self.current_epoch_seconds()?,
+        );
+        self.coordinator.abort_process(tenant, project, process)?;
+        self.clear_debug_state_for_process(tenant, project, process);
+        self.clear_operator_panel_state(tenant, project, process);
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(tenant, project);
+        self.artifact_registry.enforce_project_metadata_limit(
+            tenant,
+            project,
+            &pinned,
+            &protected_processes,
+        );
+        Ok(true)
+    }
+
     fn flush_artifact_metadata(
         &mut self,
         flush: ArtifactFlush,
@@ -552,23 +1239,59 @@ impl CoordinatorService {
         let now_epoch_seconds = self.current_epoch_seconds()?;
         self.artifact_registry
             .expire_download_links(now_epoch_seconds);
-        let pinned = self
-            .task_restart_checkpoints
-            .values()
-            .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
-            .chain(
-                self.pending_task_launches
-                    .iter()
-                    .flat_map(|pending| pending.task_spec.required_artifacts.iter()),
-            )
-            .cloned()
-            .collect::<std::collections::BTreeSet<ArtifactId>>();
+        let tenant = flush.tenant.clone();
+        let project = flush.project.clone();
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(&tenant, &project);
         self.artifact_registry
-            .flush_metadata_bounded(flush, &pinned)
+            .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
             .map(|_| ())
             .map_err(CoordinatorServiceError::Protocol)
     }
 
+    fn artifact_retention_guards_for_project(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> (
+        std::collections::BTreeSet<ArtifactScopeKey>,
+        std::collections::BTreeSet<ProcessId>,
+    ) {
+        let mut pinned = std::collections::BTreeSet::new();
+        for checkpoint in self.task_restart_checkpoints.values() {
+            if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
+            {
+                continue;
+            }
+            for artifact in &checkpoint.assignment.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &checkpoint.assignment.tenant,
+                    &checkpoint.assignment.project,
+                    artifact,
+                ));
+            }
+        }
+        for pending in &self.pending_task_launches {
+            if &pending.tenant != tenant || &pending.project != project {
+                continue;
+            }
+            for artifact in &pending.task_spec.required_artifacts {
+                pinned.insert(ArtifactScopeKey::from_refs(
+                    &pending.tenant,
+                    &pending.project,
+                    artifact,
+                ));
+            }
+        }
+        let protected_processes = self
+            .coordinator
+            .active_processes_for_project(tenant, project)
+            .into_iter()
+            .map(|process| process.id)
+            .collect();
+        (pinned, protected_processes)
+    }
+
     fn task_is_known_or_active(
         &self,
         tenant: &TenantId,
@@ -601,15 +1324,24 @@ impl CoordinatorService {
     }
 }
 
-fn checked_reported_log_bytes(
-    stdout_bytes: u64,
-    stderr_bytes: u64,
-) -> Result<u64, CoordinatorServiceError> {
-    stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
-        CoordinatorServiceError::Protocol(
-            "reported task log byte counts exceed the supported range".to_owned(),
+fn recent_log_offset_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+    task: &TaskInstanceId,
+    stream: &TaskLogStream,
+) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        task.clone(),
+        match stream {
+            TaskLogStream::Stdout => "stdout",
+            TaskLogStream::Stderr => "stderr",
+        }
+        .to_owned(),
     )
-    })
 }
 
 fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
@@ -626,11 +1358,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
     if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
         return value;
     }
-    let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
-    while !value.is_char_boundary(boundary) {
-        boundary -= 1;
+    let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES;
+    while boundary < value.len() && !value.is_char_boundary(boundary) {
+        boundary += 1;
     }
-    value.truncate(boundary);
+    value.drain(..boundary);
     *truncated = true;
     value
 }

crates/clusterflux-coordinator/src/service/main_runtime.rs

@@ -30,14 +30,14 @@ use super::{
 };
 
 #[derive(Clone)]
-struct MainScope {
-    tenant: TenantId,
-    project: ProjectId,
-    process: ProcessId,
-    task_definition: TaskDefinitionId,
-    task_instance: TaskInstanceId,
-    epoch: u64,
-    launch_id: u64,
+pub(super) struct MainScope {
+    pub(super) tenant: TenantId,
+    pub(super) project: ProjectId,
+    pub(super) process: ProcessId,
+    pub(super) task_definition: TaskDefinitionId,
+    pub(super) task_instance: TaskInstanceId,
+    pub(super) epoch: u64,
+    pub(super) launch_id: u64,
 }
 
 enum MainCommand {
@@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime {
 }
 
 impl CoordinatorMainRuntime {
+    pub(super) fn active_main_count(&self) -> usize {
+        self.controls.len()
+    }
+
+    pub(super) fn max_active_mains(&self) -> usize {
+        self.max_active_mains
+    }
+
     pub(super) fn configure(
         &mut self,
         configuration: super::CoordinatorMainRuntimeConfiguration,
@@ -729,6 +737,13 @@ impl CoordinatorService {
                 &process,
                 "coordinator main launch failed admission or validation",
             );
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Failed,
+                self.liveness_now_epoch_seconds(),
+            );
             let _ = self.coordinator.abort_process(&tenant, &project, &process);
         }
         result
@@ -889,7 +904,7 @@ impl CoordinatorService {
         }
     }
 
-    fn record_coordinator_main_completion(
+    pub(super) fn record_coordinator_main_completion(
         &mut self,
         scope: MainScope,
         result: Result<WasmTaskResult, String>,
@@ -929,6 +944,7 @@ impl CoordinatorService {
             ),
             Err(error) => (TaskTerminalState::Failed, None, error),
         };
+        let main_completed = matches!(terminal_state, TaskTerminalState::Completed);
         let main_state = match terminal_state {
             TaskTerminalState::Completed => "completed",
             TaskTerminalState::Failed => "failed",
@@ -970,6 +986,12 @@ impl CoordinatorService {
             }
         }
         let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
+        self.main_runtime.controls.remove(&process_key);
+        if main_completed {
+            let _ =
+                self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
+            return;
+        }
         for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
             if task_tenant == scope.tenant
                 && task_project == scope.project
@@ -985,10 +1007,16 @@ impl CoordinatorService {
             }
         }
         self.process_aborts.insert(process_key.clone());
+        self.record_process_terminal(
+            &scope.tenant,
+            &scope.project,
+            &scope.process,
+            super::ProcessFinalResult::Failed,
+            self.liveness_now_epoch_seconds(),
+        );
         let _ = self
             .coordinator
             .abort_process(&scope.tenant, &scope.project, &scope.process);
-        self.main_runtime.controls.remove(&process_key);
         self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process);
         self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process);
     }
@@ -1157,6 +1185,120 @@ mod tests {
         assert!(!service.main_runtime.controls.contains_key(&process_key));
         assert!(!service.debug_epochs.contains_key(&process_key));
         assert!(!service.debug_epoch_runtime.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+    }
+
+    #[test]
+    fn completed_main_keeps_process_and_debug_state_for_active_children() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-current");
+        let main_task = TaskInstanceId::from("ti:vp-current:main");
+        let child_task = TaskInstanceId::from("ti:vp-current:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task.clone(),
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(
+            scope,
+            Ok(WasmTaskResult::completed(
+                main_task,
+                TaskBoundaryValue::SmallJson(serde_json::Value::Null),
+            )),
+        );
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_some());
+        assert!(!service.main_runtime.controls.contains_key(&process_key));
+        assert!(service.debug_epochs.contains_key(&process_key));
+        assert!(!service.process_aborts.contains(&process_key));
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(!service.task_aborts.contains(&child_key));
+    }
+
+    #[test]
+    fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() {
+        let mut service = CoordinatorService::new(7);
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let process = ProcessId::from("vp-failed-main");
+        let main_task = TaskInstanceId::from("ti:vp-failed-main:main");
+        let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1");
+        let child_node = NodeId::from("worker");
+        let scope = MainScope {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task_definition: TaskDefinitionId::from("build"),
+            task_instance: main_task.clone(),
+            epoch: 7,
+            launch_id: 1,
+        };
+        service
+            .coordinator
+            .start_process(tenant.clone(), project.clone(), process.clone());
+        let process_key = process_control_key(&tenant, &project, &process);
+        service.main_runtime.controls.insert(
+            process_key.clone(),
+            CoordinatorMainControl {
+                task_definition: scope.task_definition.clone(),
+                task_instance: main_task,
+                abort: Arc::new(AtomicBool::new(false)),
+                debug: Arc::new(WasmDebugControl::default()),
+                state: "running".to_owned(),
+                stopped_probe_symbol: None,
+                handles: Arc::new(Mutex::new(HashMap::new())),
+                launch_id: 1,
+            },
+        );
+        let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
+        service.active_tasks.insert(child_key.clone());
+        service.debug_epochs.insert(process_key.clone(), 2);
+
+        service.record_coordinator_main_completion(scope, Err("main crashed".to_owned()));
+
+        assert!(service
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .is_none());
+        assert!(service.active_tasks.contains(&child_key));
+        assert!(service.task_aborts.contains(&child_key));
         assert!(service.process_aborts.contains(&process_key));
+        assert!(!service.debug_epochs.contains_key(&process_key));
+        assert!(service.task_events.iter().any(|event| {
+            event.process == process
+                && event.executor == TaskExecutor::CoordinatorMain
+                && event.terminal_state == TaskTerminalState::Failed
+        }));
     }
 }

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     SourceProviderKind, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{
     bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@@ -27,9 +27,9 @@ impl CoordinatorService {
         unix_timestamp_seconds()
     }
 
-    pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
+    pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
         self.node_last_seen_epoch_seconds
-            .get(node)
+            .get(scope)
             .is_some_and(|last_seen| {
                 self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
                     <= self.node_stale_after_seconds
@@ -41,7 +41,11 @@ impl CoordinatorService {
             .values()
             .cloned()
             .map(|mut descriptor| {
-                descriptor.online = self.node_is_live(&descriptor.id);
+                descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
+                    &descriptor.tenant,
+                    &descriptor.project,
+                    &descriptor.id,
+                ));
                 descriptor
             })
             .collect()
@@ -165,7 +169,11 @@ impl CoordinatorService {
             !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
         });
         self.coordinator.ensure_tenant_active(&tenant)?;
-        if self.coordinator.node_identity(&node).is_none() {
+        if self
+            .coordinator
+            .node_identity(&tenant, &project, &node)
+            .is_none()
+        {
             self.quota.ensure_node_admission(
                 &tenant,
                 self.coordinator.node_identity_count_for_tenant(&tenant),
@@ -197,12 +205,21 @@ impl CoordinatorService {
 
     pub(super) fn handle_node_heartbeat(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         node_signature: Option<NodeSignedRequest>,
         payload_digest: &Digest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
-        self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
+        self.authenticate_node_request(
+            &NodeScopeKey::new(tenant, project, node.clone()),
+            node_signature,
+            "node_heartbeat",
+            payload_digest,
+        )?;
         Ok(CoordinatorResponse::NodeHeartbeat {
             node,
             epoch: self.coordinator.coordinator_epoch(),
@@ -225,16 +242,13 @@ impl CoordinatorService {
         let tenant = TenantId::new(tenant);
         let project = ProjectId::new(project);
         let node = NodeId::new(node);
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "node capability report is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
+        debug_assert_eq!(identity.tenant, tenant);
+        debug_assert_eq!(identity.project, project);
         capabilities.validate_public_report()?;
         for (kind, count) in [
             ("cached environments", cached_environment_digests.len()),
@@ -274,16 +288,20 @@ impl CoordinatorService {
             .into_iter()
             .map(ArtifactId::new)
             .collect::<BTreeSet<_>>();
-        self.artifact_registry
-            .reconcile_node_retention(&node, &artifact_locations);
+        self.artifact_registry.reconcile_node_retention(
+            &tenant,
+            &project,
+            &node,
+            &artifact_locations,
+        );
 
-        let online = self.node_is_live(&node);
+        let online = self.node_is_live(&node_scope);
         self.node_descriptors.insert(
-            node.clone(),
+            node_scope,
             NodeDescriptor {
                 id: node.clone(),
-                tenant,
-                project,
+                tenant: tenant.clone(),
+                project: project.clone(),
                 capabilities,
                 cached_environments: cached_environment_digests.into_iter().collect(),
                 dependency_caches: dependency_cache_digests.into_iter().collect(),
@@ -293,9 +311,14 @@ impl CoordinatorService {
                 online,
             },
         );
+        let node_descriptors = self
+            .node_descriptors
+            .values()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .count();
         Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
             node,
-            node_descriptors: self.node_descriptors.len(),
+            node_descriptors,
         })
     }
 
@@ -333,9 +356,13 @@ impl CoordinatorService {
             actor: Actor::User(actor.clone()),
         };
         self.coordinator.revoke_node_credential(&context, &node)?;
-        let descriptor_removed = self.node_descriptors.remove(&node).is_some();
-        self.node_last_seen_epoch_seconds.remove(&node);
-        self.artifact_registry.garbage_collect_node(&node);
+        let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
+        let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
+        self.node_last_seen_epoch_seconds.remove(&node_scope);
+        self.node_replay_nonces
+            .retain(|(retained_scope, _), _| retained_scope != &node_scope);
+        self.artifact_registry
+            .garbage_collect_node(&tenant, &project, &node);
         let queued_assignments_removed = self
             .task_assignments
             .remove(&(tenant.clone(), project.clone(), node.clone()))
@@ -369,14 +396,14 @@ impl CoordinatorService {
 
     pub(super) fn authenticate_node_request(
         &mut self,
-        node: &NodeId,
+        scope: &NodeScopeKey,
         node_signature: Option<NodeSignedRequest>,
         request_kind: &str,
         payload_digest: &Digest,
     ) -> Result<(), CoordinatorServiceError> {
         let identity = self
             .coordinator
-            .node_identity(node)
+            .node_identity(&scope.tenant, &scope.project, &scope.node)
             .ok_or(CoordinatorError::UnknownNode)?;
         let signature = node_signature.ok_or_else(|| {
             CoordinatorError::Unauthorized(
@@ -401,7 +428,7 @@ impl CoordinatorService {
             )
             .into());
         }
-        let replay_key = (node.clone(), signature.nonce.clone());
+        let replay_key = (scope.clone(), signature.nonce.clone());
         self.node_replay_nonces.retain(|_, accepted_at| {
             now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
         });
@@ -413,7 +440,7 @@ impl CoordinatorService {
         }
         verify_node_request_signature(
             &identity.public_key,
-            node,
+            &scope.node,
             request_kind,
             payload_digest,
             &signature,
@@ -422,7 +449,7 @@ impl CoordinatorService {
         if self
             .node_replay_nonces
             .keys()
-            .filter(|(retained_node, _)| retained_node == node)
+            .filter(|(retained_scope, _)| retained_scope == scope)
             .count()
             >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
         {
@@ -436,8 +463,8 @@ impl CoordinatorService {
             .insert(replay_key, now_epoch_seconds);
         let seen_at = self.liveness_now_epoch_seconds();
         self.node_last_seen_epoch_seconds
-            .insert(node.clone(), seen_at);
-        if let Some(descriptor) = self.node_descriptors.get_mut(node) {
+            .insert(scope.clone(), seen_at);
+        if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
             descriptor.online = true;
         }
         Ok(())

crates/clusterflux-coordinator/src/service/panels.rs

@@ -254,7 +254,8 @@ impl CoordinatorService {
             .rev()
             .find_map(|event| event.artifact_path.as_ref())
         {
-            let artifact = artifact_id_from_path(path);
+            let artifact = artifact_id_from_path(path)
+                .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
             let context = clusterflux_core::AuthContext {
                 tenant,
                 project,

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -118,14 +118,14 @@ impl CoordinatorService {
         let mut objects = BTreeMap::new();
         let mut missing_required_artifact = false;
         for artifact in &task_spec.required_artifacts {
-            let Some(metadata) = self.artifact_registry.metadata(artifact) else {
+            let Some(metadata) =
+                self.artifact_registry
+                    .metadata(&assignment.tenant, &assignment.project, artifact)
+            else {
                 missing_required_artifact = true;
                 continue;
             };
-            if metadata.tenant != assignment.tenant
-                || metadata.project != assignment.project
-                || metadata.retaining_nodes.is_empty()
-            {
+            if metadata.retaining_nodes.is_empty() {
                 missing_required_artifact = true;
                 continue;
             }
@@ -483,17 +483,14 @@ impl CoordinatorService {
         .validate()
         .map_err(CoordinatorServiceError::Protocol)?;
         for artifact in &task_spec.required_artifacts {
-            let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
+            let metadata = self
+                .artifact_registry
+                .metadata(&tenant, &project, artifact)
+                .ok_or_else(|| {
                     CoordinatorError::Unauthorized(format!(
-                    "required artifact {artifact} is unavailable or has expired"
+                        "required artifact {artifact} is unavailable or has expired in this tenant/project scope"
                     ))
                 })?;
-            if metadata.tenant != tenant || metadata.project != project {
-                return Err(CoordinatorError::Unauthorized(format!(
-                    "required artifact {artifact} is outside the task tenant/project scope"
-                ))
-                .into());
-            }
             if metadata.retaining_nodes.is_empty() {
                 return Err(CoordinatorError::Unauthorized(format!(
                     "required artifact {artifact} has no retaining node"
@@ -548,13 +545,22 @@ impl CoordinatorService {
                     task_spec,
                     wasm_module_base64,
                 });
+                let queued_tasks = self
+                    .pending_task_launches
+                    .iter()
+                    .filter(|pending| {
+                        pending.tenant == tenant
+                            && pending.project == project
+                            && pending.process == process
+                    })
+                    .count();
                 return Ok(CoordinatorResponse::TaskQueued {
                     process,
                     task,
                     actor,
                     reason,
                     charged_spawns,
-                    queued_tasks: self.pending_task_launches.len(),
+                    queued_tasks,
                 });
             }
             Err(err) => return Err(err.into()),
@@ -665,7 +671,9 @@ impl CoordinatorService {
                 ));
             };
             self.task_attempts.remove(&removable);
+            self.task_terminal_states.remove(&removable);
         }
+        self.task_terminal_states.remove(&key);
         let attempts = self.task_attempts.entry(key).or_default();
         for attempt in attempts.iter_mut() {
             attempt.current = false;

crates/clusterflux-coordinator/src/service/processes.rs

@@ -7,7 +7,7 @@ use clusterflux_core::{
     TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
 };
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::keys::{process_control_key, task_control_key};
 use super::{
@@ -47,14 +47,10 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "task assignment poll is outside the enrolled tenant/project scope".to_owned(),
-            )
-            .into());
-        }
+        debug_assert_eq!(identity.tenant, tenant);
+        debug_assert_eq!(identity.project, project);
         let assignment_key = (tenant.clone(), project.clone(), node.clone());
         let assignment = self
             .task_assignments
@@ -77,7 +73,11 @@ impl CoordinatorService {
         project: &ProjectId,
         node: &NodeId,
     ) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
-        let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
+        let Some(descriptor) = self
+            .node_descriptors
+            .get(&NodeScopeKey::from_refs(tenant, project, node))
+            .cloned()
+        else {
             return Ok(None);
         };
         let mut remaining = VecDeque::new();
@@ -233,16 +233,14 @@ impl CoordinatorService {
         let node = NodeId::new(node);
         let identity = self
             .coordinator
-            .node_identity(&node)
+            .node_identity(&tenant, &project, &node)
             .ok_or(CoordinatorError::UnknownNode)?;
-        if identity.tenant != tenant || identity.project != project {
-            return Err(CoordinatorError::Unauthorized(
-                "source preparation completion is outside the enrolled tenant/project scope"
-                    .to_owned(),
-            )
-            .into());
-        }
-        let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
+        debug_assert_eq!(identity.tenant, tenant);
+        debug_assert_eq!(identity.project, project);
+        let descriptor = self
+            .node_descriptors
+            .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
+            .ok_or_else(|| {
                 CoordinatorError::Unauthorized(
                     "source preparation completion requires a node capability report".to_owned(),
                 )
@@ -384,6 +382,10 @@ impl CoordinatorService {
                     || attempt_project != &project
                     || attempt_process != &process
             });
+        self.task_terminal_states
+            .retain(|(task_tenant, task_project, task_process, _), _| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
         self.restart_launches
             .retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
                 attempt_tenant != &tenant
@@ -394,11 +396,12 @@ impl CoordinatorService {
             event.tenant != tenant || event.project != project || event.process != process
         });
         let active = self.coordinator.start_process_for_launch_attempt(
-            tenant,
-            project,
+            tenant.clone(),
+            project.clone(),
             process.clone(),
             launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
         );
+        self.record_process_started(&tenant, &project, &process, now_epoch_seconds);
         Ok(CoordinatorResponse::ProcessStarted {
             process,
             launch_attempt: active
@@ -413,14 +416,18 @@ impl CoordinatorService {
 
     pub(super) fn handle_reconnect_node(
         &mut self,
+        tenant: String,
+        project: String,
         node: String,
         process: String,
         epoch: u64,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
         let node = NodeId::new(node);
         let process = ProcessId::new(process);
         self.coordinator
-            .reconnect_node(&node, Some((&process, epoch)))?;
+            .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
         Ok(CoordinatorResponse::NodeReconnected { node, process })
     }
 
@@ -511,6 +518,13 @@ impl CoordinatorService {
         }
         let process_key = process_control_key(&tenant, &project, &process);
         if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
+            self.record_process_terminal(
+                &tenant,
+                &project,
+                &process,
+                super::ProcessFinalResult::Cancelled,
+                self.current_epoch_seconds()?,
+            );
             self.coordinator
                 .abort_process(&tenant, &project, &process)?;
             self.clear_operator_panel_state(&tenant, &project, &process);
@@ -596,6 +610,13 @@ impl CoordinatorService {
             }
         }
 
+        self.record_process_terminal(
+            &tenant,
+            &project,
+            &process,
+            super::ProcessFinalResult::Cancelled,
+            self.current_epoch_seconds()?,
+        );
         if let Some(launch_attempt) = launch_attempt.as_ref() {
             self.coordinator.abort_process_for_launch_attempt(
                 &tenant,
@@ -654,10 +675,18 @@ impl CoordinatorService {
             .map(|active| {
                 let process_key = process_control_key(&active.tenant, &active.project, &active.id);
                 let main = self.main_runtime.controls.get(&process_key);
+                let stored = self.process_summaries.get(&process_key);
+                let stored_main_state = stored
+                    .and_then(|summary| summary.main_terminal_state.as_ref())
+                    .map(|state| match state {
+                        super::TaskTerminalState::Completed => "completed",
+                        super::TaskTerminalState::Failed => "failed",
+                        super::TaskTerminalState::Cancelled => "cancelled",
+                    });
                 let state = if self.process_cancellations.contains(&process_key) {
                     "cancelling"
                 } else {
-                    main.map_or("running", |main| main.state.as_str())
+                    "running"
                 };
                 let main_wait_state = main.and_then(|main| {
                     if main.state != "running" {
@@ -682,9 +711,15 @@ impl CoordinatorService {
                 VirtualProcessStatus {
                     process: active.id,
                     state: state.to_owned(),
-                    main_task_definition: main.map(|main| main.task_definition.clone()),
-                    main_task_instance: main.map(|main| main.task_instance.clone()),
-                    main_state: main.map(|main| main.state.clone()),
+                    main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
+                        || stored.and_then(|summary| summary.main_task_definition.clone()),
+                    ),
+                    main_task_instance: main
+                        .map(|main| main.task_instance.clone())
+                        .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
+                    main_state: main
+                        .map(|main| main.state.clone())
+                        .or_else(|| stored_main_state.map(str::to_owned)),
                     main_wait_state,
                     main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
                     connected_nodes: active.connected_nodes.into_iter().collect(),

crates/clusterflux-coordinator/src/service/protocol/responses.rs

@@ -183,6 +183,121 @@ pub struct VirtualProcessStatus {
     pub coordinator_epoch: u64,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSummary {
+    pub id: NodeId,
+    pub display_name: String,
+    pub online: bool,
+    pub stale: bool,
+    pub last_seen_epoch_seconds: Option<u64>,
+    pub capabilities: NodeCapabilities,
+    pub direct_connectivity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessLifecycleState {
+    Active,
+    RecentTerminal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessActivityState {
+    Running,
+    WaitingForNode,
+    WaitingForTask,
+    AwaitingAction,
+    DebugEpochPartial,
+    Cancelling,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProcessFinalResult {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpochSummary {
+    pub epoch: u64,
+    pub command: String,
+    pub fully_frozen: bool,
+    pub partially_frozen: bool,
+    pub fully_resumed: bool,
+    pub failed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProcessSummary {
+    pub process: ProcessId,
+    pub lifecycle: ProcessLifecycleState,
+    pub activity: ProcessActivityState,
+    pub main_wait_state: Option<String>,
+    pub started_at_epoch_seconds: u64,
+    pub ended_at_epoch_seconds: Option<u64>,
+    pub final_result: Option<ProcessFinalResult>,
+    pub connected_nodes: Vec<NodeId>,
+    pub current_debug_epoch: Option<DebugEpochSummary>,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactAvailability {
+    Available,
+    NodeOffline,
+    Unavailable,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ArtifactRetentionState {
+    NodeRetained,
+    ExplicitStorage,
+    Lost,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactSummary {
+    pub id: ArtifactId,
+    pub display_path: String,
+    pub display_name: String,
+    pub process: ProcessId,
+    pub producer_task: TaskInstanceId,
+    pub safe_node: Option<NodeId>,
+    pub digest: Digest,
+    pub size_bytes: u64,
+    pub availability: ArtifactAvailability,
+    pub downloadable_now: bool,
+    pub retention_state: ArtifactRetentionState,
+    pub explicit_storage: bool,
+    pub order_cursor: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskLogStream {
+    Stdout,
+    Stderr,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecentLogEntry {
+    pub sequence: u64,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub stream: TaskLogStream,
+    pub text: String,
+    pub server_timestamp_epoch_seconds: u64,
+    pub truncated: bool,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum SourcePreparationDisposition {
     Pending { reason: String },
@@ -213,7 +328,7 @@ pub enum CoordinatorResponse {
         manual_review: bool,
         sanitized_reason: Option<String>,
         next_actions: Vec<String>,
-        private_moderation_details_exposed: bool,
+        sensitive_moderation_details_exposed: bool,
         signup_failure_details_exposed: bool,
     },
     AdminStatus {
@@ -282,6 +397,11 @@ pub enum CoordinatorResponse {
         descriptors: Vec<NodeDescriptor>,
         actor: UserId,
     },
+    NodeSummaries {
+        nodes: Vec<NodeSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     NodeCredentialRevoked {
         node: NodeId,
         tenant: TenantId,
@@ -373,6 +493,11 @@ pub enum CoordinatorResponse {
         processes: Vec<VirtualProcessStatus>,
         actor: UserId,
     },
+    ProcessSummaries {
+        processes: Vec<ProcessSummary>,
+        next_cursor: Option<String>,
+        actor: UserId,
+    },
     QuotaStatus {
         tenant: TenantId,
         project: ProjectId,
@@ -430,6 +555,7 @@ pub enum CoordinatorResponse {
     DebugBreakpoints {
         process: ProcessId,
         actor: UserId,
+        revision: u64,
         probe_symbols: Vec<String>,
         hit_epoch: Option<u64>,
         hit_task: Option<TaskInstanceId>,
@@ -482,6 +608,17 @@ pub enum CoordinatorResponse {
         stderr_tail: String,
         backpressured: bool,
     },
+    TaskLogChunkRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        sequence: Option<u64>,
+        next_offset: u64,
+    },
+    RecentLogs {
+        entries: Vec<RecentLogEntry>,
+        next_sequence: Option<u64>,
+        history_truncated: bool,
+    },
     VfsMetadataRecorded {
         process: ProcessId,
         task: TaskInstanceId,
@@ -518,6 +655,13 @@ pub enum CoordinatorResponse {
     ArtifactDownloadLink {
         link: DownloadLink,
     },
+    Artifacts {
+        artifacts: Vec<ArtifactSummary>,
+        next_cursor: Option<String>,
+    },
+    Artifact {
+        artifact: ArtifactSummary,
+    },
     ArtifactDownloadLinkRevoked {
         link: DownloadLink,
     },
@@ -542,6 +686,24 @@ pub enum CoordinatorResponse {
         artifact_size_bytes: u64,
     },
     Error {
-        message: String,
+        #[serde(flatten)]
+        error: clusterflux_core::ApiError,
     },
 }
+
+impl CoordinatorResponse {
+    pub fn error(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        Self::Error {
+            error: clusterflux_core::ApiError::from_message(request_id, message),
+        }
+    }
+
+    pub fn service_error(
+        request_id: impl Into<String>,
+        error: &crate::service::CoordinatorServiceError,
+    ) -> Self {
+        Self::Error {
+            error: error.api_error(request_id),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/quota.rs

@@ -260,22 +260,6 @@ impl CoordinatorQuota {
         self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
     }
 
-    pub(super) fn can_charge_log_bytes(
-        &self,
-        tenant: &TenantId,
-        project: &ProjectId,
-        bytes: u64,
-        now_epoch_seconds: u64,
-    ) -> Result<(), LimitError> {
-        self.can_charge(
-            tenant,
-            project,
-            LimitKind::LogBytes,
-            bytes,
-            now_epoch_seconds,
-        )
-    }
-
     pub(super) fn charge_log_bytes(
         &mut self,
         tenant: &TenantId,

crates/clusterflux-coordinator/src/service/relay.rs

@@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage {
     pub egress_bytes: u64,
     pub abandoned_or_failed_bytes: u64,
     pub reserved_bytes: u64,
+    pub lifetime_ingress_bytes: u64,
+    pub lifetime_egress_bytes: u64,
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    pub completed_transfers: u64,
+    pub failed_transfers: u64,
+    pub cancelled_transfers: u64,
+    pub expired_transfers: u64,
+    pub tracked_scopes: usize,
+    pub max_active_global: usize,
+    pub max_tracked_scopes: usize,
 }
 
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
@@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState {
     pub ingress_used: u64,
     pub egress_used: u64,
     pub abandoned_or_failed_used: u64,
+    #[serde(default)]
+    pub lifetime_ingress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_egress_bytes: u64,
+    #[serde(default)]
+    pub lifetime_abandoned_or_failed_bytes: u64,
+    #[serde(default)]
+    pub completed_transfers: u64,
+    #[serde(default)]
+    pub failed_transfers: u64,
+    #[serde(default)]
+    pub cancelled_transfers: u64,
+    #[serde(default)]
+    pub expired_transfers: u64,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger {
     ingress_used: u64,
     egress_used: u64,
     abandoned_or_failed_used: u64,
+    lifetime_ingress_bytes: u64,
+    lifetime_egress_bytes: u64,
+    lifetime_abandoned_or_failed_bytes: u64,
+    completed_transfers: u64,
+    failed_transfers: u64,
+    cancelled_transfers: u64,
+    expired_transfers: u64,
 }
 
 impl Default for ArtifactRelayLedger {
@@ -165,6 +196,13 @@ impl ArtifactRelayLedger {
             ingress_used: 0,
             egress_used: 0,
             abandoned_or_failed_used: 0,
+            lifetime_ingress_bytes: 0,
+            lifetime_egress_bytes: 0,
+            lifetime_abandoned_or_failed_bytes: 0,
+            completed_transfers: 0,
+            failed_transfers: 0,
+            cancelled_transfers: 0,
+            expired_transfers: 0,
         }
     }
 
@@ -218,6 +256,13 @@ impl ArtifactRelayLedger {
             ingress_used: state.ingress_used,
             egress_used: state.egress_used,
             abandoned_or_failed_used: state.abandoned_or_failed_used,
+            lifetime_ingress_bytes: state.lifetime_ingress_bytes,
+            lifetime_egress_bytes: state.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: state.completed_transfers,
+            failed_transfers: state.failed_transfers,
+            cancelled_transfers: state.cancelled_transfers,
+            expired_transfers: state.expired_transfers,
         }
     }
 
@@ -260,6 +305,13 @@ impl ArtifactRelayLedger {
             ingress_used: self.ingress_used,
             egress_used: self.egress_used,
             abandoned_or_failed_used: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
         }
     }
 
@@ -491,9 +543,11 @@ impl ArtifactRelayLedger {
         if ingress {
             reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
             self.ingress_used = self.ingress_used.saturating_add(bytes);
+            self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
         } else {
             reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
             self.egress_used = self.egress_used.saturating_add(bytes);
+            self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
         }
         let project_key = (
             reservation.scope.tenant.clone(),
@@ -554,10 +608,29 @@ impl ArtifactRelayLedger {
     pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
         if let Some(reservation) = self.reservations.remove(key) {
             if reason != RelayFinishReason::Completed {
+                let abandoned_or_failed_bytes = reservation
+                    .ingress_bytes
+                    .saturating_add(reservation.egress_bytes);
                 self.abandoned_or_failed_used = self
                     .abandoned_or_failed_used
-                    .saturating_add(reservation.ingress_bytes)
-                    .saturating_add(reservation.egress_bytes);
+                    .saturating_add(abandoned_or_failed_bytes);
+                self.lifetime_abandoned_or_failed_bytes = self
+                    .lifetime_abandoned_or_failed_bytes
+                    .saturating_add(abandoned_or_failed_bytes);
+            }
+            match reason {
+                RelayFinishReason::Completed => {
+                    self.completed_transfers = self.completed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Failed => {
+                    self.failed_transfers = self.failed_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Cancelled => {
+                    self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
+                }
+                RelayFinishReason::Expired => {
+                    self.expired_transfers = self.expired_transfers.saturating_add(1);
+                }
             }
         }
     }
@@ -580,6 +653,18 @@ impl ArtifactRelayLedger {
             ingress_bytes: self.ingress_used,
             egress_bytes: self.egress_used,
             abandoned_or_failed_bytes: self.abandoned_or_failed_used,
+            lifetime_ingress_bytes: self.lifetime_ingress_bytes,
+            lifetime_egress_bytes: self.lifetime_egress_bytes,
+            lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
+            completed_transfers: self.completed_transfers,
+            failed_transfers: self.failed_transfers,
+            cancelled_transfers: self.cancelled_transfers,
+            expired_transfers: self.expired_transfers,
+            tracked_scopes: self.project_used.len()
+                + self.tenant_used.len()
+                + self.account_used.len(),
+            max_active_global: self.configuration.max_active_global,
+            max_tracked_scopes: self.configuration.max_tracked_scopes,
             reserved_bytes: self
                 .reservations
                 .values()
@@ -588,3 +673,68 @@ impl ArtifactRelayLedger {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
+        let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
+        ledger
+            .reserve(
+                "transfer".to_owned(),
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                16,
+                16,
+                100,
+                1,
+            )
+            .unwrap();
+        ledger.charge_ingress("transfer", 24, 1).unwrap();
+        ledger.charge_egress("transfer", 24, 1).unwrap();
+        ledger.finish("transfer", RelayFinishReason::Completed);
+
+        let usage = ledger.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+
+        ledger.prepare_period(u64::MAX);
+        let usage = ledger.usage();
+        assert_eq!(usage.ingress_bytes, 0);
+        assert_eq!(usage.egress_bytes, 0);
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+
+        let restored = ArtifactRelayLedger::from_durable(
+            ArtifactRelayConfiguration::unlimited(),
+            ledger.durable_state(),
+        );
+        let usage = restored.usage();
+        assert_eq!(usage.lifetime_ingress_bytes, 24);
+        assert_eq!(usage.lifetime_egress_bytes, 24);
+        assert_eq!(usage.completed_transfers, 1);
+    }
+
+    #[test]
+    fn older_durable_relay_state_defaults_new_metrics() {
+        let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
+            "reservations": {},
+            "period": 1,
+            "project_used": [],
+            "tenant_used": [],
+            "account_used": [],
+            "ingress_used": 3,
+            "egress_used": 4,
+            "abandoned_or_failed_used": 2
+        }))
+        .unwrap();
+
+        assert_eq!(state.lifetime_ingress_bytes, 0);
+        assert_eq!(state.lifetime_egress_bytes, 0);
+        assert_eq!(state.completed_transfers, 0);
+    }
+}

crates/clusterflux-coordinator/src/service/routing.rs

@@ -5,6 +5,9 @@ impl CoordinatorService {
         &mut self,
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        request
+            .validate_external_identifiers()
+            .map_err(CoordinatorServiceError::Protocol)?;
         self.pump_main_runtime_commands();
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
@@ -42,7 +45,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -271,9 +274,17 @@ impl CoordinatorService {
                 enrollment_grant,
             ),
             CoordinatorRequest::NodeHeartbeat {
+                tenant,
+                project,
                 node,
                 node_signature,
-            } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
+            } => self.handle_node_heartbeat(
+                tenant,
+                project,
+                node,
+                node_signature,
+                &request_payload_digest,
+            ),
             CoordinatorRequest::SignedNode {
                 node,
                 node_signature,
@@ -287,6 +298,13 @@ impl CoordinatorService {
                 project,
                 actor_user,
             } => self.handle_list_node_descriptors(tenant, project, actor_user),
+            CoordinatorRequest::ListNodeSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::RevokeNodeCredential {
                 tenant,
                 project,
@@ -410,6 +428,13 @@ impl CoordinatorService {
                 project,
                 actor_user,
             } => self.handle_list_processes(tenant, project, actor_user),
+            CoordinatorRequest::ListProcessSummaries {
+                tenant,
+                project,
+                actor_user,
+                cursor,
+                limit,
+            } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit),
             CoordinatorRequest::QuotaStatus {
                 tenant,
                 project,
@@ -455,7 +480,8 @@ impl CoordinatorService {
             CoordinatorRequest::PollDebugCommand { .. }
             | CoordinatorRequest::ReportDebugState { .. }
             | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
-            CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportTaskLog { .. }
+            | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
             CoordinatorRequest::ListTaskEvents {
@@ -470,6 +496,23 @@ impl CoordinatorService {
                 actor_user,
                 process,
             } => self.handle_list_task_snapshots(tenant, project, actor_user, process),
+            CoordinatorRequest::ListRecentLogs {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             CoordinatorRequest::JoinTask {
                 tenant,
                 project,
@@ -516,6 +559,20 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            CoordinatorRequest::ListArtifacts {
+                tenant,
+                project,
+                actor_user,
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit),
+            CoordinatorRequest::GetArtifact {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+            } => self.handle_get_artifact(tenant, project, actor_user, artifact),
             CoordinatorRequest::OpenArtifactDownloadStream {
                 tenant,
                 project,
@@ -598,7 +655,7 @@ impl CoordinatorService {
                     manual_review: account_state.manual_review,
                     sanitized_reason: account_state.sanitized_reason,
                     next_actions: account_state.next_actions,
-                    private_moderation_details_exposed: false,
+                    sensitive_moderation_details_exposed: false,
                     signup_failure_details_exposed: false,
                 })
             }
@@ -685,6 +742,14 @@ impl CoordinatorService {
                     context.project.as_str().to_owned(),
                     actor.as_str().to_owned(),
                 ),
+            AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self
+                .handle_list_node_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
                 .handle_revoke_node_credential(
                     context.tenant.as_str().to_owned(),
@@ -736,6 +801,14 @@ impl CoordinatorService {
                 context.project.as_str().to_owned(),
                 actor.as_str().to_owned(),
             ),
+            AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self
+                .handle_list_process_summaries(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    cursor,
+                    limit,
+                ),
             AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -791,6 +864,20 @@ impl CoordinatorService {
                     actor.as_str().to_owned(),
                     process,
                 ),
+            AuthenticatedCoordinatorRequest::ListRecentLogs {
+                process,
+                task,
+                after_sequence,
+                limit,
+            } => self.handle_list_recent_logs(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                after_sequence,
+                limit,
+            ),
             AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
                 context.tenant.as_str().to_owned(),
                 context.project.as_str().to_owned(),
@@ -810,6 +897,24 @@ impl CoordinatorService {
                 max_bytes,
                 ttl_seconds,
             ),
+            AuthenticatedCoordinatorRequest::ListArtifacts {
+                process,
+                cursor,
+                limit,
+            } => self.handle_list_artifacts(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                cursor,
+                limit,
+            ),
+            AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+            ),
             AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
                 artifact,
                 max_bytes,

crates/clusterflux-coordinator/src/service/signed_nodes.rs

@@ -1,6 +1,6 @@
-use clusterflux_core::{NodeId, NodeSignedRequest};
+use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
 
-use crate::CoordinatorError;
+use crate::{CoordinatorError, NodeScopeKey};
 
 use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
 
@@ -12,22 +12,24 @@ impl CoordinatorService {
         request: CoordinatorRequest,
     ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
         let request_kind = signed_node_request_kind(&request)?;
-        let request_node = signed_node_request_node(&request)?;
+        let request_scope = signed_node_request_scope(&request)?;
         let request_payload = serde_json::to_value(&request).map_err(|error| {
             CoordinatorServiceError::Protocol(format!(
                 "failed to canonicalize signed node request: {error}"
             ))
         })?;
         let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
-        let signed_node = NodeId::new(signed_node);
-        if request_node != signed_node {
+        let signed_node = NodeId::try_new(signed_node).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
+        })?;
+        if request_scope.node != signed_node {
             return Err(CoordinatorError::Unauthorized(
                 "signed node request node does not match the wrapped request node".to_owned(),
             )
             .into());
         }
         self.authenticate_node_request(
-            &signed_node,
+            &request_scope,
             Some(node_signature),
             request_kind,
             &payload_digest,
@@ -144,11 +146,26 @@ impl CoordinatorService {
                 provider,
                 source_snapshot,
             ),
+            CoordinatorRequest::RequestRendezvous {
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_request_rendezvous(
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            ),
             CoordinatorRequest::ReconnectNode {
+                tenant,
+                project,
                 node,
                 process,
                 epoch,
-            } => self.handle_reconnect_node(node, process, epoch),
+            } => self.handle_reconnect_node(tenant, project, node, process, epoch),
             CoordinatorRequest::PollTaskControl {
                 tenant,
                 project,
@@ -236,6 +253,29 @@ impl CoordinatorService {
                 stderr_truncated,
                 backpressured,
             ),
+            CoordinatorRequest::ReportTaskLogChunk {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            } => self.handle_report_task_log_chunk(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stream,
+                offset,
+                source_bytes,
+                text,
+                truncated,
+            ),
             CoordinatorRequest::ReportVfsMetadata {
                 tenant,
                 project,
@@ -322,12 +362,14 @@ fn signed_node_request_kind(
         CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
         CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
         CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
+        CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"),
         CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
         CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
         CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
         CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
         CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
         CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
+        CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"),
         CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
         CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
         _ => Err(CoordinatorError::Unauthorized(
@@ -337,29 +379,135 @@ fn signed_node_request_kind(
     }
 }
 
-fn signed_node_request_node(
+fn signed_node_request_scope(
     request: &CoordinatorRequest,
-) -> Result<NodeId, CoordinatorServiceError> {
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
     match request {
-        CoordinatorRequest::ReportNodeCapabilities { node, .. }
-        | CoordinatorRequest::PollTaskAssignment { node, .. }
-        | CoordinatorRequest::PollArtifactTransfer { node, .. }
-        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
-        | CoordinatorRequest::FailArtifactTransfer { node, .. }
-        | CoordinatorRequest::LaunchChildTask { node, .. }
-        | CoordinatorRequest::JoinChildTask { node, .. }
-        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
-        | CoordinatorRequest::ReconnectNode { node, .. }
-        | CoordinatorRequest::PollTaskControl { node, .. }
-        | CoordinatorRequest::PollDebugCommand { node, .. }
-        | CoordinatorRequest::ReportDebugState { node, .. }
-        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
-        | CoordinatorRequest::ReportTaskLog { node, .. }
-        | CoordinatorRequest::ReportVfsMetadata { node, .. }
-        | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
+        CoordinatorRequest::ReportNodeCapabilities {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollTaskAssignment {
+            tenant,
+            project,
+            node,
+        }
+        | CoordinatorRequest::PollArtifactTransfer {
+            tenant,
+            project,
+            node,
+        }
+        | CoordinatorRequest::UploadArtifactTransferChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::FailArtifactTransfer {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::LaunchChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::JoinChildTask {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::CompleteSourcePreparation {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReconnectNode {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollTaskControl {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::PollDebugCommand {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugState {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportDebugProbeHit {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLog {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportTaskLogChunk {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::ReportVfsMetadata {
+            tenant,
+            project,
+            node,
+            ..
+        }
+        | CoordinatorRequest::TaskCompleted {
+            tenant,
+            project,
+            node,
+            ..
+        } => node_scope_from_strings(tenant, project, node),
+        CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
+            scope.tenant.clone(),
+            scope.project.clone(),
+            source.node.clone(),
+        )),
         _ => Err(CoordinatorError::Unauthorized(
             "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
         )
         .into()),
     }
 }
+
+fn node_scope_from_strings(
+    tenant: &str,
+    project: &str,
+    node: &str,
+) -> Result<NodeScopeKey, CoordinatorServiceError> {
+    let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
+    })?;
+    let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
+    })?;
+    let node = NodeId::try_new(node.to_owned()).map_err(|error| {
+        CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
+    })?;
+    Ok(NodeScopeKey::new(tenant, project, node))
+}

crates/clusterflux-coordinator/src/service/summaries.rs

@@ -0,0 +1,565 @@
+use std::collections::BTreeSet;
+use std::time::Instant;
+
+use clusterflux_core::{
+    ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId,
+};
+
+use super::keys::{process_control_key, ProcessControlKey};
+use super::{
+    ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse,
+    CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary,
+    NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
+    TaskAttemptState,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct StoredProcessSummary {
+    pub(super) started_at_epoch_seconds: u64,
+    pub(super) ended_at_epoch_seconds: Option<u64>,
+    pub(super) final_result: Option<ProcessFinalResult>,
+    pub(super) connected_nodes: Vec<NodeId>,
+    pub(super) main_task_definition: Option<TaskDefinitionId>,
+    pub(super) main_task_instance: Option<TaskInstanceId>,
+    pub(super) main_terminal_state: Option<super::TaskTerminalState>,
+    pub(super) order: u64,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_list_node_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = cursor.as_deref();
+        let mut nodes = self
+            .node_descriptors
+            .iter()
+            .filter(|(scope, descriptor)| {
+                scope.tenant == tenant
+                    && scope.project == project
+                    && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor)
+            })
+            .map(|(scope, descriptor)| {
+                let online = self.node_is_live(scope);
+                NodeSummary {
+                    id: descriptor.id.clone(),
+                    display_name: descriptor.id.as_str().to_owned(),
+                    online,
+                    stale: !online,
+                    last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(),
+                    capabilities: descriptor.capabilities.clone(),
+                    direct_connectivity: descriptor.direct_connectivity,
+                }
+            })
+            .collect::<Vec<_>>();
+        nodes.sort_by(|left, right| left.id.cmp(&right.id));
+        let has_more = nodes.len() > limit as usize;
+        nodes.truncate(limit as usize);
+        let next_cursor = has_more
+            .then(|| nodes.last().map(|node| node.id.as_str().to_owned()))
+            .flatten();
+        Ok(CoordinatorResponse::NodeSummaries {
+            nodes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    pub(super) fn record_process_started(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) {
+            logs.retain(|entry| &entry.process != process);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(tenant.clone(), project.clone()));
+            }
+        }
+        self.recent_log_dropped_through.remove(&key);
+        self.recent_log_accounted_bytes.retain(
+            |(entry_tenant, entry_project, entry_process, _, _), _| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.recent_log_quota_truncated_streams.retain(
+            |(entry_tenant, entry_project, entry_process, _, _)| {
+                entry_tenant != tenant || entry_project != project || entry_process != process
+            },
+        );
+        self.process_summary_order
+            .retain(|retained| retained != &key);
+        self.evict_process_summaries_for_project(tenant, project);
+        self.evict_process_summaries_total();
+        let order = self.next_process_summary_order;
+        self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+        self.process_summaries.insert(
+            key.clone(),
+            StoredProcessSummary {
+                started_at_epoch_seconds: now_epoch_seconds,
+                ended_at_epoch_seconds: None,
+                final_result: None,
+                connected_nodes: Vec::new(),
+                main_task_definition: None,
+                main_task_instance: None,
+                main_terminal_state: None,
+                order,
+            },
+        );
+        self.process_summary_order.push_back(key);
+    }
+
+    pub(super) fn record_process_terminal(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        final_result: ProcessFinalResult,
+        now_epoch_seconds: u64,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        let connected_nodes = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or_default();
+        let order = self.next_process_summary_order;
+        let entry = self
+            .process_summaries
+            .entry(key.clone())
+            .or_insert_with(|| {
+                self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
+                self.process_summary_order.push_back(key.clone());
+                StoredProcessSummary {
+                    started_at_epoch_seconds: now_epoch_seconds,
+                    ended_at_epoch_seconds: None,
+                    final_result: None,
+                    connected_nodes: Vec::new(),
+                    main_task_definition: None,
+                    main_task_instance: None,
+                    main_terminal_state: None,
+                    order,
+                }
+            });
+        entry.ended_at_epoch_seconds = Some(now_epoch_seconds);
+        entry.final_result = Some(final_result);
+        entry.connected_nodes = connected_nodes;
+    }
+
+    pub(super) fn record_main_terminal_state(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task_definition: TaskDefinitionId,
+        task_instance: TaskInstanceId,
+        terminal_state: super::TaskTerminalState,
+    ) {
+        let key = process_control_key(tenant, project, process);
+        if !self.process_summaries.contains_key(&key) {
+            self.record_process_started(
+                tenant,
+                project,
+                process,
+                self.liveness_now_epoch_seconds(),
+            );
+        }
+        if let Some(summary) = self.process_summaries.get_mut(&key) {
+            summary.main_task_definition = Some(task_definition);
+            summary.main_task_instance = Some(task_instance);
+            summary.main_terminal_state = Some(terminal_state);
+        }
+    }
+
+    pub(super) fn handle_list_process_summaries(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let cursor = parse_order_cursor(cursor.as_deref(), "process")?;
+        let mut stored = self
+            .process_summaries
+            .iter()
+            .filter(|((entry_tenant, entry_project, _), summary)| {
+                entry_tenant == &tenant
+                    && entry_project == &project
+                    && cursor.is_none_or(|cursor| summary.order < cursor)
+            })
+            .map(|(key, summary)| (key.clone(), summary.clone()))
+            .collect::<Vec<_>>();
+        stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order));
+        let has_more = stored.len() > limit as usize;
+        stored.truncate(limit as usize);
+        let processes = stored
+            .into_iter()
+            .map(|(key, stored)| self.process_summary_from_stored(&key, stored))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| processes.last().map(|process| process.order_cursor.clone()))
+            .flatten();
+        Ok(CoordinatorResponse::ProcessSummaries {
+            processes,
+            next_cursor,
+            actor,
+        })
+    }
+
+    fn process_summary_from_stored(
+        &self,
+        key: &ProcessControlKey,
+        stored: StoredProcessSummary,
+    ) -> ProcessSummary {
+        let (tenant, project, process) = key;
+        let active = self.coordinator.active_process(tenant, project, process);
+        let process_key = process_control_key(tenant, project, process);
+        let main_wait_state = active.and_then(|_| {
+            if self.pending_task_launches.iter().any(|pending| {
+                &pending.tenant == tenant
+                    && &pending.project == project
+                    && &pending.process == process
+            }) {
+                Some("waiting_for_node".to_owned())
+            } else if self
+                .main_runtime
+                .is_waiting_for_task(tenant, project, process)
+            {
+                Some("waiting_for_task".to_owned())
+            } else {
+                None
+            }
+        });
+        let current_debug_epoch = self.debug_epoch_summary(&process_key);
+        let awaiting_action = self.task_attempts.iter().any(
+            |((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
+                attempt_tenant == tenant
+                    && attempt_project == project
+                    && attempt_process == process
+                    && attempts.iter().any(|attempt| {
+                        attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction
+                    })
+            },
+        );
+        let activity = if let Some(result) = &stored.final_result {
+            match result {
+                ProcessFinalResult::Completed => ProcessActivityState::Completed,
+                ProcessFinalResult::Failed => ProcessActivityState::Failed,
+                ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled,
+            }
+        } else if self.process_cancellations.contains(&process_key) {
+            ProcessActivityState::Cancelling
+        } else if current_debug_epoch
+            .as_ref()
+            .is_some_and(|epoch| epoch.partially_frozen)
+        {
+            ProcessActivityState::DebugEpochPartial
+        } else if awaiting_action {
+            ProcessActivityState::AwaitingAction
+        } else {
+            match main_wait_state.as_deref() {
+                Some("waiting_for_node") => ProcessActivityState::WaitingForNode,
+                Some("waiting_for_task") => ProcessActivityState::WaitingForTask,
+                _ => ProcessActivityState::Running,
+            }
+        };
+        let connected_nodes = active
+            .map(|active| active.connected_nodes.iter().cloned().collect())
+            .unwrap_or(stored.connected_nodes);
+        ProcessSummary {
+            process: process.clone(),
+            lifecycle: if active.is_some() {
+                ProcessLifecycleState::Active
+            } else {
+                ProcessLifecycleState::RecentTerminal
+            },
+            activity,
+            main_wait_state,
+            started_at_epoch_seconds: stored.started_at_epoch_seconds,
+            ended_at_epoch_seconds: stored.ended_at_epoch_seconds,
+            final_result: stored.final_result,
+            connected_nodes,
+            current_debug_epoch,
+            order_cursor: format!("process:{}", stored.order),
+        }
+    }
+
+    fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option<DebugEpochSummary> {
+        let runtime = self.debug_epoch_runtime.get(key)?;
+        let acknowledgements = runtime.acknowledgements.values().collect::<Vec<_>>();
+        let all_acknowledged = !runtime.expected.is_empty()
+            && runtime
+                .expected
+                .iter()
+                .all(|participant| runtime.acknowledgements.contains_key(participant));
+        let fully_frozen = runtime.command == "freeze"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Frozen);
+        let freeze_deadline_elapsed =
+            runtime.command == "freeze" && Instant::now() >= runtime.deadline;
+        let frozen_count = acknowledgements
+            .iter()
+            .filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
+            .count();
+        let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen;
+        let fully_resumed = runtime.command == "resume"
+            && all_acknowledged
+            && acknowledgements
+                .iter()
+                .all(|ack| ack.state == DebugAcknowledgementState::Running);
+        let failed = acknowledgements
+            .iter()
+            .any(|ack| ack.state == DebugAcknowledgementState::Failed)
+            || (freeze_deadline_elapsed
+                && runtime
+                    .expected
+                    .iter()
+                    .any(|participant| !runtime.acknowledgements.contains_key(participant)));
+        Some(DebugEpochSummary {
+            epoch: runtime.epoch,
+            command: runtime.command.clone(),
+            fully_frozen,
+            partially_frozen,
+            fully_resumed,
+            failed,
+        })
+    }
+
+    pub(super) fn handle_list_artifacts(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: Option<String>,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = process.map(ProcessId::new);
+        if let Some(process) = &process {
+            self.authorize_task_event_process_scope(&tenant, &project, process)?;
+        }
+        let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?;
+        let mut metadata = self
+            .artifact_registry
+            .metadata_for_project(&tenant, &project)
+            .filter(|metadata| {
+                process
+                    .as_ref()
+                    .is_none_or(|process| &metadata.process == process)
+                    && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor)
+            })
+            .cloned()
+            .collect::<Vec<_>>();
+        metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch));
+        let has_more = metadata.len() > limit as usize;
+        metadata.truncate(limit as usize);
+        let artifacts = metadata
+            .into_iter()
+            .map(|metadata| self.artifact_summary(metadata))
+            .collect::<Vec<_>>();
+        let next_cursor = has_more
+            .then(|| {
+                artifacts
+                    .last()
+                    .map(|artifact| artifact.order_cursor.clone())
+            })
+            .flatten();
+        Ok(CoordinatorResponse::Artifacts {
+            artifacts,
+            next_cursor,
+        })
+    }
+
+    pub(super) fn handle_get_artifact(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let metadata = self
+            .artifact_registry
+            .metadata(&tenant, &project, &artifact)
+            .cloned()
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        Ok(CoordinatorResponse::Artifact {
+            artifact: self.artifact_summary(metadata),
+        })
+    }
+
+    fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary {
+        let live_retaining_nodes = metadata
+            .retaining_nodes
+            .iter()
+            .filter(|node| {
+                self.node_is_live(&crate::NodeScopeKey::from_refs(
+                    &metadata.tenant,
+                    &metadata.project,
+                    node,
+                ))
+            })
+            .cloned()
+            .collect::<BTreeSet<_>>();
+        let safe_node = live_retaining_nodes
+            .iter()
+            .next()
+            .cloned()
+            .or_else(|| metadata.retaining_nodes.iter().next().cloned());
+        let explicit_storage = !metadata.explicit_locations.is_empty();
+        let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage;
+        let availability = if downloadable_now {
+            ArtifactAvailability::Available
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactAvailability::NodeOffline
+        } else {
+            ArtifactAvailability::Unavailable
+        };
+        let retention_state = if explicit_storage {
+            ArtifactRetentionState::ExplicitStorage
+        } else if !metadata.retaining_nodes.is_empty() {
+            ArtifactRetentionState::NodeRetained
+        } else {
+            ArtifactRetentionState::Lost
+        };
+        let display_suffix = metadata.id.as_str().replace(':', "/");
+        let display_path = format!("/vfs/artifacts/{display_suffix}");
+        let display_name = display_suffix
+            .rsplit('/')
+            .next()
+            .unwrap_or(metadata.id.as_str())
+            .to_owned();
+        ArtifactSummary {
+            id: metadata.id,
+            display_path,
+            display_name,
+            process: metadata.process,
+            producer_task: metadata.producer_task,
+            safe_node,
+            digest: metadata.digest,
+            size_bytes: metadata.size,
+            availability,
+            downloadable_now,
+            retention_state,
+            explicit_storage,
+            order_cursor: format!("artifact:{}", metadata.flushed_epoch),
+        }
+    }
+
+    fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) {
+        while self
+            .process_summaries
+            .keys()
+            .filter(|(entry_tenant, entry_project, _)| {
+                entry_tenant == tenant && entry_project == project
+            })
+            .count()
+            >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT
+        {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                &key.0 == tenant
+                    && &key.1 == project
+                    && self
+                        .process_summaries
+                        .get(*key)
+                        .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn evict_process_summaries_total(&mut self) {
+        while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL {
+            let candidate = self.process_summary_order.iter().find(|key| {
+                self.process_summaries
+                    .get(*key)
+                    .is_some_and(|summary| summary.final_result.is_some())
+            });
+            let Some(candidate) = candidate.cloned() else {
+                break;
+            };
+            self.remove_process_summary_state(&candidate);
+        }
+    }
+
+    fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
+        self.process_summaries.remove(key);
+        self.recent_log_dropped_through.remove(key);
+        if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
+            logs.retain(|entry| entry.process != key.2);
+            if logs.is_empty() {
+                self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
+            }
+        }
+        self.recent_log_accounted_bytes
+            .retain(|(tenant, project, process, _, _), _| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.recent_log_quota_truncated_streams
+            .retain(|(tenant, project, process, _, _)| {
+                tenant != &key.0 || project != &key.1 || process != &key.2
+            });
+        self.process_summary_order
+            .retain(|retained| retained != key);
+    }
+}
+
+fn parse_order_cursor(
+    cursor: Option<&str>,
+    expected_kind: &str,
+) -> Result<Option<u64>, CoordinatorServiceError> {
+    cursor
+        .map(|cursor| {
+            let (kind, order) = cursor.split_once(':').ok_or_else(|| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })?;
+            if kind != expected_kind {
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                )));
+            }
+            order.parse::<u64>().map_err(|_| {
+                CoordinatorServiceError::Protocol(format!(
+                    "invalid {expected_kind} pagination cursor"
+                ))
+            })
+        })
+        .transpose()
+}

crates/clusterflux-coordinator/src/service/tcp.rs

@@ -79,17 +79,15 @@ impl CoordinatorService {
                 continue;
             }
             let response = match decode_wire_request(&line) {
-                Ok(request) => match authorize_client_request(&request, authority_mode)
+                Ok((request_id, request)) => {
+                    match authorize_client_request(&request, authority_mode)
                         .and_then(|()| self.handle_request(request))
                     {
                         Ok(response) => response,
-                    Err(err) => CoordinatorResponse::Error {
-                        message: err.to_string(),
-                    },
-                },
-                Err(err) => CoordinatorResponse::Error {
-                    message: err.to_string(),
-                },
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
+                    }
+                }
+                Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
             };
             serde_json::to_writer(&mut writer, &response)?;
             writer.write_all(b"\n")?;
@@ -114,25 +112,19 @@ fn handle_shared_stream(
             continue;
         }
         let response = match decode_wire_request(&line) {
-            Ok(request) => match authorize_client_request(&request, authority_mode) {
+            Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) {
                 Ok(()) => match service.lock() {
                     Ok(mut service) => match service.handle_request(request) {
                         Ok(response) => response,
-                        Err(err) => CoordinatorResponse::Error {
-                            message: err.to_string(),
+                        Err(err) => CoordinatorResponse::service_error(request_id, &err),
                     },
+                    Err(_) => {
+                        CoordinatorResponse::error(request_id, "coordinator service lock poisoned")
+                    }
                 },
-                    Err(_) => CoordinatorResponse::Error {
-                        message: "coordinator service lock poisoned".to_owned(),
-                    },
-                },
-                Err(err) => CoordinatorResponse::Error {
-                    message: err.to_string(),
-                },
-            },
-            Err(err) => CoordinatorResponse::Error {
-                message: err.to_string(),
+                Err(err) => CoordinatorResponse::service_error(request_id, &err),
             },
+            Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
         };
         serde_json::to_writer(&mut writer, &response)?;
         writer.write_all(b"\n")?;
@@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato
     Ok((listener, addr))
 }
 
-fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
+fn decode_wire_request(
+    line: &str,
+) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> {
     serde_json::from_str::<super::CoordinatorWireRequest>(line)?
-        .into_request()
+        .into_parts()
         .map_err(CoordinatorServiceError::Protocol)
 }
 
+fn wire_request_id_hint(line: &str) -> String {
+    serde_json::from_str::<serde_json::Value>(line)
+        .ok()
+        .and_then(|value| {
+            value
+                .get("request_id")
+                .and_then(|value| value.as_str())
+                .map(str::to_owned)
+        })
+        .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok())
+        .unwrap_or_else(|| "unavailable".to_owned())
+}
+
 fn authorize_client_request(
     request: &CoordinatorRequest,
     authority_mode: ClientAuthorityMode,
@@ -180,15 +187,21 @@ fn authorize_client_request(
         }
         | CoordinatorRequest::AdminStatus { .. }
         | CoordinatorRequest::SuspendTenant { .. } => Ok(()),
-        _ => Err(CoordinatorServiceError::Protocol(
+        _ => Err(crate::CoordinatorError::Unauthorized(
             "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
                 .to_owned(),
-        )),
+        )
+        .into()),
     }
 }
 
 #[cfg(test)]
 mod transport_boundary_tests {
+    use std::io::{BufRead as _, BufReader};
+
+    use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId};
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -197,4 +210,99 @@ mod transport_boundary_tests {
         let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
         assert!(error.to_string().contains("restricted to loopback"));
     }
+
+    #[test]
+    fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() {
+        let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
+        let mut coordinator = CoordinatorService::new(11);
+        coordinator
+            .issue_cli_session(
+                TenantId::from("tenant"),
+                ProjectId::from("project"),
+                UserId::from("user"),
+                "healthy-session",
+                None,
+            )
+            .unwrap();
+        let shared = Arc::new(Mutex::new(coordinator));
+        let server_shared = Arc::clone(&shared);
+        let server = std::thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap();
+        });
+
+        let mut stream = TcpStream::connect(addr).unwrap();
+        let mut reader = BufReader::new(stream.try_clone().unwrap());
+        for (index, malformed_process) in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0process".to_owned(),
+            "bad process!".to_owned(),
+            "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1),
+        ]
+        .into_iter()
+        .enumerate()
+        {
+            let malformed = coordinator_wire_request(
+                format!("malformed-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": {
+                        "type": "abort_process",
+                        "process": malformed_process,
+                        "launch_attempt": "valid-attempt"
+                    }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &malformed).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            let mut line = String::new();
+            reader.read_line(&mut line).unwrap();
+            let CoordinatorResponse::Error { error } =
+                serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
+            else {
+                panic!("malformed identifier request unexpectedly succeeded");
+            };
+            assert!(
+                error.message.contains("malformed external identifier")
+                    && error.message.contains("request.request.process"),
+                "unexpected malformed identifier response: {}",
+                error.message
+            );
+            assert_eq!(error.request_id, format!("malformed-{index}"));
+            assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
+
+            let valid = coordinator_wire_request(
+                format!("healthy-{index}"),
+                json!({
+                    "type": "authenticated",
+                    "session_secret": "healthy-session",
+                    "request": { "type": "auth_status" }
+                }),
+            );
+            serde_json::to_writer(&mut stream, &valid).unwrap();
+            stream.write_all(b"\n").unwrap();
+            stream.flush().unwrap();
+
+            line.clear();
+            reader.read_line(&mut line).unwrap();
+            assert!(
+                matches!(
+                    serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
+                    CoordinatorResponse::AuthStatus {
+                        authenticated: true,
+                        ..
+                    }
+                ),
+                "valid authenticated traffic failed after malformed request {index}"
+            );
+        }
+
+        stream.shutdown(std::net::Shutdown::Both).unwrap();
+        server.join().unwrap();
+        assert!(!shared.is_poisoned());
+    }
 }

crates/clusterflux-coordinator/src/service/wire_protocol.rs

@@ -1,4 +1,4 @@
-use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
+use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest {
 
 impl CoordinatorWireRequest {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         match self {
-            Self::Envelope(envelope) => envelope.into_request(),
+            Self::Envelope(envelope) => envelope.into_parts(),
         }
     }
 }
@@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope {
 
 impl CoordinatorRequestEnvelope {
     pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        self.into_parts().map(|(_, request)| request)
+    }
+
+    pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
         if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
             return Err(format!(
                 "unsupported coordinator wire request type {}; expected {}",
@@ -44,9 +52,9 @@ impl CoordinatorRequestEnvelope {
                 self.protocol_version, COORDINATOR_PROTOCOL_VERSION
             ));
         }
-        if self.request_id.trim().is_empty() {
-            return Err("coordinator wire request_id must be non-empty".to_owned());
-        }
+        RequestId::try_new(self.request_id.clone())
+            .map_err(|error| format!("malformed coordinator wire request_id: {error}"))?;
+        self.payload.validate_external_identifiers()?;
         let payload_operation = self.payload.operation()?;
         if self.operation != payload_operation {
             return Err(format!(
@@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope {
                 self.operation, payload_operation
             ));
         }
-        Ok(self.payload)
+        Ok((self.request_id, self.payload))
     }
 }

crates/clusterflux-core/src/api_error.rs

@@ -0,0 +1,296 @@
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCode {
+    Unauthenticated,
+    SessionExpired,
+    AccountSuspended,
+    Forbidden,
+    ValidationError,
+    NotFound,
+    Conflict,
+    ActiveProcessExists,
+    NodeOffline,
+    NoCapableNode,
+    TaskNotRestartable,
+    ArtifactUnavailable,
+    ArtifactLimitExceeded,
+    QuotaExceeded,
+    TemporaryCapacity,
+    DebugEpochPartial,
+    InternalError,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ApiErrorCategory {
+    Authentication,
+    Authorization,
+    Validation,
+    State,
+    Availability,
+    Resource,
+    Internal,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ApiError {
+    pub code: ApiErrorCode,
+    pub category: ApiErrorCategory,
+    pub message: String,
+    pub retryable: bool,
+    pub request_id: String,
+}
+
+impl fmt::Display for ApiError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            formatter,
+            "{} (code {:?}, request {})",
+            self.message, self.code, self.request_id
+        )
+    }
+}
+
+impl std::error::Error for ApiError {}
+
+impl ApiError {
+    pub fn new(
+        code: ApiErrorCode,
+        category: ApiErrorCategory,
+        message: impl Into<String>,
+        retryable: bool,
+        request_id: impl Into<String>,
+    ) -> Self {
+        Self {
+            code,
+            category,
+            message: message.into(),
+            retryable,
+            request_id: request_id.into(),
+        }
+    }
+
+    pub fn from_message(request_id: impl Into<String>, message: impl Into<String>) -> Self {
+        let request_id = request_id.into();
+        let message = message.into();
+        let normalized = message.to_ascii_lowercase();
+        let (code, category, retryable) = if normalized.contains("session credential has expired")
+            || normalized.contains("session expired")
+        {
+            (
+                ApiErrorCode::SessionExpired,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("tenant is suspended")
+            || normalized.contains("account is suspended")
+            || normalized.contains("suspended by hosted")
+        {
+            (
+                ApiErrorCode::AccountSuspended,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("session credential")
+            || normalized.contains("no authenticated")
+            || normalized.contains("not authenticated")
+            || normalized.contains("credential is not")
+        {
+            (
+                ApiErrorCode::Unauthenticated,
+                ApiErrorCategory::Authentication,
+                false,
+            )
+        } else if normalized.contains("project already has active virtual process") {
+            (
+                ApiErrorCode::ActiveProcessExists,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("no capable node") {
+            (
+                ApiErrorCode::NoCapableNode,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("node offline")
+            || normalized.contains("node is not live")
+            || normalized.contains("source node is not connected")
+            || normalized.contains("direct connectivity unavailable")
+        {
+            (
+                ApiErrorCode::NodeOffline,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("restart")
+            && (normalized.contains("not restartable")
+                || normalized.contains("requires whole")
+                || normalized.contains("clean boundary"))
+        {
+            (
+                ApiErrorCode::TaskNotRestartable,
+                ApiErrorCategory::State,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("exceeds download limit")
+                || normalized.contains("download session limit")
+                || normalized.contains("artifact limit"))
+        {
+            (
+                ApiErrorCode::ArtifactLimitExceeded,
+                ApiErrorCategory::Resource,
+                false,
+            )
+        } else if normalized.contains("artifact")
+            && (normalized.contains("does not exist")
+                || normalized.contains("not found")
+                || normalized.contains("unknown artifact"))
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("artifact")
+            && (normalized.contains("unavailable") || normalized.contains("retention"))
+        {
+            (
+                ApiErrorCode::ArtifactUnavailable,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("resource limit")
+            || normalized.contains("quota")
+            || normalized.contains("limit exceeded")
+        {
+            (
+                ApiErrorCode::QuotaExceeded,
+                ApiErrorCategory::Resource,
+                true,
+            )
+        } else if normalized.contains("capacity")
+            || normalized.contains("temporarily full")
+            || normalized.contains("replay window is full")
+        {
+            (
+                ApiErrorCode::TemporaryCapacity,
+                ApiErrorCategory::Availability,
+                true,
+            )
+        } else if normalized.contains("partial debug epoch")
+            || normalized.contains("debug epoch is partially")
+        {
+            (
+                ApiErrorCode::DebugEpochPartial,
+                ApiErrorCategory::State,
+                true,
+            )
+        } else if normalized.contains("malformed")
+            || normalized.contains("invalid ")
+            || normalized.contains("protocol")
+            || normalized.contains("unknown field")
+            || normalized.contains("missing field")
+            || normalized.contains("must ")
+        {
+            (
+                ApiErrorCode::ValidationError,
+                ApiErrorCategory::Validation,
+                false,
+            )
+        } else if normalized.contains("outside")
+            || normalized.contains("unauthorized")
+            || normalized.contains("denied")
+            || normalized.contains("requires an authenticated")
+            || normalized.contains("may only")
+        {
+            (
+                ApiErrorCode::Forbidden,
+                ApiErrorCategory::Authorization,
+                false,
+            )
+        } else if normalized.contains("not found")
+            || normalized.contains("does not exist")
+            || normalized.contains("unknown ")
+        {
+            (ApiErrorCode::NotFound, ApiErrorCategory::State, false)
+        } else if normalized.contains("already")
+            || normalized.contains("conflict")
+            || normalized.contains("requires an active")
+        {
+            (ApiErrorCode::Conflict, ApiErrorCategory::State, false)
+        } else {
+            (
+                ApiErrorCode::InternalError,
+                ApiErrorCategory::Internal,
+                false,
+            )
+        };
+        Self::new(code, category, message, retryable, request_id)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn required_machine_codes_are_stably_serialized() {
+        let required = [
+            ApiErrorCode::Unauthenticated,
+            ApiErrorCode::SessionExpired,
+            ApiErrorCode::AccountSuspended,
+            ApiErrorCode::Forbidden,
+            ApiErrorCode::ValidationError,
+            ApiErrorCode::NotFound,
+            ApiErrorCode::Conflict,
+            ApiErrorCode::ActiveProcessExists,
+            ApiErrorCode::NodeOffline,
+            ApiErrorCode::NoCapableNode,
+            ApiErrorCode::TaskNotRestartable,
+            ApiErrorCode::ArtifactUnavailable,
+            ApiErrorCode::ArtifactLimitExceeded,
+            ApiErrorCode::QuotaExceeded,
+            ApiErrorCode::TemporaryCapacity,
+            ApiErrorCode::DebugEpochPartial,
+        ];
+        let serialized = required
+            .into_iter()
+            .map(|code| serde_json::to_value(code).unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            serialized,
+            vec![
+                "unauthenticated",
+                "session_expired",
+                "account_suspended",
+                "forbidden",
+                "validation_error",
+                "not_found",
+                "conflict",
+                "active_process_exists",
+                "node_offline",
+                "no_capable_node",
+                "task_not_restartable",
+                "artifact_unavailable",
+                "artifact_limit_exceeded",
+                "quota_exceeded",
+                "temporary_capacity",
+                "debug_epoch_partial",
+            ]
+        );
+    }
+
+    #[test]
+    fn message_classification_keeps_request_identity() {
+        let error = ApiError::from_message(
+            "request-17",
+            "CLI session credential has expired; run login again",
+        );
+        assert_eq!(error.code, ApiErrorCode::SessionExpired);
+        assert_eq!(error.category, ApiErrorCategory::Authentication);
+        assert_eq!(error.request_id, "request-17");
+        assert!(!error.retryable);
+    }
+}

crates/clusterflux-core/src/artifact.rs

@@ -10,7 +10,7 @@ use crate::{
 
 const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
 const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
-const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
+const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
     pub coordinator_has_large_bytes: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct ArtifactScopeKey {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub artifact: ArtifactId,
+}
+
+impl ArtifactScopeKey {
+    pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
+        Self {
+            tenant,
+            project,
+            artifact,
+        }
+    }
+
+    pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
+        Self::new(tenant.clone(), project.clone(), artifact.clone())
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ArtifactFlush {
     pub id: ArtifactId,
@@ -148,7 +169,7 @@ pub struct ArtifactFlush {
 
 #[derive(Clone, Debug, Default)]
 pub struct ArtifactRegistry {
-    artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
+    artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
     issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
     next_epoch: u64,
 }
@@ -162,42 +183,46 @@ impl ArtifactRegistry {
     pub fn flush_metadata_bounded(
         &mut self,
         flush: ArtifactFlush,
-        pinned: &BTreeSet<ArtifactId>,
+        pinned: &BTreeSet<ArtifactScopeKey>,
     ) -> Result<ArtifactMetadata, String> {
-        let replacing_existing = self.artifacts.contains_key(&flush.id);
-        while !replacing_existing
-            && self
-                .artifacts
-                .values()
-                .filter(|metadata| {
-                    metadata.tenant == flush.tenant
-                        && metadata.project == flush.project
-                        && metadata.process == flush.process
-                })
-                .count()
-                >= MAX_ARTIFACT_METADATA_PER_PROCESS
-        {
-            let candidate = self
-                .artifacts
-                .values()
-                .filter(|metadata| {
-                    metadata.tenant == flush.tenant
-                        && metadata.project == flush.project
-                        && metadata.process == flush.process
-                        && !pinned.contains(&metadata.id)
-                        && !self.issued_download_links.values().any(|issued| {
-                            issued.link.artifact == metadata.id
-                        })
-                })
-                .min_by_key(|metadata| metadata.flushed_epoch)
-                .map(|metadata| metadata.id.clone())
-                .ok_or_else(|| {
-                    "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
-                        .to_owned()
-                })?;
-            self.artifacts.remove(&candidate);
+        self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
     }
-        self.next_epoch += 1;
+
+    pub fn flush_metadata_with_protected_processes(
+        &mut self,
+        flush: ArtifactFlush,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> Result<ArtifactMetadata, String> {
+        let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
+        let replacing = self.artifacts.contains_key(&key);
+        let retained_for_project = self
+            .artifacts
+            .values()
+            .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
+            .count();
+        let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
+            None
+        } else {
+            let mut protected_processes = protected_processes.clone();
+            protected_processes.insert(flush.process.clone());
+            Some(
+                self.project_metadata_eviction_candidate(
+                    &flush.tenant,
+                    &flush.project,
+                    pinned,
+                    &protected_processes,
+                )
+                .ok_or_else(|| {
+                    format!(
+                        "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
+                         exhausted by active or retained artifacts"
+                    )
+                })?,
+            )
+        };
+
+        self.next_epoch = self.next_epoch.saturating_add(1);
         let metadata = ArtifactMetadata {
             id: flush.id.clone(),
             tenant: flush.tenant,
@@ -212,36 +237,112 @@ impl ArtifactRegistry {
             explicit_locations: Vec::new(),
             coordinator_has_large_bytes: false,
         };
-        self.artifacts.insert(flush.id, metadata.clone());
+        if let Some(eviction) = eviction {
+            self.artifacts.remove(&eviction);
+        }
+        self.artifacts.insert(key, metadata.clone());
         Ok(metadata)
     }
 
+    pub fn enforce_project_metadata_limit(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> usize {
+        let mut evicted = 0;
+        while self
+            .artifacts
+            .values()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+            .count()
+            > MAX_ARTIFACT_METADATA_PER_PROJECT
+        {
+            let candidate = self.project_metadata_eviction_candidate(
+                tenant,
+                project,
+                pinned,
+                protected_processes,
+            );
+            let Some(candidate) = candidate else {
+                break;
+            };
+            self.artifacts.remove(&candidate);
+            evicted += 1;
+        }
+        evicted
+    }
+
+    fn project_metadata_eviction_candidate(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        pinned: &BTreeSet<ArtifactScopeKey>,
+        protected_processes: &BTreeSet<ProcessId>,
+    ) -> Option<ArtifactScopeKey> {
+        self.artifacts
+            .values()
+            .filter(|metadata| {
+                &metadata.tenant == tenant
+                    && &metadata.project == project
+                    && !pinned.contains(&ArtifactScopeKey::from_refs(
+                        &metadata.tenant,
+                        &metadata.project,
+                        &metadata.id,
+                    ))
+                    && !protected_processes.contains(&metadata.process)
+                    && metadata.explicit_locations.is_empty()
+                    && !self.issued_download_links.values().any(|issued| {
+                        !issued.revoked
+                            && issued.link.tenant == metadata.tenant
+                            && issued.link.project == metadata.project
+                            && issued.link.artifact == metadata.id
+                    })
+            })
+            .min_by_key(|metadata| metadata.flushed_epoch)
+            .map(|metadata| {
+                ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
+            })
+    }
+
     pub fn sync_to_explicit_store(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         artifact: &ArtifactId,
         location: impl Into<String>,
     ) -> Result<(), ArtifactUnavailable> {
         let metadata = self
             .artifacts
-            .get_mut(artifact)
+            .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
             .ok_or(ArtifactUnavailable)?;
         metadata.explicit_locations.push(location.into());
         Ok(())
     }
 
-    pub fn garbage_collect_node(&mut self, node: &NodeId) {
-        for metadata in self.artifacts.values_mut() {
+    pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
+        for metadata in self
+            .artifacts
+            .values_mut()
+            .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
+        {
             metadata.retaining_nodes.remove(node);
         }
     }
 
     pub fn reconcile_node_retention(
         &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
         node: &NodeId,
         retained_artifacts: &BTreeSet<ArtifactId>,
     ) {
-        for (artifact, metadata) in &mut self.artifacts {
-            if retained_artifacts.contains(artifact) {
+        for (key, metadata) in &mut self.artifacts {
+            if &key.tenant != tenant || &key.project != project {
+                continue;
+            }
+            if retained_artifacts.contains(&key.artifact) {
                 metadata.retaining_nodes.insert(node.clone());
             } else {
                 metadata.retaining_nodes.remove(node);
@@ -249,8 +350,24 @@ impl ArtifactRegistry {
         }
     }
 
-    pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
-        self.artifacts.get(artifact)
+    pub fn metadata(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        artifact: &ArtifactId,
+    ) -> Option<&ArtifactMetadata> {
+        self.artifacts
+            .get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
+    }
+
+    pub fn metadata_for_project<'a>(
+        &'a self,
+        tenant: &'a TenantId,
+        project: &'a ProjectId,
+    ) -> impl Iterator<Item = &'a ArtifactMetadata> + 'a {
+        self.artifacts
+            .values()
+            .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project)
     }
 
     pub fn download_action(
@@ -261,7 +378,11 @@ impl ArtifactRegistry {
     ) -> Result<DownloadAction, DownloadError> {
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let scope = Scope {
             tenant: metadata.tenant.clone(),
@@ -315,7 +436,11 @@ impl ArtifactRegistry {
         self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         Ok(metadata.size)
     }
@@ -333,7 +458,11 @@ impl ArtifactRegistry {
         if self
             .issued_download_links
             .values()
-            .filter(|issued| issued.link.artifact == *artifact)
+            .filter(|issued| {
+                issued.link.tenant == context.tenant
+                    && issued.link.project == context.project
+                    && issued.link.artifact == *artifact
+            })
             .count()
             >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
         {
@@ -345,7 +474,11 @@ impl ArtifactRegistry {
         let action = self.download_action(context, artifact, policy)?;
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
         let policy_context_digest =
@@ -398,7 +531,11 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact || issued.link.actor != context.actor {
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
+            || issued.link.actor != context.actor
+        {
             return Err(DownloadError::InvalidToken);
         }
         self.download_action(
@@ -434,7 +571,9 @@ impl ArtifactRegistry {
             .issued_download_links
             .get(presented_token_digest)
             .ok_or(DownloadError::InvalidToken)?;
-        if issued.link.artifact != *artifact
+        if issued.link.tenant != context.tenant
+            || issued.link.project != context.project
+            || issued.link.artifact != *artifact
             || issued.link.max_bytes != policy.max_bytes
             || issued.link.actor != context.actor
         {
@@ -452,7 +591,11 @@ impl ArtifactRegistry {
         }
         let metadata = self
             .artifacts
-            .get(artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &context.tenant,
+                &context.project,
+                artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if download_policy_context_digest(metadata, &action.source, policy)
             != issued.link.policy_context_digest
@@ -477,7 +620,11 @@ impl ArtifactRegistry {
     ) -> Result<(), DownloadError> {
         let metadata = self
             .artifacts
-            .get(&stream.link.artifact)
+            .get(&ArtifactScopeKey::from_refs(
+                &stream.link.tenant,
+                &stream.link.project,
+                &stream.link.artifact,
+            ))
             .ok_or(DownloadError::NotFound)?;
         if !source_is_available(metadata, &stream.link.source) {
             return Err(DownloadError::Unavailable);
@@ -596,7 +743,13 @@ mod tests {
     #[test]
     fn flush_publishes_metadata_without_coordinator_bytes() {
         let registry = registry_with_artifact();
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(!metadata.coordinator_has_large_bytes);
         assert_eq!(metadata.id, ArtifactId::from("artifact"));
@@ -615,7 +768,11 @@ mod tests {
     #[test]
     fn unsynced_node_loss_surfaces_as_unavailable() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -636,9 +793,20 @@ mod tests {
     #[test]
     fn signed_node_retention_inventory_removes_garbage_collected_location() {
         let mut registry = registry_with_artifact();
-        registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+            &BTreeSet::new(),
+        );
 
-        let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+            )
+            .unwrap();
 
         assert!(metadata.retaining_nodes.is_empty());
     }
@@ -648,11 +816,26 @@ mod tests {
         let mut registry = registry_with_artifact();
         let node = NodeId::from("node");
         let artifact = ArtifactId::from("artifact");
-        registry.garbage_collect_node(&node);
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+        );
 
-        registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()]));
+        registry.reconcile_node_retention(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &node,
+            &BTreeSet::from([artifact.clone()]),
+        );
 
-        let metadata = registry.metadata(&artifact).unwrap();
+        let metadata = registry
+            .metadata(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &artifact,
+            )
+            .unwrap();
         assert!(metadata.retaining_nodes.contains(&node));
     }
 
@@ -660,9 +843,18 @@ mod tests {
     fn explicit_user_storage_location_survives_node_retention_loss() {
         let mut registry = registry_with_artifact();
         registry
-            .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
+            .sync_to_explicit_store(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                &ArtifactId::from("artifact"),
+                "s3://bucket/app",
+            )
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -683,7 +875,11 @@ mod tests {
         );
         assert!(
             !registry
-                .metadata(&ArtifactId::from("artifact"))
+                .metadata(
+                    &TenantId::from("tenant"),
+                    &ProjectId::from("project"),
+                    &ArtifactId::from("artifact"),
+                )
                 .unwrap()
                 .coordinator_has_large_bytes
         );
@@ -714,7 +910,7 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
     }
 
     #[test]
@@ -734,13 +930,206 @@ mod tests {
             )
             .unwrap_err();
 
-        assert!(matches!(error, DownloadError::Unauthorized(_)));
+        assert_eq!(error, DownloadError::NotFound);
+    }
+
+    #[test]
+    fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
+        let artifact = ArtifactId::from("shared-artifact");
+        let tenant_a = TenantId::from("tenant-a");
+        let project_a = ProjectId::from("project-a");
+        let tenant_b = TenantId::from("tenant-b");
+        let project_b = ProjectId::from("project-b");
+        let project_c = ProjectId::from("project-c");
+        let node_a = NodeId::from("node-a");
+        let node_b = NodeId::from("node-b");
+        let node_c = NodeId::from("node-c");
+        let mut registry = ArtifactRegistry::default();
+        for (tenant, project, process, node, content, size) in [
+            (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
+            (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
+            (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
+        ] {
+            registry.flush_metadata(ArtifactFlush {
+                id: artifact.clone(),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: ProcessId::from(process),
+                producer_task: TaskInstanceId::from("producer"),
+                retaining_node: node.clone(),
+                digest: Digest::sha256(content),
+                size,
+            });
+        }
+
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_c, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-c")
+        );
+        registry.flush_metadata(ArtifactFlush {
+            id: artifact.clone(),
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            process: ProcessId::from("process-a"),
+            producer_task: TaskInstanceId::from("producer-repeat"),
+            retaining_node: node_a.clone(),
+            digest: Digest::sha256("bytes-a-repeat"),
+            size: 18,
+        });
+        assert_eq!(registry.artifact_count(), 3);
+        assert_eq!(
+            registry
+                .metadata(&tenant_a, &project_a, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-a-repeat")
+        );
+        assert_eq!(
+            registry
+                .metadata(&tenant_b, &project_b, &artifact)
+                .unwrap()
+                .digest,
+            Digest::sha256("bytes-b")
+        );
+
+        registry
+            .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
+            .unwrap();
+        registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
+        let context_a = AuthContext {
+            tenant: tenant_a.clone(),
+            project: project_a.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let context_b = AuthContext {
+            tenant: tenant_b.clone(),
+            project: project_b.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let policy = DownloadPolicy { max_bytes: 100 };
+        assert_eq!(
+            registry
+                .download_action(&context_a, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::ExplicitStore("store://tenant-a".to_owned())
+        );
+        assert_eq!(
+            registry
+                .download_action(&context_b, &artifact, &policy)
+                .unwrap()
+                .source,
+            StorageLocation::RetainedNode(node_b)
+        );
+
+        let first_a = registry
+            .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        let first_b = registry
+            .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
+            .unwrap();
+        assert_ne!(
+            first_a.scoped_token_digest, first_b.scoped_token_digest,
+            "the full artifact scope must contribute to download tokens"
+        );
+        for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
+            registry
+                .create_download_link(
+                    &context_a,
+                    &artifact,
+                    &policy,
+                    &format!("tenant-a-{index}"),
+                    10,
+                    60,
+                )
+                .unwrap();
+        }
+        assert!(matches!(
+            registry.create_download_link(
+                &context_a,
+                &artifact,
+                &policy,
+                "tenant-a-over-limit",
+                10,
+                60,
+            ),
+            Err(DownloadError::Usage(_))
+        ));
+        registry
+            .create_download_link(
+                &context_b,
+                &artifact,
+                &policy,
+                "tenant-b-still-independent",
+                10,
+                60,
+            )
+            .unwrap();
+        registry
+            .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
+            .unwrap();
+
+        let limits = ResourceLimits {
+            limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
+        };
+        let mut meter = ResourceMeter::default();
+        registry
+            .open_download_stream(
+                DownloadStreamRequest {
+                    context: &context_b,
+                    artifact: &artifact,
+                    policy: &policy,
+                    presented_token_digest: &first_b.scoped_token_digest,
+                    now_epoch_seconds: 11,
+                    limits: &limits,
+                },
+                &mut meter,
+            )
+            .expect("revoking tenant A's link must not revoke tenant B's link");
+        assert_eq!(
+            registry
+                .open_download_stream(
+                    DownloadStreamRequest {
+                        context: &context_b,
+                        artifact: &artifact,
+                        policy: &policy,
+                        presented_token_digest: &first_a.scoped_token_digest,
+                        now_epoch_seconds: 11,
+                        limits: &limits,
+                    },
+                    &mut meter,
+                )
+                .unwrap_err(),
+            DownloadError::InvalidToken
+        );
     }
 
     #[test]
     fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
         let mut registry = registry_with_artifact();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
         let context = AuthContext {
             tenant: TenantId::from("tenant"),
             project: ProjectId::from("project"),
@@ -1017,51 +1406,200 @@ mod tests {
     }
 
     #[test]
-    fn artifact_metadata_is_bounded_without_evicting_pins() {
+    fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
         let mut registry = ArtifactRegistry::default();
-        let mut pinned = BTreeSet::new();
-        for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
-            let id = ArtifactId::new(format!("artifact-{index}"));
-            pinned.insert(id.clone());
-            registry
-                .flush_metadata_bounded(
-                    ArtifactFlush {
-                        id,
-                        tenant: TenantId::from("tenant"),
-                        project: ProjectId::from("project"),
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        registry.flush_metadata(ArtifactFlush {
+            id: ArtifactId::from("artifact-0"),
+            tenant: TenantId::from("other-tenant"),
+            project: ProjectId::from("other-project"),
             process: ProcessId::from("process"),
+            producer_task: TaskInstanceId::from("other-task"),
+            retaining_node: NodeId::from("other-node"),
+            digest: Digest::sha256("other-content"),
+            size: 1,
+        });
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
+            let id = ArtifactId::new(format!("artifact-{index}"));
+            registry.flush_metadata(ArtifactFlush {
+                id,
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: ProcessId::new(if index == 3 {
+                    "active-process-3".to_owned()
+                } else {
+                    format!("completed-process-{index}")
+                }),
                 producer_task: TaskInstanceId::new(format!("task-{index}")),
                 retaining_node: NodeId::from("node"),
                 digest: Digest::sha256(format!("content-{index}")),
                 size: 1,
-                    },
-                    &pinned,
+            });
+        }
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        let pinned = BTreeSet::from([ArtifactScopeKey::new(
+            tenant.clone(),
+            project.clone(),
+            ArtifactId::from("artifact-0"),
+        )]);
+        registry
+            .sync_to_explicit_store(
+                &tenant,
+                &project,
+                &ArtifactId::from("artifact-1"),
+                "store://retained-export",
             )
             .unwrap();
-        }
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
+        let context = AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(UserId::from("user")),
+        };
+        registry
+            .create_download_link(
+                &context,
+                &ArtifactId::from("artifact-2"),
+                &DownloadPolicy { max_bytes: 1 },
+                "active-download",
+                10,
+                60,
+            )
+            .unwrap();
+        let protected_processes = BTreeSet::from([
+            ProcessId::from("active-process-3"),
+            ProcessId::from("active-process"),
+        ]);
         let next = ArtifactFlush {
             id: ArtifactId::from("artifact-next"),
-            tenant: TenantId::from("tenant"),
-            project: ProjectId::from("project"),
-            process: ProcessId::from("process"),
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: ProcessId::from("active-process"),
             producer_task: TaskInstanceId::from("task-next"),
             retaining_node: NodeId::from("node"),
             digest: Digest::sha256("next"),
             size: 1,
         };
+        registry
+            .flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
+            .unwrap();
+        assert_eq!(
+            registry.artifact_count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT + 1
+        );
+        for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
+            assert!(
+                registry
+                    .metadata(&tenant, &project, &ArtifactId::from(id))
+                    .is_some(),
+                "{id} was evicted despite being pinned, exported, downloaded, or live"
+            );
+        }
+        assert!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
+                .is_none(),
+            "the oldest completed unprotected metadata should be evicted"
+        );
         assert!(registry
-            .flush_metadata_bounded(next.clone(), &pinned)
-            .unwrap_err()
-            .contains("pinned"));
-
-        pinned.remove(&ArtifactId::from("artifact-0"));
-        registry.flush_metadata_bounded(next, &pinned).unwrap();
-        assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
-        assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
-        assert!(registry
-            .metadata(&ArtifactId::from("artifact-next"))
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
             .is_some());
+        assert_eq!(
+            registry
+                .metadata(
+                    &TenantId::from("other-tenant"),
+                    &ProjectId::from("other-project"),
+                    &ArtifactId::from("artifact-0"),
+                )
+                .unwrap()
+                .digest,
+            Digest::sha256("other-content"),
+            "eviction and pins in one scope must not affect an equal ID in another scope"
+        );
+    }
+
+    #[test]
+    fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
+        let mut registry = ArtifactRegistry::default();
+        let tenant = TenantId::from("tenant");
+        let project = ProjectId::from("project");
+        let active_process = ProcessId::from("active-process");
+        for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
+            registry.flush_metadata(ArtifactFlush {
+                id: ArtifactId::new(format!("artifact-{index}")),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::new(format!("task-{index}")),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256(format!("content-{index}")),
+                size: 1,
+            });
+        }
+
+        let replacement = ArtifactFlush {
+            id: ArtifactId::from("artifact-0"),
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: active_process.clone(),
+            producer_task: TaskInstanceId::from("replacement-task"),
+            retaining_node: NodeId::from("node"),
+            digest: Digest::sha256("replacement"),
+            size: 2,
+        };
+        registry
+            .flush_metadata_with_protected_processes(
+                replacement,
+                &BTreeSet::new(),
+                &BTreeSet::from([active_process.clone()]),
+            )
+            .expect("replacement must remain possible at the metadata bound");
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement")
+        );
+
+        let epoch_before_rejection = registry.next_epoch;
+        let result = registry.flush_metadata_with_protected_processes(
+            ArtifactFlush {
+                id: ArtifactId::from("artifact-rejected"),
+                tenant: tenant.clone(),
+                project: project.clone(),
+                process: active_process.clone(),
+                producer_task: TaskInstanceId::from("rejected-task"),
+                retaining_node: NodeId::from("node"),
+                digest: Digest::sha256("rejected"),
+                size: 3,
+            },
+            &BTreeSet::new(),
+            &BTreeSet::from([active_process]),
+        );
+
+        assert!(result
+            .unwrap_err()
+            .contains("exhausted by active or retained artifacts"));
+        assert_eq!(registry.next_epoch, epoch_before_rejection);
+        assert_eq!(
+            registry.metadata_for_project(&tenant, &project).count(),
+            MAX_ARTIFACT_METADATA_PER_PROJECT
+        );
+        assert!(registry
+            .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
+            .is_none());
+        assert_eq!(
+            registry
+                .metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
+                .unwrap()
+                .digest,
+            Digest::sha256("replacement"),
+            "rejection must not modify existing metadata"
+        );
     }
 
     #[test]
@@ -1150,7 +1688,11 @@ mod tests {
         registry
             .stream_download_chunk(&mut stream, &limits, &mut meter, 16)
             .unwrap();
-        registry.garbage_collect_node(&NodeId::from("node"));
+        registry.garbage_collect_node(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            &NodeId::from("node"),
+        );
 
         assert_eq!(
             registry

crates/clusterflux-core/src/auth.rs

@@ -364,7 +364,13 @@ pub fn agent_workflow_request_scope_from_payload(
                 .get("task_instance")
                 .and_then(Value::as_str)
                 .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
-            (process, Some(TaskInstanceId::from(task)))
+            (
+                process,
+                Some(
+                    TaskInstanceId::try_new(task)
+                        .map_err(|error| format!("malformed launch_task task instance: {error}"))?,
+                ),
+            )
         }
         _ => {
             return Err(format!(
@@ -373,10 +379,13 @@ pub fn agent_workflow_request_scope_from_payload(
         }
     };
     AgentWorkflowRequestScope::new(
-        TenantId::from(tenant),
-        ProjectId::from(project),
+        TenantId::try_new(tenant)
+            .map_err(|error| format!("malformed agent workflow tenant: {error}"))?,
+        ProjectId::try_new(project)
+            .map_err(|error| format!("malformed agent workflow project: {error}"))?,
         request_kind,
-        ProcessId::from(process),
+        ProcessId::try_new(process)
+            .map_err(|error| format!("malformed agent workflow process: {error}"))?,
         task,
     )
 }

crates/clusterflux-core/src/ids.rs

@@ -1,18 +1,102 @@
+#[cfg(not(target_arch = "wasm32"))]
+use serde::{de::Error as _, Deserializer};
 use serde::{Deserialize, Serialize};
 
+pub const MAX_EXTERNAL_ID_BYTES: usize = 255;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OpaqueTokenError {
+    reason: String,
+}
+
+impl std::fmt::Display for OpaqueTokenError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.reason)
+    }
+}
+
+impl std::error::Error for OpaqueTokenError {}
+
+pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> {
+    let reason = if value.trim().is_empty() {
+        Some("value must not be empty or whitespace-only".to_owned())
+    } else if value.len() > max_bytes {
+        Some(format!("value exceeds the {max_bytes}-byte limit"))
+    } else if value.chars().any(char::is_control) {
+        Some("control characters are forbidden".to_owned())
+    } else {
+        None
+    };
+    reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason }))
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct IdParseError {
+    id_type: &'static str,
+    reason: &'static str,
+}
+
+impl IdParseError {
+    fn new(id_type: &'static str, reason: &'static str) -> Self {
+        Self { id_type, reason }
+    }
+}
+
+impl std::fmt::Display for IdParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} is invalid: {}", self.id_type, self.reason)
+    }
+}
+
+impl std::error::Error for IdParseError {}
+
+fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> {
+    if value.is_empty() || value.trim().is_empty() {
+        return Err(IdParseError::new(
+            id_type,
+            "value must not be empty or whitespace-only",
+        ));
+    }
+    if value.len() > MAX_EXTERNAL_ID_BYTES {
+        return Err(IdParseError::new(
+            id_type,
+            "value exceeds the 255-byte limit",
+        ));
+    }
+    if value.chars().any(char::is_control) {
+        return Err(IdParseError::new(
+            id_type,
+            "control characters are forbidden",
+        ));
+    }
+    if !value.bytes().all(|byte| {
+        byte.is_ascii_alphanumeric()
+            || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+')
+    }) {
+        return Err(IdParseError::new(
+            id_type,
+            "only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed",
+        ));
+    }
+    Ok(())
+}
+
 macro_rules! id_type {
     ($name:ident) => {
-        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+        #[cfg_attr(target_arch = "wasm32", derive(Deserialize))]
+        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
         pub struct $name(String);
 
         impl $name {
-            pub fn new(value: impl Into<String>) -> Self {
+            pub fn try_new(value: impl Into<String>) -> Result<Self, IdParseError> {
                 let value = value.into();
-                assert!(
-                    !value.trim().is_empty(),
-                    concat!(stringify!($name), " cannot be empty")
-                );
-                Self(value)
+                validate_id(&value, stringify!($name))?;
+                Ok(Self(value))
+            }
+
+            /// Constructs an identifier from a trusted, internally generated value.
+            pub fn new(value: impl Into<String>) -> Self {
+                Self::try_new(value).unwrap_or_else(|error| panic!("{error}"))
             }
 
             pub fn as_str(&self) -> &str {
@@ -26,6 +110,17 @@ macro_rules! id_type {
             }
         }
 
+        #[cfg(not(target_arch = "wasm32"))]
+        impl<'de> Deserialize<'de> for $name {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: Deserializer<'de>,
+            {
+                let value = String::deserialize(deserializer)?;
+                Self::try_new(value).map_err(D::Error::custom)
+            }
+        }
+
         impl std::fmt::Display for $name {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                 f.write_str(&self.0)
@@ -44,3 +139,73 @@ id_type!(TaskDefinitionId);
 id_type!(TaskInstanceId);
 id_type!(TenantId);
 id_type!(UserId);
+
+pub type DebugSessionId = ProcessId;
+pub type RequestId = LaunchAttemptId;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_hostile_values<T>(parse: impl Fn(String) -> Result<T, IdParseError>) {
+        for value in [
+            String::new(),
+            "   ".to_owned(),
+            "bad\0id".to_owned(),
+            "bad id".to_owned(),
+            "x".repeat(MAX_EXTERNAL_ID_BYTES + 1),
+        ] {
+            assert!(parse(value).is_err());
+        }
+    }
+
+    #[test]
+    fn every_external_identifier_type_rejects_hostile_values() {
+        assert_hostile_values(AgentId::try_new);
+        assert_hostile_values(ArtifactId::try_new);
+        assert_hostile_values(DebugSessionId::try_new);
+        assert_hostile_values(NodeId::try_new);
+        assert_hostile_values(ProcessId::try_new);
+        assert_hostile_values(LaunchAttemptId::try_new);
+        assert_hostile_values(ProjectId::try_new);
+        assert_hostile_values(RequestId::try_new);
+        assert_hostile_values(TaskDefinitionId::try_new);
+        assert_hostile_values(TaskInstanceId::try_new);
+        assert_hostile_values(TenantId::try_new);
+        assert_hostile_values(UserId::try_new);
+    }
+
+    #[test]
+    fn every_identifier_type_validates_during_deserialization() {
+        macro_rules! assert_deserialization {
+            ($id:ty) => {
+                assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok());
+                let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err();
+                assert!(
+                    error.to_string().contains("is invalid"),
+                    "{} produced an unexpected error: {error}",
+                    stringify!($id)
+                );
+            };
+        }
+
+        assert_deserialization!(AgentId);
+        assert_deserialization!(ArtifactId);
+        assert_deserialization!(NodeId);
+        assert_deserialization!(ProcessId);
+        assert_deserialization!(LaunchAttemptId);
+        assert_deserialization!(ProjectId);
+        assert_deserialization!(TaskDefinitionId);
+        assert_deserialization!(TaskInstanceId);
+        assert_deserialization!(TenantId);
+        assert_deserialization!(UserId);
+    }
+
+    #[test]
+    fn opaque_tokens_are_bounded_without_using_identifier_syntax() {
+        validate_opaque_token("opaque secret/+==", 64).unwrap();
+        assert!(validate_opaque_token("", 64).is_err());
+        assert!(validate_opaque_token("bad\0token", 64).is_err());
+        assert!(validate_opaque_token(&"x".repeat(65), 64).is_err());
+    }
+}

crates/clusterflux-core/src/lib.rs

@@ -1,3 +1,4 @@
+mod api_error;
 pub mod artifact;
 pub mod auth;
 pub mod bundle;
@@ -18,10 +19,11 @@ pub mod transport;
 pub mod vfs;
 pub mod wire;
 
+pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode};
 pub use artifact::{
     ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
-    ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
-    DownloadStreamRequest, RetentionPolicy, StorageLocation,
+    ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
+    DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
 };
 pub use auth::{
     admin_request_proof, admin_request_proof_from_token_digest,
@@ -66,8 +68,9 @@ pub use execution::{
     WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
 };
 pub use ids::{
-    AgentId, ArtifactId, LaunchAttemptId, NodeId, ProcessId, ProjectId, TaskDefinitionId,
-    TaskInstanceId, TenantId, UserId,
+    validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId,
+    NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId,
+    TenantId, UserId, MAX_EXTERNAL_ID_BYTES,
 };
 pub use limits::{
     LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,

crates/clusterflux-core/src/vfs.rs

@@ -1,18 +1,48 @@
 use std::collections::BTreeMap;
 
-use serde::{Deserialize, Serialize};
+use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
 use thiserror::Error;
 
 use crate::{Digest, NodeId, TaskInstanceId};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub const MAX_VFS_PATH_BYTES: usize = 4096;
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
 pub struct VfsPath(String);
 
 impl VfsPath {
     pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
         let path = path.into();
         if !path.starts_with("/vfs/") {
-            return Err(VfsError::InvalidPath(path));
+            return Err(VfsError::invalid(path, "path must start with /vfs/"));
+        }
+        if path.len() > MAX_VFS_PATH_BYTES {
+            return Err(VfsError::invalid(
+                path,
+                "path exceeds the 4096-byte protocol limit",
+            ));
+        }
+        if path.chars().any(char::is_control) {
+            return Err(VfsError::invalid(path, "control characters are forbidden"));
+        }
+        if path.contains('\\') {
+            return Err(VfsError::invalid(path, "backslashes are forbidden"));
+        }
+        let relative = &path["/vfs/".len()..];
+        if relative.is_empty() {
+            return Err(VfsError::invalid(
+                path,
+                "path after /vfs/ must not be empty",
+            ));
+        }
+        if relative
+            .split('/')
+            .any(|component| component.is_empty() || matches!(component, "." | ".."))
+        {
+            return Err(VfsError::invalid(
+                path,
+                "empty, '.', and '..' path components are forbidden",
+            ));
         }
         Ok(Self(path))
     }
@@ -22,6 +52,16 @@ impl VfsPath {
     }
 }
 
+impl<'de> Deserialize<'de> for VfsPath {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let path = String::deserialize(deserializer)?;
+        Self::new(path).map_err(D::Error::custom)
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct VfsObject {
     pub path: VfsPath,
@@ -63,12 +103,18 @@ pub enum ReuseDecision {
 
 #[derive(Clone, Debug, Error, PartialEq, Eq)]
 pub enum VfsError {
-    #[error("VFS path must start with /vfs/: {0}")]
-    InvalidPath(String),
+    #[error("invalid VFS path {path:?}: {reason}")]
+    InvalidPath { path: String, reason: &'static str },
     #[error("path is not visible in the published VFS manifest: {0}")]
     NotVisible(String),
 }
 
+impl VfsError {
+    fn invalid(path: String, reason: &'static str) -> Self {
+        Self::InvalidPath { path, reason }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct VfsOverlay {
     task: TaskInstanceId,
@@ -238,4 +284,39 @@ mod tests {
 
         assert_eq!(overlay.pending_len(), 0);
     }
+
+    #[test]
+    fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
+        for invalid in [
+            "vfs/artifacts/app".to_owned(),
+            "/vfs/".to_owned(),
+            "/vfs/artifacts//app".to_owned(),
+            "/vfs/artifacts/app/".to_owned(),
+            "/vfs/artifacts/./app".to_owned(),
+            "/vfs/artifacts/../app".to_owned(),
+            "/vfs/artifacts\\app".to_owned(),
+            "/vfs/artifacts/bad\0app".to_owned(),
+            format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
+        ] {
+            assert!(
+                VfsPath::new(&invalid).is_err(),
+                "hostile VFS path unexpectedly passed: {invalid:?}"
+            );
+            assert!(
+                serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
+                "hostile VFS path unexpectedly deserialized"
+            );
+        }
+    }
+
+    #[test]
+    fn vfs_path_accepts_the_4096_byte_boundary() {
+        let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
+        assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
+        assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
+
+        let overlong = format!("{path}x");
+        assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
+        assert!(VfsPath::new(overlong).is_err());
+    }
 }

crates/clusterflux-dap/src/adapter.rs

@@ -55,6 +55,7 @@ enum AdapterEvent {
     },
     BreakpointsUpdated {
         generation: u64,
+        revision: u64,
         result: std::result::Result<(), String>,
     },
 }
@@ -62,6 +63,8 @@ enum AdapterEvent {
 struct LaunchCompletion {
     state: AdapterState,
     local_runtime_session: Option<LocalRuntimeSession>,
+    breakpoint_revision: u64,
+    observation_diagnostics: Vec<String>,
 }
 
 pub(crate) fn run_adapter() -> Result<()> {
@@ -93,16 +96,41 @@ pub(crate) fn run_adapter() -> Result<()> {
                 runtime_starting = false;
                 match result {
                     Ok(mut completion) => {
-                        completion.state.breakpoints = state.breakpoints.clone();
-                        completion.state.breakpoints_installed = true;
+                        let requested_breakpoints = state.breakpoints.clone();
+                        let requested_revision = state.breakpoint_revision;
+                        let installed_revision = completion.breakpoint_revision;
+                        completion.state.breakpoints = requested_breakpoints;
+                        completion.state.breakpoint_revision = requested_revision;
+                        completion.state.breakpoints_installed =
+                            installed_revision == requested_revision;
                         let previous_threads = state.threads.clone();
                         state = completion.state;
                         emit_thread_lifecycle(&mut writer, &previous_threads, &state.threads)?;
                         if let Some(session) = completion.local_runtime_session {
                             _local_runtime_session = Some(session);
                         }
+                        for diagnostic in completion.observation_diagnostics {
+                            writer.output("stderr", format!("{diagnostic}\n"))?;
+                        }
                         runtime_started = true;
+                        if state.breakpoints_installed {
                             emit_verified_breakpoints(&mut writer, &state)?;
+                        } else {
+                            let breakpoint_state = state.clone();
+                            let breakpoint_tx = event_tx.clone();
+                            let revision = state.breakpoint_revision;
+                            thread::spawn(move || {
+                                let result = set_services_debug_breakpoints(&breakpoint_state)
+                                    .map_err(|error| {
+                                        format!("coordinator breakpoint update failed: {error:#}")
+                                    });
+                                let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
+                                    generation,
+                                    revision,
+                                    result,
+                                });
+                            });
+                        }
                         writer.output(
                             "console",
                             if state.session_mode == DapSessionMode::Attach {
@@ -127,15 +155,7 @@ pub(crate) fn run_adapter() -> Result<()> {
                     Err(message) => {
                         runtime_started = false;
                         writer.output("stderr", format!("{message}\n"))?;
-                        writer.event(
-                            "stopped",
-                            json!({
-                                "reason": "exception",
-                                "description": message,
-                                "threadId": MAIN_THREAD,
-                                "allThreadsStopped": false,
-                            }),
-                        )?;
+                        writer.event("terminated", json!({ "restart": false }))?;
                     }
                 }
                 continue;
@@ -231,8 +251,17 @@ pub(crate) fn run_adapter() -> Result<()> {
                 }
                 continue;
             }
-            AdapterEvent::BreakpointsUpdated { generation, result } => {
-                if generation == runtime_generation {
+            AdapterEvent::BreakpointsUpdated {
+                generation,
+                revision,
+                result,
+            } => {
+                if breakpoint_update_is_current(
+                    generation,
+                    runtime_generation,
+                    revision,
+                    state.breakpoint_revision,
+                ) {
                     match result {
                         Ok(()) => {
                             state.breakpoints_installed = true;
@@ -306,7 +335,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                     .and_then(Value::as_bool)
                     .unwrap_or(false);
                 if let Some(process_id) = args.get("processId").and_then(Value::as_str) {
-                    state.process = clusterflux_core::ProcessId::new(process_id);
+                    match clusterflux_core::ProcessId::try_new(process_id.to_owned()) {
+                        Ok(process) => state.process = process,
+                        Err(error) => {
+                            writer.error_response(
+                                &request,
+                                format!("invalid DAP processId: {error}"),
+                            )?;
+                            continue;
+                        }
+                    }
                 }
                 if command == "attach" {
                     state.session_mode = DapSessionMode::Attach;
@@ -338,6 +376,8 @@ pub(crate) fn run_adapter() -> Result<()> {
                     requested_source_path,
                     requested_lines,
                 );
+                state.breakpoint_revision = state.breakpoint_revision.saturating_add(1);
+                let breakpoint_revision = state.breakpoint_revision;
                 let coordinator_backed = matches!(
                     state.runtime_backend,
                     RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
@@ -371,8 +411,11 @@ pub(crate) fn run_adapter() -> Result<()> {
                             set_services_debug_breakpoints(&breakpoint_state).map_err(|error| {
                                 format!("coordinator breakpoint update failed: {error:#}")
                             });
-                        let _ = breakpoint_tx
-                            .send(AdapterEvent::BreakpointsUpdated { generation, result });
+                        let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
+                            generation,
+                            revision: breakpoint_revision,
+                            result,
+                        });
                     });
                 }
             }
@@ -433,11 +476,18 @@ pub(crate) fn run_adapter() -> Result<()> {
                     let mut completed_state = launch_state;
                     let result = if completed_state.session_mode == DapSessionMode::Attach {
                         attach_services_runtime(&completed_state)
-                            .map(|record| {
+                            .and_then(|record| {
                                 completed_state.apply_attach_record(record);
+                                set_services_debug_breakpoints(&completed_state)?;
+                                Ok(())
+                            })
+                            .map(|()| {
+                                let breakpoint_revision = completed_state.breakpoint_revision;
                                 LaunchCompletion {
                                     state: completed_state,
                                     local_runtime_session: None,
+                                    breakpoint_revision,
+                                    observation_diagnostics: Vec::new(),
                                 }
                             })
                             .map_err(|error| format!("services runtime attach failed: {error:#}"))
@@ -446,10 +496,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                             RuntimeBackend::LocalServices => {
                                 run_local_services_runtime(&completed_state)
                                     .map(|(record, session)| {
+                                        let observation_diagnostics =
+                                            runtime_observation_diagnostics(&record);
                                         completed_state.apply_runtime_record(record);
+                                        let breakpoint_revision =
+                                            completed_state.breakpoint_revision;
                                         LaunchCompletion {
                                             state: completed_state,
                                             local_runtime_session: Some(session),
+                                            breakpoint_revision,
+                                            observation_diagnostics,
                                         }
                                     })
                                     .map_err(|error| {
@@ -459,10 +515,16 @@ pub(crate) fn run_adapter() -> Result<()> {
                             RuntimeBackend::LiveServices => {
                                 run_live_services_runtime(&completed_state)
                                     .map(|record| {
+                                        let observation_diagnostics =
+                                            runtime_observation_diagnostics(&record);
                                         completed_state.apply_runtime_record(record);
+                                        let breakpoint_revision =
+                                            completed_state.breakpoint_revision;
                                         LaunchCompletion {
                                             state: completed_state,
                                             local_runtime_session: None,
+                                            breakpoint_revision,
+                                            observation_diagnostics,
                                         }
                                     })
                                     .map_err(|error| {
@@ -1005,6 +1067,29 @@ fn spawn_runtime_observer(
     });
 }
 
+pub(crate) fn breakpoint_update_is_current(
+    update_generation: u64,
+    runtime_generation: u64,
+    update_revision: u64,
+    current_revision: u64,
+) -> bool {
+    update_generation == runtime_generation && update_revision == current_revision
+}
+
+fn runtime_observation_diagnostics(
+    record: &crate::virtual_model::RuntimeLaunchRecord,
+) -> Vec<String> {
+    record
+        .node_report
+        .get("observation_diagnostics")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .filter_map(Value::as_str)
+        .map(str::to_owned)
+        .collect()
+}
+
 fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> {
     if !state.breakpoints_installed {
         return Ok(());
@@ -1123,10 +1208,12 @@ fn emit_runtime_outcome(
             )?;
         }
         RuntimeContinuationOutcome::Terminal(record) => {
+            let exit_code = record.status_code.unwrap_or(1);
             apply_runtime_record_with_thread_events(writer, state, record)?;
             if state.last_task_failed {
                 writer.output("stderr", format!("{}\n", state.command_status))?;
             }
+            writer.event("exited", json!({ "exitCode": exit_code }))?;
             writer.event("terminated", json!({}))?;
         }
     }

crates/clusterflux-dap/src/main.rs

@@ -22,7 +22,7 @@ use breakpoints::{
 #[cfg(test)]
 use dap_protocol::{initialize_capabilities, read_message};
 #[cfg(test)]
-use runtime_client::{client_user_request, parse_task_restart_response};
+use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code};
 #[cfg(test)]
 use variables::variables_response;
 #[cfg(test)]
@@ -36,6 +36,26 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
 use virtual_model::{process_id, RuntimeBackend};
 
 fn main() -> Result<()> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux Debug Adapter Protocol server.\n\n\
+                 Usage: clusterflux-debug-dap\n\n\
+                 The adapter communicates over standard input and output.\n\n\
+                 Options:\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        [] => {}
+        [argument, ..] => anyhow::bail!("unknown argument: {argument}"),
+    }
     adapter::run_adapter()
 }
 

crates/clusterflux-dap/src/runtime_client.rs

@@ -422,8 +422,31 @@ fn launch_services_debug_entrypoint(
     }
     // The launch acknowledgement is the commit point. Failures while fetching
     // observation state after this line must not abort a process that is running.
-    let task_snapshots = fetch_task_snapshots(coordinator, state)?;
-    let (process_statuses, process_status) = fetch_current_process_status(coordinator, state)?;
+    let mut observation_diagnostics = Vec::new();
+    let inject_post_commit_observation_failure =
+        std::env::var_os("CLUSTERFLUX_TEST_DAP_POST_COMMIT_OBSERVATION_FAILURE").is_some();
+    let task_snapshots = (if inject_post_commit_observation_failure {
+        Err(anyhow!("injected post-commit task observation failure"))
+    } else {
+        fetch_task_snapshots(coordinator, state)
+    })
+    .unwrap_or_else(|error| {
+        observation_diagnostics.push(format!(
+            "initial task observation failed after main_launched: {error:#}"
+        ));
+        json!({ "snapshots": [] })
+    });
+    let (process_statuses, process_status) = (if inject_post_commit_observation_failure {
+        Err(anyhow!("injected post-commit process observation failure"))
+    } else {
+        fetch_current_process_status(coordinator, state)
+    })
+    .unwrap_or_else(|error| {
+        observation_diagnostics.push(format!(
+            "initial process observation failed after main_launched: {error:#}"
+        ));
+        (json!({ "processes": [] }), None)
+    });
     let node = process_status
         .as_ref()
         .and_then(|status| status.get("connected_nodes"))
@@ -441,6 +464,7 @@ fn launch_services_debug_entrypoint(
             "process_status": process_status,
             "process_statuses": process_statuses,
             "task_snapshots": task_snapshots,
+            "observation_diagnostics": observation_diagnostics,
         }),
         task_events: json!({ "events": [] }),
         placed_task_launched: true,
@@ -515,7 +539,24 @@ fn new_launch_attempt_id() -> String {
 
 fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> Result<()> {
     static INJECTED_INSTALL_FAILURE: AtomicBool = AtomicBool::new(false);
-    if std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some()
+    if std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION")
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        == Some(state.breakpoint_revision)
+    {
+        let delay_ms = std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .unwrap_or(750);
+        std::thread::sleep(Duration::from_millis(delay_ms));
+    }
+    let revision_failure =
+        std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            == Some(state.breakpoint_revision);
+    if (revision_failure
+        || std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some())
         && !state.breakpoints.is_empty()
         && !INJECTED_INSTALL_FAILURE.swap(true, Ordering::SeqCst)
     {
@@ -533,6 +574,7 @@ fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) ->
                 "project": state.project_id,
                 "actor_user": state.actor_user,
                 "process": state.process.as_str(),
+                "revision": state.breakpoint_revision,
                 "probe_symbols": state.requested_probe_symbols(),
             }),
         ),
@@ -700,14 +742,16 @@ fn fetch_current_process_status(
         client_user_request(
             state,
             json!({
-                "type": "list_processes",
+                "type": "list_process_summaries",
                 "tenant": state.tenant,
                 "project": state.project_id,
                 "actor_user": state.actor_user,
+                "cursor": null,
+                "limit": 100,
             }),
         ),
     )?;
-    let current = statuses
+    let summary = statuses
         .get("processes")
         .and_then(Value::as_array)
         .and_then(|processes| {
@@ -716,6 +760,9 @@ fn fetch_current_process_status(
             })
         })
         .cloned();
+    let current = merge_active_process_status(state, summary, |request| {
+        coordinator_request(coordinator, request)
+    })?;
     Ok((statuses, current))
 }
 
@@ -726,13 +773,15 @@ fn fetch_current_process_status_in(
     let statuses = session.request(client_user_request(
         state,
         json!({
-            "type": "list_processes",
+            "type": "list_process_summaries",
             "tenant": state.tenant,
             "project": state.project_id,
             "actor_user": state.actor_user,
+            "cursor": null,
+            "limit": 100,
         }),
     ))?;
-    let current = statuses
+    let summary = statuses
         .get("processes")
         .and_then(Value::as_array)
         .and_then(|processes| {
@@ -741,9 +790,51 @@ fn fetch_current_process_status_in(
             })
         })
         .cloned();
+    let current = merge_active_process_status(state, summary, |request| session.request(request))?;
     Ok((statuses, current))
 }
 
+fn merge_active_process_status<F>(
+    state: &AdapterState,
+    summary: Option<Value>,
+    mut request: F,
+) -> Result<Option<Value>>
+where
+    F: FnMut(Value) -> Result<Value>,
+{
+    let Some(mut summary) = summary else {
+        return Ok(None);
+    };
+    if summary.get("lifecycle").and_then(Value::as_str) != Some("active") {
+        return Ok(Some(summary));
+    }
+    let active_statuses = request(client_user_request(
+        state,
+        json!({
+            "type": "list_processes",
+            "tenant": state.tenant,
+            "project": state.project_id,
+            "actor_user": state.actor_user,
+        }),
+    ))?;
+    let active = active_statuses
+        .get("processes")
+        .and_then(Value::as_array)
+        .and_then(|processes| {
+            processes.iter().find(|process| {
+                process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
+            })
+        });
+    if let (Some(summary), Some(active)) =
+        (summary.as_object_mut(), active.and_then(Value::as_object))
+    {
+        for (key, value) in active {
+            summary.entry(key.clone()).or_insert_with(|| value.clone());
+        }
+    }
+    Ok(Some(summary))
+}
+
 pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> {
     let coordinator =
         crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
@@ -817,27 +908,7 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result<RuntimeLau
                 .count()
         })
         .unwrap_or(0);
-    let process_statuses = coordinator_request(
-        &coordinator,
-        client_user_request(
-            state,
-            json!({
-                "type": "list_processes",
-                "tenant": state.tenant,
-                "project": state.project_id,
-                "actor_user": state.actor_user,
-            }),
-        ),
-    )?;
-    let process_status = process_statuses
-        .get("processes")
-        .and_then(Value::as_array)
-        .and_then(|processes| {
-            processes.iter().find(|process| {
-                process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
-            })
-        })
-        .cloned();
+    let (process_statuses, process_status) = fetch_current_process_status(&coordinator, state)?;
     let node = process_status
         .as_ref()
         .and_then(|status| status.get("connected_nodes"))
@@ -1002,6 +1073,18 @@ pub(crate) fn observe_services_runtime(
     let inject_connection_loss =
         std::env::var("CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS").as_deref() == Ok("1");
     let mut connection_loss_injected = false;
+    let inject_snapshot_failure =
+        std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE").is_some();
+    let mut snapshot_failure_injected = false;
+    let inject_process_status_failure =
+        std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE").is_some();
+    let mut process_status_failure_injected = false;
+    let inject_fallback_failure =
+        std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE").is_some();
+    let mut fallback_failure_stage = 0_u8;
+    let inject_debug_epoch_wait_failure =
+        std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_DEBUG_EPOCH_WAIT_FAILURE").is_some();
+    let mut debug_epoch_wait_failure_injected = false;
     loop {
         if cancelled.load(Ordering::Acquire) {
             return Ok(());
@@ -1025,9 +1108,8 @@ pub(crate) fn observe_services_runtime(
                 }
             }
         }
-        // Terminal task events remain inspectable after the active process slot is
-        // released. Read them before active-process debug state so a fast main exit
-        // cannot turn a successful continuation into an authorization error.
+        // Events remain useful for output and attempt details, but terminal
+        // authority comes from the durable process summary read below.
         let current_session = session.as_mut().expect("observer session connected");
         let events = match current_session.request(client_user_request(
             state,
@@ -1064,24 +1146,13 @@ pub(crate) fn observe_services_runtime(
             std::thread::sleep(reconnect_delay);
             continue;
         }
-        if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
-            let task_snapshots = fetch_task_snapshots_in(current_session, state)
-                .unwrap_or_else(|_| json!({ "snapshots": [] }));
-            let (process_statuses, process_status) =
-                fetch_current_process_status_in(current_session, state)
-                    .unwrap_or_else(|_| (json!({ "processes": [] }), None));
-            if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
-                record.node_report = json!({
-                    "terminal_event": record.node_report.get("terminal_event"),
-                    "task_snapshots": task_snapshots,
-                    "process_status": process_status,
-                    "process_statuses": process_statuses,
-                });
-            }
-            emit(outcome);
-            return Ok(());
-        }
-        let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
+        let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
+            snapshot_failure_injected = true;
+            Err(anyhow!("injected task snapshot transport failure"))
+        } else {
+            fetch_task_snapshots_in(current_session, state)
+        };
+        let task_snapshots = match snapshot_request {
             Ok(snapshots) => snapshots,
             Err(error) => {
                 if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
@@ -1096,8 +1167,14 @@ pub(crate) fn observe_services_runtime(
                 continue;
             }
         };
-        let (process_statuses, process_status) =
-            match fetch_current_process_status_in(current_session, state) {
+        let process_status_request =
+            if inject_process_status_failure && !process_status_failure_injected {
+                process_status_failure_injected = true;
+                Err(anyhow!("injected process-status transport failure"))
+            } else {
+                fetch_current_process_status_in(current_session, state)
+            };
+        let (process_statuses, process_status) = match process_status_request {
             Ok(status) => status,
             Err(error) => {
                 if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
@@ -1113,6 +1190,22 @@ pub(crate) fn observe_services_runtime(
             }
         };
         reconnect_delay = Duration::from_millis(100);
+        if !has_current_runtime_task(&task_snapshots) {
+            if let Some(mut outcome) =
+                terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
+            {
+                if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
+                    record.node_report = json!({
+                        "terminal_event": record.node_report.get("terminal_event"),
+                        "task_snapshots": task_snapshots,
+                        "process_status": process_status,
+                        "process_statuses": process_statuses,
+                    });
+                }
+                emit(outcome);
+                return Ok(());
+            }
+        }
         if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
             let failed_task = failed_task.to_owned();
             let failed_event = events
@@ -1191,7 +1284,11 @@ pub(crate) fn observe_services_runtime(
             return Ok(());
         }
 
-        let breakpoint = match current_session.request(client_user_request(
+        let breakpoint_request = if inject_fallback_failure && fallback_failure_stage == 0 {
+            fallback_failure_stage = 1;
+            Err(anyhow!("injected breakpoint inspection transport failure"))
+        } else {
+            current_session.request(client_user_request(
                 state,
                 json!({
                     "type": "inspect_debug_breakpoints",
@@ -1200,7 +1297,9 @@ pub(crate) fn observe_services_runtime(
                     "actor_user": state.actor_user,
                     "process": state.process.as_str(),
                 }),
-        )) {
+            ))
+        };
+        let breakpoint = match breakpoint_request {
             Ok(breakpoint) => breakpoint,
             Err(inspect_error) => {
                 // Main completion records its terminal event and clears ephemeral
@@ -1208,7 +1307,11 @@ pub(crate) fn observe_services_runtime(
                 // the event read above and this inspection request. Re-read the
                 // durable event stream before treating the missing debug state as
                 // an adapter failure.
-                let events = current_session.request(client_user_request(
+                let fallback_request = if inject_fallback_failure && fallback_failure_stage == 1 {
+                    fallback_failure_stage = 2;
+                    Err(anyhow!("injected fallback event transport failure"))
+                } else {
+                    current_session.request(client_user_request(
                         state,
                         json!({
                             "type": "list_task_events",
@@ -1217,11 +1320,52 @@ pub(crate) fn observe_services_runtime(
                             "actor_user": state.actor_user,
                             "process": state.process.as_str(),
                         }),
-                ))?;
-                if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
+                    ))
+                };
+                let events = match fallback_request {
+                    Ok(events) => events,
+                    Err(fallback_error) => {
+                        if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
+                            "runtime breakpoint and fallback event observation failed: {inspect_error:#}; {fallback_error:#}; reconnecting in {} ms",
+                            reconnect_delay.as_millis()
+                        ))) {
+                            return Ok(());
+                        }
+                        session = None;
+                        std::thread::sleep(reconnect_delay);
+                        reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
+                        continue;
+                    }
+                };
+                if let Some(mut outcome) =
+                    terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
+                {
+                    let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
+                        Ok(snapshots) => snapshots,
+                        Err(snapshot_error) => {
+                            if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
+                                "runtime fallback terminal snapshot observation failed: {snapshot_error:#}; reconnecting in {} ms",
+                                reconnect_delay.as_millis()
+                            ))) {
+                                return Ok(());
+                            }
+                            session = None;
+                            std::thread::sleep(reconnect_delay);
+                            reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
+                            continue;
+                        }
+                    };
+                    if !has_current_runtime_task(&task_snapshots) {
+                        if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
+                            record.node_report = json!({
+                                "terminal_event": record.node_report.get("terminal_event"),
+                                "task_snapshots": task_snapshots,
+                            });
+                        }
                         emit(outcome);
                         return Ok(());
                     }
+                }
                 if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
                     "runtime breakpoint observation failed: {inspect_error:#}; reconnecting in {} ms",
                     reconnect_delay.as_millis()
@@ -1236,7 +1380,28 @@ pub(crate) fn observe_services_runtime(
         };
         if let Some(epoch) = breakpoint.get("hit_epoch").and_then(Value::as_u64) {
             if epoch > previous_debug_epoch {
-                let frozen = wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)?;
+                let frozen_request =
+                    if inject_debug_epoch_wait_failure && !debug_epoch_wait_failure_injected {
+                        debug_epoch_wait_failure_injected = true;
+                        Err(anyhow!("injected Debug Epoch wait transport failure"))
+                    } else {
+                        wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)
+                    };
+                let frozen = match frozen_request {
+                    Ok(frozen) => frozen,
+                    Err(error) => {
+                        if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
+                            "runtime Debug Epoch observation failed: {error:#}; reconnecting in {} ms",
+                            reconnect_delay.as_millis()
+                        ))) {
+                            return Ok(());
+                        }
+                        session = None;
+                        std::thread::sleep(reconnect_delay);
+                        reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
+                        continue;
+                    }
+                };
                 let node = frozen
                     .get("acknowledgements")
                     .and_then(Value::as_array)
@@ -1352,68 +1517,126 @@ pub(crate) fn failed_awaiting_action_snapshot(task_snapshots: &Value) -> Option<
         })
 }
 
-fn terminal_runtime_outcome(
-    coordinator: &str,
-    state: &AdapterState,
-    events: &Value,
-) -> Option<RuntimeContinuationOutcome> {
-    let event = events
-        .get("events")
-        .and_then(Value::as_array)?
-        .get(state.runtime_event_count..)?
+fn has_current_runtime_task(task_snapshots: &Value) -> bool {
+    task_snapshots
+        .get("snapshots")
+        .and_then(Value::as_array)
+        .is_some_and(|snapshots| {
+            snapshots.iter().any(|snapshot| {
+                snapshot.get("current").and_then(Value::as_bool) == Some(true)
+                    && matches!(
+                        snapshot.get("state").and_then(Value::as_str),
+                        Some("queued" | "running" | "failed_awaiting_action")
+                    )
+            })
+        })
+}
+
+#[cfg(test)]
+pub(crate) fn whole_process_status_code(
+    main_status_code: Option<i32>,
+    task_snapshots: &Value,
+) -> Option<i32> {
+    let main_status_code = main_status_code?;
+    if main_status_code != 0 {
+        return Some(main_status_code);
+    }
+
+    let snapshots = task_snapshots.get("snapshots").and_then(Value::as_array)?;
+    for snapshot in snapshots
         .iter()
-        .rev()
-        .find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?;
-    let status_code = event
+        .filter(|snapshot| snapshot.get("current").and_then(Value::as_bool) == Some(true))
+    {
+        match snapshot.get("state").and_then(Value::as_str) {
+            Some("completed") => {}
+            Some("failed" | "cancelled" | "failed_awaiting_action") => {
+                return Some(
+                    snapshot
                         .get("status_code")
                         .and_then(Value::as_i64)
-        .map(|status| status as i32)
-        .or_else(
-            || match event.get("terminal_state").and_then(Value::as_str) {
-                Some("completed") => Some(0),
-                Some("failed" | "cancelled") => Some(1),
-                _ => None,
-            },
+                        .and_then(|status| i32::try_from(status).ok())
+                        .filter(|status| *status != 0)
+                        .unwrap_or(1),
                 );
+            }
+            Some("queued" | "running") => return None,
+            _ => return Some(1),
+        }
+    }
+    Some(0)
+}
+
+fn terminal_runtime_outcome(
+    coordinator: &str,
+    _state: &AdapterState,
+    events: &Value,
+    process_status: Option<&Value>,
+) -> Option<RuntimeContinuationOutcome> {
+    let final_result = process_status?
+        .get("final_result")
+        .and_then(Value::as_str)?;
+    let status_code = match final_result {
+        "completed" => 0,
+        "failed" | "cancelled" => 1,
+        _ => return None,
+    };
+    let event = events
+        .get("events")
+        .and_then(Value::as_array)
+        .and_then(|events| {
+            events.iter().rev().find(|event| {
+                event.get("executor").and_then(Value::as_str) == Some("coordinator_main")
+            })
+        });
     Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord {
         coordinator: coordinator.to_owned(),
         node: event
-            .get("node")
+            .and_then(|event| event.get("node"))
             .and_then(Value::as_str)
+            .or_else(|| {
+                process_status
+                    .and_then(|status| status.get("connected_nodes"))
+                    .and_then(Value::as_array)
+                    .and_then(|nodes| nodes.first())
+                    .and_then(Value::as_str)
+            })
             .unwrap_or("coordinator-main")
             .to_owned(),
-        node_report: json!({ "terminal_event": event }),
+        node_report: json!({
+            "terminal_event": event,
+            "process_status": process_status,
+        }),
         task_events: events.clone(),
         placed_task_launched: true,
-        status_code,
+        status_code: Some(status_code),
         stdout_bytes: event
-            .get("stdout_bytes")
+            .and_then(|event| event.get("stdout_bytes"))
             .and_then(Value::as_u64)
             .unwrap_or(0),
         stderr_bytes: event
-            .get("stderr_bytes")
+            .and_then(|event| event.get("stderr_bytes"))
             .and_then(Value::as_u64)
             .unwrap_or(0),
         stdout_tail: event
-            .get("stdout_tail")
+            .and_then(|event| event.get("stdout_tail"))
             .and_then(Value::as_str)
             .unwrap_or_default()
             .to_owned(),
         stderr_tail: event
-            .get("stderr_tail")
+            .and_then(|event| event.get("stderr_tail"))
             .and_then(Value::as_str)
             .unwrap_or_default()
             .to_owned(),
         stdout_truncated: event
-            .get("stdout_truncated")
+            .and_then(|event| event.get("stdout_truncated"))
             .and_then(Value::as_bool)
             .unwrap_or(false),
         stderr_truncated: event
-            .get("stderr_truncated")
+            .and_then(|event| event.get("stderr_truncated"))
             .and_then(Value::as_bool)
             .unwrap_or(false),
         artifact_path: event
-            .get("artifact_path")
+            .and_then(|event| event.get("artifact_path"))
             .and_then(Value::as_str)
             .map(str::to_owned),
         event_count: events
@@ -1471,6 +1694,24 @@ mod transactional_launch_tests {
 
     use super::*;
 
+    #[test]
+    fn current_child_task_keeps_runtime_observation_alive_after_main_completion() {
+        assert!(has_current_runtime_task(&json!({
+            "snapshots": [{
+                "current": true,
+                "state": "running",
+                "task": "child"
+            }]
+        })));
+        assert!(!has_current_runtime_task(&json!({
+            "snapshots": [{
+                "current": false,
+                "state": "completed",
+                "task": "main"
+            }]
+        })));
+    }
+
     #[test]
     fn inline_bundle_limit_is_checked_before_process_creation() {
         assert_eq!(MAX_CONTROL_FRAME_BYTES, 1024 * 1024);
@@ -1481,6 +1722,53 @@ mod transactional_launch_tests {
         assert!(error.contains("no virtual process was created"));
     }
 
+    #[test]
+    fn durable_process_summary_is_terminal_authority_after_event_rotation() {
+        let state = AdapterState {
+            runtime_event_count: 10_000,
+            ..AdapterState::default()
+        };
+        let completed = terminal_runtime_outcome(
+            "127.0.0.1:1",
+            &state,
+            &json!({ "events": [] }),
+            Some(&json!({
+                "process": state.process.as_str(),
+                "lifecycle": "recent_terminal",
+                "final_result": "completed",
+                "connected_nodes": []
+            })),
+        )
+        .expect("the durable summary should terminate observation");
+        let RuntimeContinuationOutcome::Terminal(completed) = completed else {
+            panic!("expected terminal outcome");
+        };
+        assert_eq!(completed.status_code, Some(0));
+
+        let failed = terminal_runtime_outcome(
+            "127.0.0.1:1",
+            &state,
+            &json!({
+                "events": [{
+                    "executor": "coordinator_main",
+                    "terminal_state": "completed",
+                    "status_code": 0
+                }]
+            }),
+            Some(&json!({
+                "process": state.process.as_str(),
+                "lifecycle": "recent_terminal",
+                "final_result": "failed",
+                "connected_nodes": []
+            })),
+        )
+        .expect("the aggregate summary should override a successful main event");
+        let RuntimeContinuationOutcome::Terminal(failed) = failed else {
+            panic!("expected terminal outcome");
+        };
+        assert_eq!(failed.status_code, Some(1));
+    }
+
     #[test]
     fn failed_debug_launch_reconnects_and_aborts_the_process() {
         let listener = TcpListener::bind("127.0.0.1:0").unwrap();

crates/clusterflux-dap/src/runtime_client/debug_protocol.rs

@@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestart
         restarted_task_instance: response
             .get("restarted_task_instance")
             .and_then(Value::as_str)
-            .map(TaskInstanceId::new),
+            .map(TaskInstanceId::try_new)
+            .transpose()
+            .map_err(|error| anyhow!("malformed restarted task instance: {error}"))?,
         restarted_attempt_id: response
             .get("restarted_attempt_id")
             .and_then(Value::as_str)

crates/clusterflux-dap/src/tests.rs

@@ -45,6 +45,13 @@ fn initialize_capabilities_use_standard_dap_flags() {
         .all(|key| key.starts_with("supports") && !key.contains("clusterflux")));
 }
 
+#[test]
+fn stale_breakpoint_success_and_failure_completions_are_not_current() {
+    assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9));
+    assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9));
+}
+
 #[test]
 fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() {
     assert_eq!(
@@ -697,11 +704,40 @@ fn terminal_record_without_snapshots_clears_active_threads() {
 
 #[test]
 fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
-    let mut state = AdapterState::default();
-    let project = Path::new(env!("CARGO_MANIFEST_DIR"))
-        .join("../../tests/fixtures/runtime-conformance")
-        .canonicalize()
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-source-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        r#"async fn run_build_workflow() {
+    let source = prepare_source_snapshot();
+    let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
+        .name("compile linux")
+        .env(linux_env())
+        .start()
+        .await
         .unwrap();
+    let linux_thread = linux.virtual_thread_id();
+    let linux_artifact = linux.join().await.unwrap();
+    let package = clusterflux::spawn::async_task_with_arg(
+        vec![linux_artifact.clone()],
+        package_release,
+    )
+    .name("package artifacts")
+    .env(linux_env())
+    .start()
+    .await
+    .unwrap();
+    let package_artifact = package.join().await.unwrap();
+}
+"#,
+    )
+    .unwrap();
+
+    let mut state = AdapterState::default();
     state.project = project.to_string_lossy().into_owned();
     state.source_path = "src/lib.rs".to_owned();
     let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
@@ -737,6 +773,8 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
         variable["name"] == "unavailable-local-diagnostic"
             && variable["type"] == "unavailable-local"
     }));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -856,11 +894,20 @@ fn package_release(inputs: Vec<Artifact>) -> Artifact {
 
 #[test]
 fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
+    let project = std::env::temp_dir().join(format!(
+        "clusterflux-dap-wasm-locals-{}",
+        std::process::id()
+    ));
+    let src = project.join("src");
+    fs::create_dir_all(&src).unwrap();
+    fs::write(
+        src.join("lib.rs"),
+        "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n    input + 1\n}\n",
+    )
+    .unwrap();
+
     let mut state = AdapterState::default();
-    state.project = Path::new(env!("CARGO_MANIFEST_DIR"))
-        .join("../../tests/fixtures/runtime-conformance")
-        .to_string_lossy()
-        .into_owned();
+    state.project = project.to_string_lossy().into_owned();
     state.source_path = "src/lib.rs".to_owned();
     let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
     state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
@@ -888,6 +935,8 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
     assert!(!locals
         .iter()
         .any(|variable| variable["name"] == "wasm-local-diagnostic"));
+
+    let _ = fs::remove_dir_all(project);
 }
 
 #[test]
@@ -915,6 +964,98 @@ fn detects_current_failed_attempt_awaiting_operator_action() {
     );
 }
 
+#[test]
+fn whole_process_terminal_status_covers_every_current_logical_task_attempt() {
+    let snapshots = |items: serde_json::Value| json!({ "snapshots": items });
+
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful main and final child must succeed"
+    );
+    for (state, status_code) in [("failed", 23), ("cancelled", 1)] {
+        assert_eq!(
+            whole_process_status_code(
+                Some(0),
+                &snapshots(json!([
+                    {
+                        "task": "child",
+                        "current": true,
+                        "state": state,
+                        "status_code": status_code
+                    }
+                ]))
+            ),
+            Some(status_code),
+            "a terminal child must determine the whole-process result"
+        );
+    }
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "accepted-failure",
+                    "current": true,
+                    "state": "failed",
+                    "command_state": "failure_accepted",
+                    "status_code": 17
+                }
+            ]))
+        ),
+        Some(17),
+        "accepting an AwaitOperator failure releases the process but does not turn failure into success"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(0),
+            &snapshots(json!([
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-1",
+                    "current": false,
+                    "state": "failed",
+                    "status_code": 7
+                },
+                {
+                    "task": "restarted",
+                    "attempt_id": "attempt-2",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(0),
+        "a successful replacement attempt is authoritative over stale failed attempts"
+    );
+    assert_eq!(
+        whole_process_status_code(
+            Some(9),
+            &snapshots(json!([
+                {
+                    "task": "child",
+                    "current": true,
+                    "state": "completed",
+                    "status_code": 0
+                }
+            ]))
+        ),
+        Some(9),
+        "a failed coordinator main cannot be masked by successful children"
+    );
+}
+
 #[test]
 fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
     let mut state = AdapterState::default();

crates/clusterflux-dap/src/virtual_model.rs

@@ -75,6 +75,7 @@ pub(crate) struct AdapterState {
     pub(crate) debug_probes: Vec<BundleDebugProbe>,
     pub(crate) breakpoints: Vec<i64>,
     pub(crate) breakpoints_installed: bool,
+    pub(crate) breakpoint_revision: u64,
     pub(crate) threads: BTreeMap<i64, VirtualThread>,
 }
 
@@ -113,7 +114,7 @@ impl Default for AdapterState {
             project,
             source_path: "src/lib.rs".to_owned(),
             runtime_backend: RuntimeBackend::Simulated,
-            coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(),
+            coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(),
             tenant: TenantId::from("tenant"),
             project_id: ProjectId::from("project"),
             actor_user: UserId::from("dap"),
@@ -133,6 +134,7 @@ impl Default for AdapterState {
             debug_probes: Vec::new(),
             breakpoints: Vec::new(),
             breakpoints_installed: false,
+            breakpoint_revision: 0,
         }
     }
 }
@@ -198,18 +200,24 @@ impl AdapterState {
             }
             self.coordinator_endpoint =
                 crate::view_state::normalize_coordinator_endpoint(&session.coordinator);
-            self.tenant = TenantId::new(session.tenant);
-            self.project_id = ProjectId::new(session.project);
-            self.actor_user = UserId::new(session.user);
+            self.tenant = TenantId::try_new(session.tenant)
+                .map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?;
+            self.project_id = ProjectId::try_new(session.project)
+                .map_err(|error| anyhow!("invalid project in CLI session: {error}"))?;
+            self.actor_user = UserId::try_new(session.user)
+                .map_err(|error| anyhow!("invalid user in CLI session: {error}"))?;
             self.client_session_secret = Some(session.session_secret);
         } else {
             self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(
                 coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"),
             );
             if let Some(scope) = project_scope {
-                self.tenant = TenantId::new(scope.tenant);
-                self.project_id = ProjectId::new(scope.project);
-                self.actor_user = UserId::new(scope.user);
+                self.tenant = TenantId::try_new(scope.tenant)
+                    .map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?;
+                self.project_id = ProjectId::try_new(scope.project)
+                    .map_err(|error| anyhow!("invalid project in project scope: {error}"))?;
+                self.actor_user = UserId::try_new(scope.user)
+                    .map_err(|error| anyhow!("invalid user in project scope: {error}"))?;
             }
             self.client_session_secret = None;
         }
@@ -230,6 +238,7 @@ impl AdapterState {
         self.epoch = 0;
         self.breakpoints.clear();
         self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated;
+        self.breakpoint_revision = 0;
         self.threads = if self.runtime_backend == RuntimeBackend::Simulated {
             launch_threads(&self.entry)
         } else {
@@ -304,7 +313,10 @@ impl AdapterState {
             self.debug_all_threads_stopped = record.all_participants_frozen;
             self.epoch = record.debug_epoch.unwrap_or(self.epoch);
             self.coordinator_debug_epoch = record.debug_epoch;
-            self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new);
+            self.stopped_task = record
+                .stopped_task
+                .as_deref()
+                .and_then(|task| TaskInstanceId::try_new(task).ok());
             self.stopped_probe_symbol = record.stopped_probe_symbol.clone();
             if let Some(acknowledgements) = record
                 .node_report
@@ -321,6 +333,12 @@ impl AdapterState {
                     else {
                         continue;
                     };
+                    let Ok(task_id) = TaskInstanceId::try_new(task) else {
+                        continue;
+                    };
+                    let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+                        continue;
+                    };
                     let coordinator_main = acknowledgement.get("node").and_then(Value::as_str)
                         == Some("coordinator-main");
                     let thread_id = if coordinator_main {
@@ -330,14 +348,19 @@ impl AdapterState {
                             .values()
                             .find(|thread| thread.task.as_str() == task)
                             .map(|thread| thread.id)
-                            .unwrap_or_else(|| self.insert_runtime_thread(task, task_definition))
+                            .unwrap_or_else(|| {
+                                self.insert_runtime_thread(
+                                    task_id.clone(),
+                                    task_definition_id.clone(),
+                                )
+                            })
                     };
                     let thread = self
                         .threads
                         .get_mut(&thread_id)
                         .expect("runtime thread was found or inserted");
-                    thread.task = TaskInstanceId::new(task);
-                    thread.task_definition = TaskDefinitionId::new(task_definition);
+                    thread.task = task_id;
+                    thread.task_definition = task_definition_id;
                     thread.runtime_stack_frames = acknowledgement
                         .get("stack_frames")
                         .and_then(Value::as_array)
@@ -419,18 +442,22 @@ impl AdapterState {
         let _ = crate::view_state::write_view_state(self, &record);
     }
 
-    fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 {
+    fn insert_runtime_thread(
+        &mut self,
+        task: TaskInstanceId,
+        task_definition: TaskDefinitionId,
+    ) -> i64 {
         let id = self.allocate_runtime_thread_id();
         let line = self
             .debug_probes
             .iter()
             .find(|probe| {
-                probe.task.as_str() == task_definition || probe.function == task_definition
+                probe.task == task_definition || probe.function == task_definition.as_str()
             })
             .map(|probe| i64::from(probe.line_start))
             .unwrap_or(1);
-        let definition_name = task_definition.replace(['_', '-'], " ");
-        let name = if task == task_definition {
+        let definition_name = task_definition.as_str().replace(['_', '-'], " ");
+        let name = if task.as_str() == task_definition.as_str() {
             definition_name
         } else {
             format!("{definition_name} ({task})")
@@ -448,9 +475,9 @@ impl AdapterState {
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::new(task),
+                task,
                 attempt_id: "unknown-attempt".to_owned(),
-                task_definition: TaskDefinitionId::new(task_definition),
+                task_definition,
                 name,
                 line,
                 state: DebugRuntimeState::Running,
@@ -716,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId {
     ProcessId::new(format!("vp-{suffix}"))
 }
 
-fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread {
+fn coordinator_main_thread(
+    entry: &str,
+    task: TaskInstanceId,
+    task_definition: TaskDefinitionId,
+) -> VirtualThread {
+    let name = format!("{entry} coordinator main ({task})");
     VirtualThread {
         id: MAIN_THREAD,
         frame_id: 1_001,
@@ -728,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi
         target_ref: 4_501,
         vfs_ref: 5_001,
         command_ref: 5_501,
-        task: TaskInstanceId::from(task),
+        task,
         attempt_id: "main:unknown".to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
-        name: format!("{entry} coordinator main ({task})"),
+        task_definition,
+        name,
         line: 0,
         state: DebugRuntimeState::Running,
         freeze_supported: true,
@@ -767,10 +799,14 @@ fn coordinator_threads_from_events(
         .expect("launch thread model should include main");
     if let Some(status) = process_status {
         if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) {
-            main.task = TaskInstanceId::from(task);
+            if let Ok(task) = TaskInstanceId::try_new(task) {
+                main.task = task;
+            }
         }
         if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) {
-            main.task_definition = TaskDefinitionId::from(definition);
+            if let Ok(definition) = TaskDefinitionId::try_new(definition) {
+                main.task_definition = definition;
+            }
         }
         main.name = format!("{entry} coordinator main ({})", main.task);
         main.state = if status
@@ -832,6 +868,12 @@ fn coordinator_threads_from_snapshots(
                 .get("main_task_definition")
                 .and_then(Value::as_str)
                 .unwrap_or(entry);
+            let (Ok(task), Ok(task_definition)) = (
+                TaskInstanceId::try_new(task),
+                TaskDefinitionId::try_new(task_definition),
+            ) else {
+                return threads;
+            };
             let mut main = coordinator_main_thread(entry, task, task_definition);
             main.attempt_id = format!(
                 "main:{}",
@@ -886,6 +928,12 @@ fn coordinator_threads_from_snapshots(
         let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else {
             continue;
         };
+        let Ok(task_id) = TaskInstanceId::try_new(task) else {
+            continue;
+        };
+        let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
+            continue;
+        };
         let attempt_id = snapshot
             .get("attempt_id")
             .and_then(Value::as_str)
@@ -936,9 +984,9 @@ fn coordinator_threads_from_snapshots(
                 target_ref: 6000 + id,
                 vfs_ref: 7000 + id,
                 command_ref: 8000 + id,
-                task: TaskInstanceId::from(task),
+                task: task_id,
                 attempt_id,
-                task_definition: TaskDefinitionId::from(task_definition),
+                task_definition: task_definition_id,
                 name: format!("{display_name} ({task}, attempt {short_attempt})"),
                 line: snapshot
                     .get("source_line")
@@ -985,6 +1033,8 @@ fn coordinator_threads_from_snapshots(
 fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option<VirtualThread> {
     let task = event.get("task").and_then(Value::as_str)?;
     let task_definition = event.get("task_definition").and_then(Value::as_str)?;
+    let task_id = TaskInstanceId::try_new(task).ok()?;
+    let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?;
     let definition_name = task_definition.replace(['_', '-'], " ");
     let name = if task == task_definition {
         definition_name
@@ -1030,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti
         target_ref: 6000 + id,
         vfs_ref: 7000 + id,
         command_ref: 8000 + id,
-        task: TaskInstanceId::from(task),
+        task: task_id,
         attempt_id: event
             .get("attempt_id")
             .and_then(Value::as_str)
             .unwrap_or("unknown-attempt")
             .to_owned(),
-        task_definition: TaskDefinitionId::from(task_definition),
+        task_definition: task_definition_id,
         name,
         line: 0,
         state,

crates/clusterflux-node/src/assignment_runner.rs

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::io::Read;
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::{Duration, Instant};
@@ -46,6 +46,40 @@ use validation::{
     resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
 };
 
+#[derive(Debug)]
+struct AssignmentExecutionError {
+    message: String,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
+}
+
+impl std::fmt::Display for AssignmentExecutionError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for AssignmentExecutionError {}
+
+fn execution_error_with_log_bytes(
+    message: impl Into<String>,
+    stdout_source_bytes: &AtomicU64,
+    stderr_source_bytes: &AtomicU64,
+) -> Box<dyn std::error::Error> {
+    Box::new(AssignmentExecutionError {
+        message: message.into(),
+        stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed),
+        stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed),
+    })
+}
+
+pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) {
+    error
+        .downcast_ref::<AssignmentExecutionError>()
+        .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes))
+        .unwrap_or((0, 0))
+}
+
 pub(crate) fn run_verified_wasmtime_assignment(
     args: &Args,
     task: &RuntimeTask,
@@ -95,6 +129,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
             return Err("Wasm entrypoint assignment omitted its descriptor export".into());
         }
     };
+    let command_stdout_source_bytes = Arc::new(AtomicU64::new(0));
+    let command_stderr_source_bytes = Arc::new(AtomicU64::new(0));
     let (stdout, boundary_result) = match abi {
         WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
             let invocation = WasmTaskInvocation::new(
@@ -102,7 +138,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
                 task_spec.task_instance.clone(),
                 task_spec.args.clone(),
             );
-            let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
+            let result = WasmtimeTaskRuntime::new()?
+                .run_task_export_verified_with_task_host(
                     &module,
                     expected_bundle_digest,
                     export,
@@ -112,8 +149,17 @@ pub(crate) fn run_verified_wasmtime_assignment(
                         task,
                         node_private_key,
                         &module,
+                        Arc::clone(&command_stdout_source_bytes),
+                        Arc::clone(&command_stderr_source_bytes),
                     )?),
-            )?;
+                )
+                .map_err(|error| {
+                    execution_error_with_log_bytes(
+                        error.to_string(),
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    )
+                })?;
             if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
                 eprintln!(
                     "clusterflux debug control: Wasm assignment returned for task {}",
@@ -122,17 +168,35 @@ pub(crate) fn run_verified_wasmtime_assignment(
             }
             match result.outcome {
                 WasmTaskOutcome::Completed => {
-                    let boundary = result.result.ok_or("completed Wasm task omitted result")?;
+                    let boundary = result.result.ok_or_else(|| {
+                        execution_error_with_log_bytes(
+                            "completed Wasm task omitted result",
+                            &command_stdout_source_bytes,
+                            &command_stderr_source_bytes,
+                        )
+                    })?;
                     (
-                        format!("{}\n", serde_json::to_string(&boundary)?),
+                        format!(
+                            "{}\n",
+                            serde_json::to_string(&boundary).map_err(|error| {
+                                execution_error_with_log_bytes(
+                                    error.to_string(),
+                                    &command_stdout_source_bytes,
+                                    &command_stderr_source_bytes,
+                                )
+                            })?
+                        ),
                         Some(boundary),
                     )
                 }
                 WasmTaskOutcome::Failed => {
-                    return Err(result
+                    return Err(execution_error_with_log_bytes(
+                        result
                             .error
-                        .unwrap_or_else(|| "Wasm task failed without an error".to_owned())
-                        .into())
+                            .unwrap_or_else(|| "Wasm task failed without an error".to_owned()),
+                        &command_stdout_source_bytes,
+                        &command_stderr_source_bytes,
+                    ))
                 }
             }
         }
@@ -140,12 +204,18 @@ pub(crate) fn run_verified_wasmtime_assignment(
     let task_id = TaskInstanceId::new(task.task.clone());
     let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
     let manifest = artifacts.flush();
+    let stdout_source_bytes = command_stdout_source_bytes
+        .load(Ordering::Relaxed)
+        .saturating_add(stdout.len() as u64);
+    let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed);
     Ok((
         CommandOutput {
             virtual_thread: task_id,
             status_code: Some(0),
             stdout,
             stderr: String::new(),
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: false,
             stderr_truncated: false,
             log_backpressured: false,
@@ -176,6 +246,8 @@ struct CoordinatorWasmTaskHost {
     next_handle_id: u64,
     handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
     command_status: Arc<Mutex<Option<String>>>,
+    command_stdout_source_bytes: Arc<AtomicU64>,
+    command_stderr_source_bytes: Arc<AtomicU64>,
     cancellation_requested: Arc<AtomicBool>,
     abort_requested: Arc<AtomicBool>,
     debug_control: Arc<WasmDebugControl>,
@@ -188,6 +260,8 @@ impl CoordinatorWasmTaskHost {
         parent: &RuntimeTask,
         node_private_key: &str,
         module: &[u8],
+        command_stdout_source_bytes: Arc<AtomicU64>,
+        command_stderr_source_bytes: Arc<AtomicU64>,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let task_spec = parent
             .task_spec
@@ -268,6 +342,8 @@ impl CoordinatorWasmTaskHost {
             next_handle_id: 1,
             handles,
             command_status,
+            command_stdout_source_bytes,
+            command_stderr_source_bytes,
             cancellation_requested,
             abort_requested,
             debug_control,
@@ -293,7 +369,16 @@ impl CoordinatorWasmTaskHost {
             runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
         let execution =
             run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
-        let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
+        let (
+            terminal_state,
+            status_code,
+            stdout,
+            stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
+            result,
+            retained,
+        ) = match execution {
             Ok((output, _manifest, result)) => {
                 let retained = retained_result_artifact(
                     self.args.project_root.as_deref(),
@@ -306,20 +391,40 @@ impl CoordinatorWasmTaskHost {
                         output.status_code,
                         output.stdout,
                         output.stderr,
+                        output.stdout_source_bytes,
+                        output.stderr_source_bytes,
                         result,
                         retained,
                     ),
-                    Err(error) => ("failed", Some(1), String::new(), error, None, None),
-                }
-            }
                     Err(error) => (
                         "failed",
                         Some(1),
                         String::new(),
-                error.to_string(),
+                        error.clone(),
+                        output.stdout_source_bytes,
+                        output
+                            .stderr_source_bytes
+                            .saturating_add(error.len() as u64),
                         None,
                         None,
                     ),
+                }
+            }
+            Err(error) => {
+                let (stdout_source_bytes, stderr_source_bytes) =
+                    assignment_error_log_bytes(error.as_ref());
+                let error = error.to_string();
+                (
+                    "failed",
+                    Some(1),
+                    String::new(),
+                    error.clone(),
+                    stdout_source_bytes,
+                    stderr_source_bytes.saturating_add(error.len() as u64),
+                    None,
+                    None,
+                )
+            }
         };
         let artifact_path = retained
             .as_ref()
@@ -339,8 +444,8 @@ impl CoordinatorWasmTaskHost {
                     "task": runtime_task.task,
                     "terminal_state": terminal_state,
                     "status_code": status_code,
-                    "stdout_bytes": stdout.len(),
-                    "stderr_bytes": stderr.len(),
+                    "stdout_bytes": stdout_source_bytes,
+                    "stderr_bytes": stderr_source_bytes,
                     "stdout_tail": stdout,
                     "stderr_tail": stderr,
                     "stdout_truncated": false,
@@ -676,6 +781,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost {
         let mut runner = CoordinatorControlledProcessRunner::new(
             self,
             Duration::from_millis(request.timeout_ms),
+            configured_secrets.clone(),
         );
         let output = LinuxRootlessPodmanBackend
             .execute_local_checkout_task(

crates/clusterflux-node/src/assignment_runner/process_runner.rs

@@ -1,4 +1,83 @@
 use super::*;
+use std::sync::mpsc::{self, Receiver, SyncSender};
+
+struct LiveLogChunk {
+    stream: &'static str,
+    offset: u64,
+    source_bytes: u64,
+    bytes: Vec<u8>,
+    truncated: bool,
+}
+
+fn append_bounded_tail(tail: &mut Vec<u8>, bytes: &[u8], maximum: usize) {
+    if maximum == 0 {
+        tail.clear();
+        return;
+    }
+    if bytes.len() >= maximum {
+        tail.clear();
+        tail.extend_from_slice(&bytes[bytes.len() - maximum..]);
+        return;
+    }
+    let overflow = tail
+        .len()
+        .saturating_add(bytes.len())
+        .saturating_sub(maximum);
+    if overflow > 0 {
+        tail.drain(..overflow);
+    }
+    tail.extend_from_slice(bytes);
+}
+
+fn redact_safe_live_log_prefix(
+    pending: &[u8],
+    configured_secrets: &[String],
+    final_chunk: bool,
+) -> Option<(usize, String)> {
+    if pending.is_empty() {
+        return None;
+    }
+    let secret_bytes = configured_secrets
+        .iter()
+        .filter(|secret| secret.len() >= 4)
+        .map(String::as_bytes)
+        .collect::<Vec<_>>();
+    let maximum_secret_bytes = secret_bytes
+        .iter()
+        .map(|secret| secret.len())
+        .max()
+        .unwrap_or(0);
+    if !final_chunk && pending.len() <= maximum_secret_bytes {
+        return None;
+    }
+    let mut consumed = if final_chunk {
+        pending.len()
+    } else {
+        pending.len() - maximum_secret_bytes
+    };
+    loop {
+        let previous = consumed;
+        for secret in &secret_bytes {
+            for start in 0..=pending.len().saturating_sub(secret.len()) {
+                let end = start + secret.len();
+                if start < consumed && end > consumed && &pending[start..end] == *secret {
+                    consumed = end;
+                }
+            }
+        }
+        if consumed == previous {
+            break;
+        }
+    }
+    if consumed == 0 {
+        return None;
+    }
+    let text = redact_configured_values(
+        String::from_utf8_lossy(&pending[..consumed]).into_owned(),
+        configured_secrets,
+    );
+    Some((consumed, text))
+}
 
 pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) args: Args,
@@ -7,13 +86,20 @@ pub(super) struct CoordinatorControlledProcessRunner {
     pub(super) node_private_key: String,
     pub(super) debug_control: Arc<WasmDebugControl>,
     pub(super) command_status: Arc<Mutex<Option<String>>>,
+    pub(super) stdout_source_bytes: Arc<AtomicU64>,
+    pub(super) stderr_source_bytes: Arc<AtomicU64>,
     pub(super) timeout: Duration,
+    pub(super) configured_secrets: Vec<String>,
 }
 
 impl CoordinatorControlledProcessRunner {
     const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
 
-    pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
+    pub(super) fn new(
+        host: &CoordinatorWasmTaskHost,
+        timeout: Duration,
+        configured_secrets: Vec<String>,
+    ) -> Self {
         Self {
             args: host.args.clone(),
             process: host.process.clone(),
@@ -21,7 +107,10 @@ impl CoordinatorControlledProcessRunner {
             node_private_key: host.node_private_key.clone(),
             debug_control: Arc::clone(&host.debug_control),
             command_status: Arc::clone(&host.command_status),
+            stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes),
+            stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes),
             timeout,
+            configured_secrets,
         }
     }
 
@@ -201,10 +290,18 @@ impl CoordinatorControlledProcessRunner {
     fn drain_bounded(
         mut reader: impl Read + Send + 'static,
         maximum: usize,
+        stream: &'static str,
+        sender: SyncSender<LiveLogChunk>,
+        configured_secrets: Vec<String>,
+        source_bytes_total: Arc<AtomicU64>,
     ) -> thread::JoinHandle<Result<Vec<u8>, String>> {
         thread::spawn(move || {
             let mut captured = Vec::new();
             let mut buffer = [0_u8; 16 * 1024];
+            let stream_base = source_bytes_total.load(Ordering::Relaxed);
+            let mut source_bytes_read = 0_u64;
+            let mut pending_offset = stream_base;
+            let mut pending = Vec::new();
             loop {
                 let count = reader
                     .read(&mut buffer)
@@ -212,12 +309,128 @@ impl CoordinatorControlledProcessRunner {
                 if count == 0 {
                     break;
                 }
-                let remaining = maximum.saturating_sub(captured.len());
-                captured.extend_from_slice(&buffer[..count.min(remaining)]);
+                let _ = source_bytes_total.fetch_update(
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                    |current| Some(current.saturating_add(count as u64)),
+                );
+                source_bytes_read = source_bytes_read.saturating_add(count as u64);
+                append_bounded_tail(&mut captured, &buffer[..count], maximum);
+                pending.extend_from_slice(&buffer[..count]);
+                if let Some((consumed, text)) =
+                    redact_safe_live_log_prefix(&pending, &configured_secrets, false)
+                {
+                    let _ = sender.try_send(LiveLogChunk {
+                        stream,
+                        offset: pending_offset,
+                        source_bytes: consumed as u64,
+                        bytes: text.into_bytes(),
+                        truncated: false,
+                    });
+                    pending.drain(..consumed);
+                    pending_offset = pending_offset.saturating_add(consumed as u64);
+                }
+            }
+            if let Some((consumed, text)) =
+                redact_safe_live_log_prefix(&pending, &configured_secrets, true)
+            {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: pending_offset,
+                    source_bytes: consumed as u64,
+                    bytes: text.into_bytes(),
+                    truncated: false,
+                });
+            }
+            if source_bytes_read > maximum as u64 {
+                let _ = sender.try_send(LiveLogChunk {
+                    stream,
+                    offset: stream_base.saturating_add(source_bytes_read),
+                    source_bytes: 0,
+                    bytes: b"[log output truncated at node capture limit]".to_vec(),
+                    truncated: true,
+                });
             }
             Ok(captured)
         })
     }
+
+    fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
+        let args = self.args.clone();
+        let process = self.process.clone();
+        let task = self.task.clone();
+        let node_private_key = self.node_private_key.clone();
+        let configured_secrets = self.configured_secrets.clone();
+        let command_status = Arc::clone(&self.command_status);
+        thread::spawn(move || {
+            let mut log_session = None;
+            let mut delivery_available = true;
+            while let Ok(chunk) = receiver.recv() {
+                if !delivery_available {
+                    continue;
+                }
+                let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
+                text = redact_configured_values(text, &configured_secrets);
+                let mut delivered = false;
+                for _ in 0..2 {
+                    if log_session.is_none() {
+                        log_session = CoordinatorSession::connect_with_timeouts(
+                            &args.coordinator,
+                            Duration::from_millis(500),
+                            Duration::from_millis(500),
+                        )
+                        .ok();
+                    }
+                    let Some(session) = log_session.as_mut() else {
+                        continue;
+                    };
+                    let request = signed_node_request_json(
+                        &args,
+                        &node_private_key,
+                        "report_task_log_chunk",
+                        serde_json::json!({
+                            "type": "report_task_log_chunk",
+                            "tenant": &args.tenant,
+                            "project": &args.project,
+                            "process": &process,
+                            "node": &args.node,
+                            "task": &task,
+                            "stream": chunk.stream,
+                            "offset": chunk.offset,
+                            "source_bytes": chunk.source_bytes,
+                            "text": &text,
+                            "truncated": chunk.truncated,
+                        }),
+                    );
+                    let result = match request {
+                        Ok(request) => session
+                            .request(request)
+                            .map(|_| ())
+                            .map_err(|error| error.to_string()),
+                        Err(error) => Err(error.to_string()),
+                    };
+                    match result {
+                        Ok(()) => {
+                            delivered = true;
+                            break;
+                        }
+                        Err(_) => {
+                            log_session = None;
+                        }
+                    }
+                }
+                if !delivered {
+                    delivery_available = false;
+                    if let Ok(mut current) = command_status.lock() {
+                        *current = Some(
+                            "live log delivery was interrupted; final bounded output remains available"
+                                .to_owned(),
+                        );
+                    }
+                }
+            }
+        })
+    }
 }
 
 impl ProcessRunner for CoordinatorControlledProcessRunner {
@@ -245,12 +458,17 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             command.program,
             command.args.join(" ")
         ));
+        let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
         let stdout = Self::drain_bounded(
             child
                 .stdout
                 .take()
                 .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stdout",
+            live_log_sender.clone(),
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stdout_source_bytes),
         );
         let stderr = Self::drain_bounded(
             child
@@ -258,11 +476,19 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 .take()
                 .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
             Self::MAX_CAPTURE_BYTES,
+            "stderr",
+            live_log_sender,
+            self.configured_secrets.clone(),
+            Arc::clone(&self.stderr_source_bytes),
         );
+        let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
         let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
             Ok(session) => session,
             Err(error) => {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
+                let _ = stdout.join();
+                let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "establish execution control channel: {error}"
                 )));
@@ -277,6 +503,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Ok(None) => {}
                 Err(error) => {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
+                    let _ = stdout.join();
+                    let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Command(error.to_string()));
                 }
             }
@@ -289,6 +518,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                 Self::terminate_execution(&mut child, podman_container.as_deref());
                 let _ = stdout.join();
                 let _ = stderr.join();
+                let _ = live_log_reporter.join();
                 return Err(BackendError::Command(format!(
                     "native command exceeded wall-clock timeout of {} ms",
                     self.timeout.as_millis()
@@ -303,6 +533,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(BackendError::Cancelled(
                         "coordinator requested cancellation or abort".to_owned(),
                     ));
@@ -312,6 +543,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
                     Self::terminate_execution(&mut child, podman_container.as_deref());
                     let _ = stdout.join();
                     let _ = stderr.join();
+                    let _ = live_log_reporter.join();
                     return Err(error);
                 }
             }
@@ -360,6 +592,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
             .join()
             .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
             .map_err(BackendError::Command)?;
+        live_log_reporter
+            .join()
+            .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
         self.set_command_status(format!(
             "native command exited with status {:?}",
             status.code()
@@ -371,3 +606,52 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner};
+    use std::io::Cursor;
+    use std::sync::atomic::{AtomicU64, Ordering};
+    use std::sync::{mpsc, Arc};
+
+    #[test]
+    fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
+        let secrets = vec!["correct-horse".to_owned()];
+        let mut pending = b"prefix correct-".to_vec();
+        let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
+        pending.drain(..consumed);
+        pending.extend_from_slice(b"horse suffix");
+        let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
+
+        let combined = format!("{first}{second}");
+        assert_eq!(combined, "prefix [REDACTED] suffix");
+        assert!(!combined.contains("correct-"));
+        assert!(!combined.contains("horse"));
+    }
+
+    #[test]
+    fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() {
+        let source_bytes = Arc::new(AtomicU64::new(5));
+        let (sender, receiver) = mpsc::sync_channel(8);
+        let reader = CoordinatorControlledProcessRunner::drain_bounded(
+            Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()),
+            8,
+            "stdout",
+            sender,
+            Vec::new(),
+            Arc::clone(&source_bytes),
+        );
+        let captured = reader.join().unwrap().unwrap();
+        let chunks = receiver.into_iter().collect::<Vec<_>>();
+
+        assert_eq!(captured, b"opqrstuv");
+        assert_eq!(source_bytes.load(Ordering::Relaxed), 37);
+        assert_eq!(
+            chunks.iter().map(|chunk| chunk.source_bytes).sum::<u64>(),
+            32
+        );
+        assert_eq!(chunks[0].offset, 5);
+        assert_eq!(chunks.last().unwrap().offset, 37);
+        assert!(chunks.iter().any(|chunk| chunk.truncated));
+    }
+}

crates/clusterflux-node/src/assignment_runner/tests.rs

@@ -48,7 +48,10 @@ fn test_controlled_runner(
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout,
+        configured_secrets: Vec::new(),
     }
 }
 
@@ -89,7 +92,10 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
         ),
         debug_control: Arc::new(WasmDebugControl::default()),
         command_status: Arc::new(Mutex::new(None)),
+        stdout_source_bytes: Arc::new(AtomicU64::new(0)),
+        stderr_source_bytes: Arc::new(AtomicU64::new(0)),
         timeout: Duration::from_secs(30),
+        configured_secrets: Vec::new(),
     };
     let started = Instant::now();
     let error = runner

crates/clusterflux-node/src/command_runner.rs

@@ -1,6 +1,6 @@
 use clusterflux_core::{
-    CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
-    VfsOverlay, VfsPath,
+    CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay,
+    VfsPath,
 };
 use serde::{Deserialize, Serialize};
 
@@ -42,6 +42,8 @@ pub struct CommandOutput {
     pub status_code: Option<i32>,
     pub stdout: String,
     pub stderr: String,
+    pub stdout_source_bytes: u64,
+    pub stderr_source_bytes: u64,
     pub stdout_truncated: bool,
     pub stderr_truncated: bool,
     pub log_backpressured: bool,
@@ -83,6 +85,8 @@ impl LocalCommandExecutor {
             &output.stderr,
             max_log_bytes,
         );
+        let stdout_source_bytes = output.stdout.len() as u64;
+        let stderr_source_bytes = output.stderr.len() as u64;
         let staged_artifact = if let Some(path) = command.stage_stdout_as {
             Some(overlay.write(
                 path,
@@ -98,6 +102,8 @@ impl LocalCommandExecutor {
             status_code: output.status.code(),
             stdout: logs.stdout,
             stderr: logs.stderr,
+            stdout_source_bytes,
+            stderr_source_bytes,
             stdout_truncated: logs.stdout_truncated,
             stderr_truncated: logs.stderr_truncated,
             log_backpressured: logs.backpressured,
@@ -107,24 +113,20 @@ impl LocalCommandExecutor {
 }
 
 pub(super) fn capture_command_logs(
-    task: &TaskInstanceId,
+    _task: &TaskInstanceId,
     stdout: &[u8],
     stderr: &[u8],
     max_log_bytes: usize,
 ) -> CapturedCommandLogs {
-    let mut logs = LogBuffer::new(max_log_bytes);
-    logs.push(task.clone(), stdout);
-    logs.push(task.clone(), stderr);
-    let records = logs.records();
-    let stdout_record = &records[0];
-    let stderr_record = &records[1];
-    debug_assert_eq!(&stdout_record.task, task);
-    debug_assert_eq!(&stderr_record.task, task);
+    let stdout_truncated = stdout.len() > max_log_bytes;
+    let stderr_truncated = stderr.len() > max_log_bytes;
+    let stdout_start = stdout.len().saturating_sub(max_log_bytes);
+    let stderr_start = stderr.len().saturating_sub(max_log_bytes);
     CapturedCommandLogs {
-        stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
-        stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
-        stdout_truncated: stdout_record.truncated,
-        stderr_truncated: stderr_record.truncated,
-        backpressured: logs.backpressured(),
+        stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(),
+        stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(),
+        stdout_truncated,
+        stderr_truncated,
+        backpressured: stdout_truncated || stderr_truncated,
     }
 }

crates/clusterflux-node/src/coordinator_session.rs

@@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity;
 use clusterflux_control::ControlSession;
 use clusterflux_core::coordinator_wire_request;
 use serde_json::Value;
+use std::time::Duration;
 
 pub(crate) struct CoordinatorSession {
     inner: ControlSession,
@@ -15,6 +16,16 @@ impl CoordinatorSession {
         })
     }
 
+    pub(crate) fn connect_with_timeouts(
+        addr: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        Ok(Self {
+            inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?,
+        })
+    }
+
     pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
         let request_id = format!("node-{}", self.inner.requests() + 1);
         let wire_request = coordinator_wire_request(request_id, value);

crates/clusterflux-node/src/daemon.rs

@@ -7,11 +7,11 @@ use std::time::{Duration, Instant};
 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
 use clusterflux_core::{
     sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
-    TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
+    ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
 };
 use serde_json::{json, Value};
 
-use crate::assignment_runner::run_verified_wasmtime_assignment;
+use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment};
 #[cfg(test)]
 use crate::coordinator_session::control_endpoint_identity;
 use crate::coordinator_session::CoordinatorSession;
@@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
     let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
     let heartbeat_request = json!({
         "type": "node_heartbeat",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
     });
     let heartbeat_signature = sign_node_request(
@@ -222,7 +224,7 @@ fn worker_loop(
             let expiry = Instant::now()
                 .checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
                 .unwrap_or_else(Instant::now);
-            restart_pins.insert(ArtifactId::new(artifact), expiry);
+            restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry);
         }
         println!("{}", serde_json::to_string(&report)?);
         std::io::stdout().flush()?;
@@ -289,7 +291,7 @@ fn service_pending_artifact_transfer(
         return Ok(false);
     };
     let transfer_id = required_string(transfer, "transfer_id")?;
-    let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
+    let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?;
     let expected_digest: Digest = serde_json::from_value(
         transfer
             .get("expected_digest")
@@ -369,9 +371,11 @@ pub(crate) fn runtime_task_from_assignment(
             .cloned()
             .ok_or("task assignment missing task_spec")?,
     )?;
+    let process = ProcessId::try_new(required_string(value, "process")?)?;
+    let task = TaskInstanceId::try_new(required_string(value, "task")?)?;
     Ok(RuntimeTask {
-        process: required_string(value, "process")?,
-        task: required_string(value, "task")?,
+        process: process.to_string(),
+        task: task.to_string(),
         epoch: value.get("epoch").and_then(Value::as_u64),
         bundle_digest: task_spec.bundle_digest.clone(),
         task_spec: Some(task_spec),
@@ -418,6 +422,8 @@ fn run_runtime_task(
         "reconnect_node",
         json!({
         "type": "reconnect_node",
+        "tenant": &args.tenant,
+        "project": &args.project,
         "node": &args.node,
         "process": &task.process,
         "epoch": epoch,
@@ -458,6 +464,8 @@ fn run_runtime_task(
             capability_report,
             debug_command,
             node_private_key,
+            0,
+            0,
         );
     }
 
@@ -492,9 +500,13 @@ fn run_runtime_task(
                 debug_command,
                 node_private_key,
                 &error,
+                output.stdout_source_bytes,
+                output.stderr_source_bytes,
             ),
         },
         Err(error) => {
+            let (stdout_source_bytes, stderr_source_bytes) =
+                assignment_error_log_bytes(error.as_ref());
             let error = error.to_string();
             if error.contains("task execution cancelled:") {
                 record_cancelled_task(
@@ -506,6 +518,8 @@ fn run_runtime_task(
                     capability_report,
                     debug_command,
                     node_private_key,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             } else {
                 record_failed_task(
@@ -518,6 +532,8 @@ fn run_runtime_task(
                     debug_command,
                     node_private_key,
                     &error,
+                    stdout_source_bytes,
+                    stderr_source_bytes,
                 )
             }
         }
@@ -570,6 +586,12 @@ fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
         }
     }
 
+    TenantId::try_new(tenant.clone())?;
+    ProjectId::try_new(project.clone())?;
+    NodeId::try_new(node.clone())?;
+    if let Some(grant) = enrollment_grant.as_ref() {
+        clusterflux_core::validate_opaque_token(grant, 512)?;
+    }
     Ok(Args {
         coordinator: coordinator.ok_or("--coordinator is required")?,
         tenant,
@@ -622,13 +644,12 @@ mod tests {
     #[test]
     fn hosted_url_remains_an_https_control_endpoint() {
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
+            control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(),
+            "https://clusterflux.lesstuff.com/api/v1/control"
         );
         assert_eq!(
-            control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
-                .unwrap(),
-            "https://clusterflux.michelpaulissen.com/api/v1/control"
+            control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
+            "https://clusterflux.lesstuff.com/api/v1/control"
         );
         assert_eq!(
             control_endpoint_identity("127.0.0.1:7999").unwrap(),

crates/clusterflux-node/src/lib.rs

@@ -741,7 +741,7 @@ mod tests {
     }
 
     #[test]
-    fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() {
+    fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() {
         let invocation = CommandInvocation {
             program: "cargo".to_owned(),
             args: vec!["build".to_owned()],
@@ -774,7 +774,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
+        assert_eq!(output.stdout, "cdef");
         assert!(output.stdout_truncated);
         assert!(output.log_backpressured);
         assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
@@ -992,7 +992,7 @@ mod tests {
 
     #[cfg(unix)]
     #[test]
-    fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() {
+    fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() {
         let executor = LocalCommandExecutor {
             node: clusterflux_core::NodeId::from("node"),
             hosted_control_plane: false,
@@ -1021,15 +1021,15 @@ mod tests {
             .unwrap();
 
         assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
-        assert_eq!(output.stdout, "abcd");
-        assert_eq!(output.stderr, "");
+        assert_eq!(output.stdout, "cdef");
+        assert_eq!(output.stderr, "err");
         assert!(output.stdout_truncated);
-        assert!(output.stderr_truncated);
+        assert!(!output.stderr_truncated);
         assert!(output.log_backpressured);
     }
 
     #[test]
-    fn public_node_crate_does_not_require_hosted_private_types() {
+    fn public_node_crate_does_not_require_hosted_service_types() {
         let _tenant = TenantId::from("tenant");
         let _project = ProjectId::from("project");
         let _backend = LinuxRootlessPodmanBackend;

crates/clusterflux-node/src/main.rs

@@ -8,5 +8,34 @@ mod task_artifacts;
 mod task_reports;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
+    match raw_args.as_slice() {
+        [flag] if matches!(flag.as_str(), "--version" | "-V") => {
+            println!("clusterflux-node {}", env!("CARGO_PKG_VERSION"));
+            return Ok(());
+        }
+        [flag] if matches!(flag.as_str(), "--help" | "-h") => {
+            println!(
+                "Clusterflux node worker.\n\n\
+                 Usage: clusterflux-node --coordinator <URL> [OPTIONS]\n\n\
+                 Options:\n  \
+                   --coordinator <URL>\n  \
+                   --tenant <TENANT>                   [default: tenant]\n  \
+                   --project-id <PROJECT>              [default: project]\n  \
+                   --node <NODE>                       [default: node]\n  \
+                   --project-root <PATH>\n  \
+                   --enrollment-grant <GRANT>\n  \
+                   --public-key <KEY>\n  \
+                   --worker\n  \
+                   --emit-ready\n  \
+                   --control-poll-ms <MILLISECONDS>\n  \
+                   --assignment-poll-ms <MILLISECONDS> [default: 500]\n  \
+                   -h, --help\n  \
+                   -V, --version"
+            );
+            return Ok(());
+        }
+        _ => {}
+    }
     daemon::run()
 }

crates/clusterflux-node/src/task_reports.rs

@@ -47,8 +47,8 @@ pub(crate) fn record_completed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": output.stdout.len(),
-            "stderr_bytes": output.stderr.len(),
+            "stdout_bytes": output.stdout_source_bytes,
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -85,8 +85,8 @@ pub(crate) fn record_completed_task(
             "node": &args.node,
             "task": &task.task,
             "status_code": output.status_code,
-            "stdout_bytes": output.stdout.len(),
-            "stderr_bytes": output.stderr.len(),
+            "stdout_bytes": output.stdout_source_bytes,
+            "stderr_bytes": output.stderr_source_bytes,
             "stdout_tail": &output.stdout,
             "stderr_tail": &output.stderr,
             "stdout_truncated": output.stdout_truncated,
@@ -123,8 +123,11 @@ pub(crate) fn record_failed_task(
     debug_command: Value,
     node_private_key: &str,
     error: &str,
+    stdout_source_bytes: u64,
+    command_stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let error = bounded_runtime_error(error);
+    let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64);
     let log_event = session.request(signed_node_request_json(
         args,
         node_private_key,
@@ -136,8 +139,8 @@ pub(crate) fn record_failed_task(
             "process": &task.process,
             "node": &args.node,
             "task": &task.task,
-            "stdout_bytes": 0,
-            "stderr_bytes": error.len(),
+            "stdout_bytes": stdout_source_bytes,
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -175,8 +178,8 @@ pub(crate) fn record_failed_task(
             "task": &task.task,
             "terminal_state": "failed",
             "status_code": -1,
-            "stdout_bytes": 0,
-            "stderr_bytes": error.len(),
+            "stdout_bytes": stdout_source_bytes,
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": &error,
             "stdout_truncated": false,
@@ -189,6 +192,8 @@ pub(crate) fn record_failed_task(
     Ok(failed_node_report(
         task,
         &error,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -210,6 +215,8 @@ pub(crate) fn record_cancelled_task(
     capability_report: Value,
     debug_command: Value,
     node_private_key: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
 ) -> Result<Value, Box<dyn std::error::Error>> {
     let recorded = session.request(signed_node_request_json(
         args,
@@ -224,12 +231,12 @@ pub(crate) fn record_cancelled_task(
             "task": &task.task,
             "terminal_state": "cancelled",
             "status_code": null,
-            "stdout_bytes": 0,
-            "stderr_bytes": 0,
+            "stdout_bytes": stdout_source_bytes,
+            "stderr_bytes": stderr_source_bytes,
             "stdout_tail": "",
             "stderr_tail": "",
-            "stdout_truncated": false,
-            "stderr_truncated": false,
+            "stdout_truncated": stdout_source_bytes > 0,
+            "stderr_truncated": stderr_source_bytes > 0,
             "artifact_path": null,
             "artifact_digest": null,
             "artifact_size_bytes": null,
@@ -238,6 +245,8 @@ pub(crate) fn record_cancelled_task(
     )?)?;
     Ok(cancelled_node_report(
         task,
+        stdout_source_bytes,
+        stderr_source_bytes,
         registration,
         heartbeat,
         capability_report,
@@ -281,8 +290,8 @@ pub(crate) fn completed_node_report(
         "virtual_thread": output.virtual_thread,
         "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
         "status_code": output.status_code,
-        "stdout_bytes": output.stdout.len(),
-        "stderr_bytes": output.stderr.len(),
+        "stdout_bytes": output.stdout_source_bytes,
+        "stderr_bytes": output.stderr_source_bytes,
         "stdout_tail": &output.stdout,
         "stderr_tail": &output.stderr,
         "stdout_truncated": output.stdout_truncated,
@@ -305,6 +314,8 @@ pub(crate) fn completed_node_report(
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn cancelled_node_report(
     task: &RuntimeTask,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -318,12 +329,12 @@ pub(crate) fn cancelled_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "cancelled",
         "status_code": null,
-        "stdout_bytes": 0,
-        "stderr_bytes": 0,
+        "stdout_bytes": stdout_source_bytes,
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": "",
-        "stdout_truncated": false,
-        "stderr_truncated": false,
+        "stdout_truncated": stdout_source_bytes > 0,
+        "stderr_truncated": stderr_source_bytes > 0,
         "log_backpressured": false,
         "staged_artifact": null,
         "large_bytes_uploaded": false,
@@ -343,6 +354,8 @@ pub(crate) fn cancelled_node_report(
 pub(crate) fn failed_node_report(
     task: &RuntimeTask,
     error: &str,
+    stdout_source_bytes: u64,
+    stderr_source_bytes: u64,
     registration_response: Value,
     heartbeat_response: Value,
     capability_response: Value,
@@ -357,8 +370,8 @@ pub(crate) fn failed_node_report(
         "virtual_thread": &task.task,
         "terminal_state": "failed",
         "status_code": -1,
-        "stdout_bytes": 0,
-        "stderr_bytes": error.len(),
+        "stdout_bytes": stdout_source_bytes,
+        "stderr_bytes": stderr_source_bytes,
         "stdout_tail": "",
         "stderr_tail": error,
         "stdout_truncated": false,
@@ -377,3 +390,40 @@ pub(crate) fn failed_node_report(
         "coordinator_response": coordinator_response,
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use clusterflux_core::TaskInstanceId;
+
+    #[test]
+    fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() {
+        let report = completed_node_report(
+            CommandOutput {
+                virtual_thread: TaskInstanceId::from("task"),
+                status_code: Some(0),
+                stdout: "bounded tail".to_owned(),
+                stderr: String::new(),
+                stdout_source_bytes: 519,
+                stderr_source_bytes: 0,
+                stdout_truncated: true,
+                stderr_truncated: false,
+                log_backpressured: false,
+                staged_artifact: None,
+            },
+            false,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            Value::Null,
+            1,
+        );
+
+        assert_eq!(report["stdout_bytes"], 519);
+        assert_eq!(report["stdout_tail"], "bounded tail");
+    }
+}

docs/artifacts.md

@@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context.
 The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
 transfers. The CLI verifies the final digest. Hosted policy may impose
 per-project, per-tenant, and per-account size, concurrency, and period limits.
-Operators may disable relay traffic as an emergency safety control; one tenant's
-period usage does not consume a shared customer byte quota.
+There is no hosted global circuit breaker, and one tenant's period usage does
+not consume a shared customer byte quota.

docs/contributing/releases.md

@@ -1,60 +0,0 @@
-# Release candidates
-
-This is a contributor and release-engineering procedure, not an end-user setup
-path. Publication is a three-stage transaction:
-
-1. `candidate` builds immutable archives and a manifest with paths relative to
-   the manifest directory.
-2. `live-test` downloads that exact candidate in a clean job, deploys it, and
-   records the full 25-check production-shaped acceptance result plus deployment,
-   runtime configuration, and proxy configuration identities.
-3. `final` downloads the candidate and evidence in another clean job, verifies
-   every binding, and publishes without rebuilding any binary.
-
-Set `CLUSTERFLUX_RELEASE_STAGE=candidate` while creating the candidate and
-`CLUSTERFLUX_RELEASE_STAGE=final` while finalizing it. The final stage requires
-`CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST` and complete live evidence. There is no
-incomplete-evidence publication override.
-
-The `clusterflux-release` runner must provide `CLUSTERFLUX_DEPLOY_COMMAND` as a
-protected secret. The command runs locally with
-`CLUSTERFLUX_CANDIDATE_ARCHIVE`, `CLUSTERFLUX_CANDIDATE_COORDINATOR`, and their
-SHA-256 identities exported. It must deploy that executable and restart the
-configured service. `scripts/deploy-release-candidate.sh` then independently
-compares the running `/proc/<MainPID>/exe` digest with the candidate and records
-the service and proxy unit identities; a mismatch stops the release.
-
-Clusterflux release binaries are built once. The public client/node archive and
-the private-source hosted-service archive are both digest-bound to the same
-candidate. The hosted archive is deployed, the strict production-shaped batch
-uses the public archive against it, and finalization copies both archives
-without rebuilding.
-
-Create the candidate in a dedicated directory:
-
-~~~bash
-CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/release-candidate \
-CLUSTERFLUX_RELEASE_STAGE=candidate \
-./scripts/prepare-public-release.js
-~~~
-
-Deploy `target/release-candidate/assets/clusterflux-public-binaries-*.tar.gz`.
-Set `CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST` to the candidate manifest while
-running the strict batch. Set `CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH` to the
-JSON record from the private and public acceptance commands. The result records
-the source commit, source-tree and
-public-tree identities, candidate binary digests, deployment generation, and
-configuration identity.
-
-Finalize into a different directory after the strict result passes:
-
-~~~bash
-CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/public-release \
-CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST=target/release-candidate/public-release-manifest.json \
-CLUSTERFLUX_FINAL_RESULT_PATH=target/acceptance/cli-happy-path-live.json \
-./scripts/prepare-public-release.js
-~~~
-
-Finalization rejects a changed commit, source tree, public tree, candidate
-archive, binary digest set, deployment binding, or strict result. It never runs
-the release binary build when a candidate manifest is supplied.

docs/getting-started.md

@@ -11,6 +11,9 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node
 cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
 ~~~
 
+On NixOS or another system with Nix, the equivalent package is available with
+`nix profile install .#clusterflux-tools`.
+
 Install rootless Podman on each Linux node that will execute container-backed
 environments.
 
@@ -52,7 +55,7 @@ same stored identity:
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <hosted-project-id> \
   --node workstation \

docs/nodes.md

@@ -23,7 +23,7 @@ key pair.
 
 ~~~bash
 clusterflux-node \
-  --coordinator https://clusterflux.michelpaulissen.com \
+  --coordinator https://clusterflux.lesstuff.com \
   --tenant "$TENANT" \
   --project-id <project-id> \
   --node workstation \

flake.nix

@@ -9,6 +9,18 @@
       forAllSystems = nixpkgs.lib.genAttrs systems;
     in
     {
+      packages = forAllSystems (system:
+        let
+          pkgs = import nixpkgs { inherit system; };
+          publicPackages = import ./packages.nix { inherit pkgs self; };
+          privatePackages =
+            if builtins.pathExists ./web/packages.nix then
+              import ./web/packages.nix { inherit pkgs self; }
+            else
+              { };
+        in
+        publicPackages // privatePackages);
+
       devShells = forAllSystems (system:
         let
           pkgs = import nixpkgs { inherit system; };

packages.nix

@@ -0,0 +1,74 @@
+{ pkgs, self }:
+let
+  clusterflux-tools = pkgs.rustPlatform.buildRustPackage {
+    pname = "clusterflux-tools";
+    version = "0.1.0";
+    src = self;
+    cargoLock.lockFile = ./Cargo.lock;
+    nativeBuildInputs = [
+      pkgs.git
+      pkgs.lld
+      pkgs.makeWrapper
+    ];
+    cargoBuildFlags = [
+      "--package"
+      "clusterflux-cli"
+      "--package"
+      "clusterflux-node"
+      "--package"
+      "clusterflux-coordinator"
+      "--package"
+      "clusterflux-dap"
+    ];
+    cargoTestFlags = [
+      "--package"
+      "clusterflux-cli"
+      "--package"
+      "clusterflux-node"
+      "--package"
+      "clusterflux-coordinator"
+      "--package"
+      "clusterflux-dap"
+    ];
+    NIX_BUILD_CORES = "2";
+    RUST_MIN_STACK = "1073741824";
+    postInstall = ''
+      test -x "$out/bin/clusterflux"
+      test -x "$out/bin/clusterflux-node"
+      test -x "$out/bin/clusterflux-coordinator"
+      test -x "$out/bin/clusterflux-debug-dap"
+      for command in \
+        clusterflux \
+        clusterflux-node \
+        clusterflux-coordinator \
+        clusterflux-debug-dap
+      do
+        ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --version >/dev/null
+        ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --help >/dev/null
+      done
+    '';
+    postFixup =
+      let
+        runtimePath = pkgs.lib.makeBinPath [
+          pkgs.cargo
+          pkgs.git
+          pkgs.lld
+          pkgs.rustc
+        ];
+      in
+      ''
+        wrapProgram "$out/bin/clusterflux" --prefix PATH : ${runtimePath}
+        wrapProgram "$out/bin/clusterflux-node" --prefix PATH : ${runtimePath}
+        wrapProgram "$out/bin/clusterflux-debug-dap" --prefix PATH : ${runtimePath}
+      '';
+    meta = {
+      description = "Clusterflux CLI, node, coordinator, and debugger adapter";
+      mainProgram = "clusterflux";
+    };
+  };
+in
+{
+  inherit clusterflux-tools;
+  clusterflux = clusterflux-tools;
+  default = clusterflux-tools;
+}

scripts/acceptance-private.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$repo"
-
-if [[ ! -d private/hosted-policy ]]; then
-  exit 0
-fi
-
-scripts/check-old-name.sh
-node scripts/check-docs.js
-scripts/check-code-size.sh
-cargo fmt --all --manifest-path private/hosted-policy/Cargo.toml --check
-cargo clippy --manifest-path private/hosted-policy/Cargo.toml --all-targets -- -D warnings
-cargo test --locked --manifest-path private/hosted-policy/Cargo.toml --all-targets
-node private/hosted-policy/scripts/prepare-hosted-deployment.js
-if command -v podman >/dev/null 2>&1; then
-  node private/hosted-policy/scripts/postgres-durable-smoke.js
-elif command -v nix >/dev/null 2>&1; then
-  nix shell nixpkgs#podman --command node private/hosted-policy/scripts/postgres-durable-smoke.js
-else
-  node private/hosted-policy/scripts/postgres-durable-smoke.js
-fi
-if [[ -n "${CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR:-}" ]]; then
-  node private/hosted-policy/scripts/hosted-service-live-check.js
-fi

scripts/acceptance-public.sh

@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$repo"
-
-scripts/check-old-name.sh
-node scripts/check-docs.js
-scripts/check-code-size.sh
-scripts/release-source-scan.sh
-cargo fmt --all --check
-cargo clippy --workspace --all-targets -- -D warnings
-cargo test --workspace --all-targets
-cargo build --workspace --all-targets
-cargo build -p runtime-conformance --target wasm32-unknown-unknown
-cargo build -p hello-build --target wasm32-unknown-unknown
-cargo build -p recovery-build --target wasm32-unknown-unknown
-node scripts/resource-metering-contract-smoke.js
-node scripts/hostile-input-contract-smoke.js
-node scripts/tenant-isolation-contract-smoke.js
-node scripts/self-hosted-coordinator-smoke.js
-node scripts/public-local-demo-matrix-smoke.js
-node scripts/cli-output-mode-smoke.js
-node scripts/cli-login-smoke.js
-node scripts/cli-error-exit-smoke.js
-node scripts/cli-browser-login-flow-smoke.js
-node scripts/cli-install-smoke.js
-node scripts/user-session-token-boundary-smoke.js
-node scripts/sdk-spawn-runtime-smoke.js
-node scripts/node-lifecycle-contract-smoke.js
-node scripts/wasmtime-node-smoke.js
-node scripts/wasmtime-assignment-smoke.js
-if command -v podman >/dev/null 2>&1; then
-  node scripts/podman-backend-smoke.js
-elif command -v nix >/dev/null 2>&1; then
-  nix shell nixpkgs#podman --command node scripts/podman-backend-smoke.js
-else
-  node scripts/podman-backend-smoke.js
-fi
-node scripts/vscode-extension-smoke.js
-node scripts/vscode-f5-smoke.js
-node scripts/node-attach-smoke.js
-node scripts/cli-local-run-smoke.js
-node scripts/artifact-download-smoke.js
-node scripts/artifact-export-smoke.js
-node scripts/operator-panel-smoke.js
-node scripts/source-preparation-smoke.js
-node scripts/scheduler-placement-smoke.js
-node scripts/windows-best-effort-smoke.js
-node scripts/quic-smoke.js
-node scripts/dap-smoke.js
-node scripts/recovery-build-smoke.js
-node scripts/flagship-demo-smoke.js
-scripts/verify-public-split.sh

scripts/agent-signing.js

@@ -1,99 +0,0 @@
-const crypto = require("crypto");
-
-const { nodeIdentity, signedRequestPayloadDigest } = require("./node-signing");
-
-function agentIdentity(seedPrefix, agent) {
-  const identity = nodeIdentity(seedPrefix, agent);
-  return {
-    ...identity,
-    publicKeyFingerprint: `sha256:${crypto
-      .createHash("sha256")
-      .update(identity.publicKey)
-      .digest("hex")}`,
-  };
-}
-
-function agentWorkflowSignatureMessage({
-  tenant,
-  project,
-  agent,
-  requestKind,
-  process: processId,
-  task = "",
-  payloadDigest,
-  nonce,
-  issuedAtEpochSeconds,
-}) {
-  const parts = [
-    "clusterflux-agent-workflow-signature:v2",
-    tenant,
-    project,
-    agent,
-    requestKind,
-    processId,
-    task,
-    payloadDigest,
-    nonce,
-    String(issuedAtEpochSeconds),
-  ];
-  return Buffer.concat(
-    parts.flatMap((part) => [
-      Buffer.from(`${Buffer.byteLength(part)}:`),
-      Buffer.from(part),
-      Buffer.from("\n"),
-    ])
-  );
-}
-
-function signedAgentWorkflowProof(identity, request, options = {}) {
-  const nonce =
-    options.nonce ||
-    `${request.type}-${process.pid}-${Date.now()}-${crypto
-      .randomBytes(8)
-      .toString("hex")}`;
-  const issuedAtEpochSeconds =
-    options.issuedAtEpochSeconds || Math.floor(Date.now() / 1000);
-  const processId =
-    request.type === "launch_task" ? request.task_spec?.process : request.process;
-  const task =
-    request.type === "launch_task" ? request.task_spec?.task_instance : request.task || "";
-  const signature = crypto.sign(
-    null,
-    agentWorkflowSignatureMessage({
-      tenant: request.tenant,
-      project: request.project,
-      agent: request.actor_agent,
-      requestKind: request.type,
-      process: processId,
-      task,
-      payloadDigest: signedRequestPayloadDigest(request),
-      nonce,
-      issuedAtEpochSeconds,
-    }),
-    identity.privateKeyObject
-  );
-  return {
-    nonce,
-    issued_at_epoch_seconds: issuedAtEpochSeconds,
-    signature: `ed25519:${signature.toString("base64")}`,
-  };
-}
-
-function signedAgentWorkflowRequest(identity, request, options = {}) {
-  const unsignedRequest = {
-    ...request,
-    agent_public_key_fingerprint:
-      options.publicKeyFingerprint || identity.publicKeyFingerprint,
-  };
-  return {
-    ...unsignedRequest,
-    agent_signature: signedAgentWorkflowProof(identity, unsignedRequest, options),
-  };
-}
-
-module.exports = {
-  agentIdentity,
-  agentWorkflowSignatureMessage,
-  signedAgentWorkflowProof,
-  signedAgentWorkflowRequest,
-};

scripts/artifact-download-smoke.js

@@ -1,401 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const crypto = require("crypto");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-const { nodeIdentity, signedNodeRequest } = require("./node-signing");
-const {
-  ensureRootlessPodman,
-  flagshipNodeCapabilities,
-  launchFlagship,
-  repo,
-  runFlagshipWorker,
-  send,
-  waitForJsonLine,
-} = require("./real-flagship-harness");
-
-const downloadNode = "node-download";
-const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode);
-
-const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
-
-async function downloadRetainedBytes(addr, link, artifact, expectedSize) {
-  const chunks = [];
-  let offset = 0;
-  for (let attempt = 0; attempt < 500; attempt += 1) {
-    const response = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: link.link.scoped_token_digest,
-      chunk_bytes: 256 * 1024,
-    });
-    assert.strictEqual(response.type, "artifact_download_stream");
-    if (!response.content_bytes_available) {
-      assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending");
-      await delay(10);
-      continue;
-    }
-    assert.strictEqual(response.content_source, "retaining_node_reverse_stream");
-    assert.strictEqual(response.content_offset, offset);
-    const bytes = Buffer.from(response.content_base64, "base64");
-    assert.strictEqual(response.streamed_bytes, bytes.length);
-    chunks.push(bytes);
-    offset += bytes.length;
-    if (response.content_eof) {
-      const content = Buffer.concat(chunks);
-      assert.strictEqual(content.length, expectedSize);
-      return { content, response };
-    }
-  }
-  throw new Error("timed out waiting for retained artifact reverse stream");
-}
-
-function downloadNodeCapabilities() {
-  return flagshipNodeCapabilities();
-}
-
-(async () => {
-  ensureRootlessPodman();
-  const coordinator = cp.spawn(
-    "cargo",
-    [
-      "run",
-      "-q",
-      "-p",
-      "clusterflux-coordinator",
-      "--bin",
-      "clusterflux-coordinator",
-      "--",
-      "--listen",
-      "127.0.0.1:0",
-      "--allow-local-trusted-loopback",
-    ],
-    { cwd: repo }
-  );
-
-  let worker;
-  try {
-    const ready = await waitForJsonLine(coordinator);
-    const [host, portText] = ready.listen.split(":");
-    const addr = { host, port: Number(portText) };
-    assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
-
-    worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity);
-    const workerReady = await worker.ready;
-    assert.strictEqual(workerReady.node_status, "ready");
-    assert.strictEqual(workerReady.mode, "worker");
-    const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr);
-    assert.strictEqual(compileEvent.status_code, 0);
-    assert.strictEqual(packageEvent.status_code, 0);
-    assert.deepStrictEqual(compileEvent.result, {
-      Artifact: {
-        id: compileEvent.artifact_path.slice("/vfs/artifacts/".length),
-        digest: compileEvent.artifact_digest,
-        size_bytes: compileEvent.artifact_size_bytes,
-      },
-    });
-    assert.deepStrictEqual(packageEvent.result, {
-      Artifact: {
-        id: packageEvent.artifact_path.slice("/vfs/artifacts/".length),
-        digest: packageEvent.artifact_digest,
-        size_bytes: packageEvent.artifact_size_bytes,
-      },
-    });
-    assert.ok(compileEvent.artifact_size_bytes > 0);
-    assert.strictEqual(packageEvent.artifact_size_bytes, compileEvent.artifact_size_bytes);
-    assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
-    assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
-    const artifactPath = packageEvent.artifact_path;
-    assert.match(
-      artifactPath,
-      /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/
-    );
-    const artifact = artifactPath.slice("/vfs/artifacts/".length);
-
-    const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
-      type: "report_node_capabilities",
-      tenant: "tenant",
-      project: "project",
-      node: downloadNode,
-      capabilities: downloadNodeCapabilities(),
-      cached_environment_digests: [],
-      dependency_cache_digests: [],
-      source_snapshots: [],
-      artifact_locations: [artifact],
-      direct_connectivity: false,
-      online: true,
-    }));
-    assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded");
-
-    const disconnectedLink = await send(addr, {
-      type: "create_artifact_download_link",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      ttl_seconds: 60,
-    });
-    assert.strictEqual(disconnectedLink.type, "artifact_download_link");
-
-    const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
-      type: "report_node_capabilities",
-      tenant: "tenant",
-      project: "project",
-      node: downloadNode,
-      capabilities: downloadNodeCapabilities(),
-      cached_environment_digests: [],
-      dependency_cache_digests: [],
-      source_snapshots: [],
-      artifact_locations: [artifact],
-      direct_connectivity: true,
-      online: true,
-    }));
-    assert.strictEqual(connectedReport.type, "node_capabilities_recorded");
-
-    const link = await send(addr, {
-      type: "create_artifact_download_link",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      ttl_seconds: 60,
-    });
-    assert.strictEqual(link.type, "artifact_download_link");
-    assert.strictEqual(link.link.tenant, "tenant");
-    assert.strictEqual(link.link.project, "project");
-    assert.strictEqual(link.link.process, virtualProcess);
-    assert.deepStrictEqual(link.link.actor, { User: "user" });
-    assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
-    assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000));
-    assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60);
-    assert.ok(
-      link.link.url_path.endsWith(
-        `/artifacts/tenant/project/${virtualProcess}/${artifact}`
-      )
-    );
-    assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" });
-
-    const crossTenant = await send(addr, {
-      type: "create_artifact_download_link",
-      tenant: "other",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      ttl_seconds: 60,
-    });
-    assert.strictEqual(crossTenant.type, "error");
-    assert.match(crossTenant.message, /tenant mismatch/);
-
-    const crossProject = await send(addr, {
-      type: "create_artifact_download_link",
-      tenant: "tenant",
-      project: "other-project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      ttl_seconds: 60,
-    });
-    assert.strictEqual(crossProject.type, "error");
-    assert.match(crossProject.message, /project mismatch/);
-
-    const crossTenantOpen = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "other",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: link.link.scoped_token_digest,
-      chunk_bytes: 1,
-    });
-    assert.strictEqual(crossTenantOpen.type, "error");
-    assert.match(crossTenantOpen.message, /tenant mismatch/);
-
-    const crossProjectOpen = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "tenant",
-      project: "other-project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: link.link.scoped_token_digest,
-      chunk_bytes: 1,
-    });
-    assert.strictEqual(crossProjectOpen.type, "error");
-    assert.match(crossProjectOpen.message, /project mismatch/);
-
-    const guessed = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: "sha256:guessed",
-      chunk_bytes: 1,
-    });
-    assert.strictEqual(guessed.type, "error");
-    assert.match(guessed.message, /token is invalid/);
-
-    const crossActorOpen = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "other-user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: link.link.scoped_token_digest,
-      chunk_bytes: 1,
-    });
-    assert.strictEqual(crossActorOpen.type, "error");
-    assert.match(crossActorOpen.message, /token is invalid/);
-
-    const downloaded = await downloadRetainedBytes(
-      addr,
-      link,
-      artifact,
-      packageEvent.artifact_size_bytes,
-    );
-    const downloadedDigest = `sha256:${crypto
-      .createHash("sha256")
-      .update(downloaded.content)
-      .digest("hex")}`;
-    assert.strictEqual(downloadedDigest, packageEvent.artifact_digest);
-    const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-"));
-    try {
-      const executable = path.join(inspect, "hello-clusterflux");
-      fs.writeFileSync(executable, downloaded.content);
-      fs.chmodSync(executable, 0o755);
-      assert.strictEqual(
-        cp.execFileSync(executable, { encoding: "utf8" }),
-        "hello from a real Clusterflux build\n"
-      );
-    } finally {
-      fs.rmSync(inspect, { recursive: true, force: true });
-    }
-    const cliDownloadDirectory = fs.mkdtempSync(
-      path.join(os.tmpdir(), "clusterflux-cli-download-")
-    );
-    try {
-      const cliDownloadPath = path.join(cliDownloadDirectory, "hello-clusterflux");
-      const cliDownload = JSON.parse(
-        cp.execFileSync(
-          "cargo",
-          [
-            "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--",
-            "artifact", "download", artifact,
-            "--to", cliDownloadPath,
-            "--max-bytes", "1048576",
-            "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`,
-            "--tenant", "tenant",
-            "--project-id", "project",
-            "--user", "user",
-            "--json",
-          ],
-          { cwd: repo, env: process.env, encoding: "utf8" }
-        )
-      );
-      assert.strictEqual(cliDownload.command, "artifact download");
-      assert.strictEqual(cliDownload.local_download.status, "local_bytes_written");
-      assert.strictEqual(
-        cliDownload.local_download.verified_digest,
-        packageEvent.artifact_digest
-      );
-      assert.strictEqual(
-        `sha256:${crypto
-          .createHash("sha256")
-          .update(fs.readFileSync(cliDownloadPath))
-          .digest("hex")}`,
-        packageEvent.artifact_digest
-      );
-    } finally {
-      fs.rmSync(cliDownloadDirectory, { recursive: true, force: true });
-    }
-    assert.strictEqual(downloaded.response.content_eof, true);
-    assert.strictEqual(
-      downloaded.response.charged_download_bytes,
-      packageEvent.artifact_size_bytes
-    );
-    assert.strictEqual(downloaded.response.link.artifact, artifact);
-
-    const crossActorRevoke = await send(addr, {
-      type: "revoke_artifact_download_link",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "other-user",
-      artifact,
-      token_digest: link.link.scoped_token_digest,
-    });
-    assert.strictEqual(crossActorRevoke.type, "error");
-    assert.match(crossActorRevoke.message, /token is invalid/);
-
-    const revoked = await send(addr, {
-      type: "revoke_artifact_download_link",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      token_digest: link.link.scoped_token_digest,
-    });
-    assert.strictEqual(revoked.type, "artifact_download_link_revoked");
-    assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest);
-
-    const revokedOpen = await send(addr, {
-      type: "open_artifact_download_stream",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      token_digest: link.link.scoped_token_digest,
-      chunk_bytes: 1,
-    });
-    assert.strictEqual(revokedOpen.type, "error");
-    assert.match(revokedOpen.message, /revoked/);
-
-    const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
-      type: "report_node_capabilities",
-      tenant: "tenant",
-      project: "project",
-      node: downloadNode,
-      capabilities: downloadNodeCapabilities(),
-      cached_environment_digests: [],
-      dependency_cache_digests: [],
-      source_snapshots: [],
-      artifact_locations: [],
-      direct_connectivity: false,
-      online: true,
-    }));
-    assert.strictEqual(gcReport.type, "node_capabilities_recorded");
-
-    const collectedLink = await send(addr, {
-      type: "create_artifact_download_link",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      max_bytes: 1024 * 1024,
-      ttl_seconds: 60,
-    });
-    assert.strictEqual(collectedLink.type, "error");
-    assert.match(collectedLink.message, /unavailable from current retention/);
-  } finally {
-    worker?.child.kill("SIGTERM");
-    coordinator.kill("SIGTERM");
-  }
-
-  console.log("Artifact download smoke passed");
-})().catch((error) => {
-  console.error(error.stack || error.message);
-  process.exit(1);
-});

scripts/artifact-export-smoke.js

@@ -1,275 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-const { nodeIdentity, signedNodeRequest } = require("./node-signing");
-const {
-  ensureRootlessPodman,
-  flagshipNodeCapabilities,
-  launchFlagship,
-  repo,
-  runFlagshipWorker,
-  send,
-  waitForJsonLine,
-  waitForNodeStatus,
-} = require("./real-flagship-harness");
-
-const sourceNode = "node-export-source";
-const sourceIdentity = nodeIdentity("artifact-export-smoke", sourceNode);
-
-function nodeCapabilities() {
-  return flagshipNodeCapabilities();
-}
-
-function runJson(command, args, options = {}) {
-  return new Promise((resolve, reject) => {
-    const child = cp.spawn(command, args, { cwd: repo, ...options });
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk.toString();
-    });
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk.toString();
-    });
-    child.once("error", reject);
-    child.once("exit", (code) => {
-      if (code !== 0) {
-        reject(
-          new Error(
-            `${command} ${args.join(" ")} failed with code ${code}\n${stderr}\n${stdout}`
-          )
-        );
-        return;
-      }
-      try {
-        resolve(JSON.parse(stdout));
-      } catch (error) {
-        reject(new Error(`${command} did not return JSON\n${stdout}\n${error.message}`));
-      }
-    });
-  });
-}
-
-async function reportNode(
-  addr,
-  node,
-  identity,
-  { directConnectivity = true, online = true, artifacts = [] } = {}
-) {
-  const response = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", {
-    type: "report_node_capabilities",
-    tenant: "tenant",
-    project: "project",
-    node,
-    capabilities: nodeCapabilities(),
-    cached_environment_digests: [],
-    dependency_cache_digests: [],
-    source_snapshots: [],
-    artifact_locations: artifacts,
-    direct_connectivity: directConnectivity,
-    online,
-  }));
-  assert.strictEqual(response.type, "node_capabilities_recorded");
-  assert.strictEqual(response.node, node);
-}
-
-(async () => {
-  ensureRootlessPodman();
-  const coordinator = cp.spawn(
-    "cargo",
-    [
-      "run",
-      "-q",
-      "-p",
-      "clusterflux-coordinator",
-      "--bin",
-      "clusterflux-coordinator",
-      "--",
-      "--listen",
-      "127.0.0.1:0",
-      "--allow-local-trusted-loopback",
-    ],
-    {
-      cwd: repo,
-      env: { ...process.env, CLUSTERFLUX_NODE_STALE_AFTER_SECONDS: "1" },
-    }
-  );
-
-  let worker;
-  try {
-    const ready = await waitForJsonLine(coordinator);
-    const [host, portText] = ready.listen.split(":");
-    const addr = { host, port: Number(portText) };
-    assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
-
-    worker = await runFlagshipWorker(addr, sourceNode, sourceIdentity);
-    const workerReady = await worker.ready;
-    assert.strictEqual(workerReady.node_status, "ready");
-    assert.strictEqual(workerReady.mode, "worker");
-    const firstNodeTaskCompletion = waitForNodeStatus(worker.child, "completed");
-    const { compileEvent, process: virtualProcess } = await launchFlagship(addr);
-    const workerCompletion = await firstNodeTaskCompletion;
-    assert.strictEqual(workerCompletion.node_status, "completed");
-    assert.strictEqual(
-      workerCompletion.task_assignment_response.task_spec.task_definition,
-      "snapshot_current_project"
-    );
-    assert.strictEqual(
-      workerCompletion.virtual_thread,
-      workerCompletion.task_assignment_response.task_spec.task_instance
-    );
-    assert.strictEqual(compileEvent.status_code, 0);
-    assert.match(
-      compileEvent.artifact_path,
-      /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/
-    );
-    const artifact = compileEvent.artifact_path.slice("/vfs/artifacts/".length);
-
-    await reportNode(addr, sourceNode, sourceIdentity, {
-      artifacts: [artifact],
-    });
-
-    const receiverIdentity = nodeIdentity("artifact-export-smoke", "node-export-receiver");
-    const attachedReceiver = await send(addr, {
-      type: "attach_node",
-      tenant: "tenant",
-      project: "project",
-      node: "node-export-receiver",
-      public_key: receiverIdentity.publicKey,
-    });
-    assert.strictEqual(attachedReceiver.type, "node_attached");
-    await reportNode(addr, "node-export-receiver", receiverIdentity);
-
-    const exportPlan = await send(addr, {
-      type: "export_artifact_to_node",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      receiver_node: "node-export-receiver",
-      direct_connectivity: true,
-      failure_reason: "",
-    });
-    assert.strictEqual(exportPlan.type, "artifact_export_plan");
-    assert.strictEqual(exportPlan.source_node, "node-export-source");
-    assert.strictEqual(exportPlan.receiver_node, "node-export-receiver");
-    assert.strictEqual(exportPlan.plan.transport, "NativeQuic");
-    assert.strictEqual(exportPlan.plan.scope.tenant, "tenant");
-    assert.strictEqual(exportPlan.plan.scope.project, "project");
-    assert.strictEqual(exportPlan.plan.scope.process, virtualProcess);
-    assert.deepStrictEqual(exportPlan.plan.scope.object, { Artifact: artifact });
-    assert.strictEqual(exportPlan.plan.source.node, "node-export-source");
-    assert.strictEqual(exportPlan.plan.destination.node, "node-export-receiver");
-    assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true);
-    assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false);
-    assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/);
-    assert.strictEqual(exportPlan.artifact_size_bytes, compileEvent.artifact_size_bytes);
-
-    const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-artifact-export-"));
-    const exportPath = path.join(temp, "hello-clusterflux");
-    cp.execFileSync(
-      "cargo",
-      ["build", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux"],
-      { cwd: repo, stdio: "inherit" }
-    );
-    const cliBinary = path.join(
-      path.resolve(repo, process.env.CARGO_TARGET_DIR || "target"),
-      "debug",
-      process.platform === "win32" ? "clusterflux.exe" : "clusterflux"
-    );
-    await reportNode(addr, sourceNode, sourceIdentity, {
-      artifacts: [artifact],
-    });
-    const cliExport = await runJson(cliBinary, [
-      "artifact",
-      "export",
-      "--coordinator",
-      `${addr.host}:${addr.port}`,
-      "--tenant",
-      "tenant",
-      "--project-id",
-      "project",
-      "--user",
-      "user",
-      "--json",
-      artifact,
-      "--receiver-node",
-      "node-export-receiver",
-      "--to",
-      exportPath,
-    ]);
-    assert.strictEqual(cliExport.command, "artifact export");
-    assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true);
-    assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written");
-    assert.strictEqual(
-      cliExport.export_plan.bytes_written,
-      compileEvent.artifact_size_bytes
-    );
-    assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false);
-    assert.strictEqual(
-      cliExport.local_export.verified_digest,
-      compileEvent.artifact_digest
-    );
-    fs.chmodSync(exportPath, 0o755);
-    assert.strictEqual(
-      cp.execFileSync(exportPath, { encoding: "utf8" }),
-      "hello from a real Clusterflux build\n"
-    );
-
-    const crossTenant = await send(addr, {
-      type: "export_artifact_to_node",
-      tenant: "other",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      receiver_node: "node-export-receiver",
-      direct_connectivity: true,
-      failure_reason: "",
-    });
-    assert.strictEqual(crossTenant.type, "error");
-    assert.match(crossTenant.message, /tenant mismatch/);
-
-    await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] });
-    const failedDirect = await send(addr, {
-      type: "export_artifact_to_node",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      receiver_node: "node-export-receiver",
-      direct_connectivity: false,
-      failure_reason: "nat traversal failed",
-    });
-    assert.strictEqual(failedDirect.type, "error");
-    assert.match(failedDirect.message, /nat traversal failed/);
-    assert.match(failedDirect.message, /coordinator bulk relay is disabled/);
-
-    await reportNode(addr, "node-export-receiver", receiverIdentity, { online: false });
-    await new Promise((resolve) => setTimeout(resolve, 2100));
-    await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] });
-    const offlineReceiver = await send(addr, {
-      type: "export_artifact_to_node",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      artifact,
-      receiver_node: "node-export-receiver",
-      direct_connectivity: true,
-      failure_reason: "",
-    });
-    assert.strictEqual(offlineReceiver.type, "error");
-    assert.match(offlineReceiver.message, /offline/);
-  } finally {
-    worker?.child.kill("SIGTERM");
-    coordinator.kill("SIGTERM");
-  }
-
-  console.log("Artifact export smoke passed");
-})().catch((error) => {
-  console.error(error.stack || error.message);
-  process.exit(1);
-});

scripts/check-code-size.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$repo_root"
-
-maximum_lines=3000
-failed=0
-source_roots=(crates)
-if [[ -d private/hosted-policy/src ]]; then
-  source_roots+=(private/hosted-policy/src)
-fi
-while IFS= read -r -d '' file; do
-  case "$file" in
-    */tests.rs|*/tests/*) continue ;;
-  esac
-  lines="$(wc -l < "$file")"
-  if ((lines > maximum_lines)); then
-    printf '%s has %s lines; production file limit is %s\n' "$file" "$lines" "$maximum_lines" >&2
-    failed=1
-  fi
-done < <(find "${source_roots[@]}" -type f -name '*.rs' -print0)
-
-if ((failed)); then
-  exit 1
-fi
-printf 'production source file size guard passed (%s lines maximum)\n' "$maximum_lines"

scripts/check-docs.js

@@ -1,113 +0,0 @@
-#!/usr/bin/env node
-const fs = require("node:fs");
-const path = require("node:path");
-
-const root = path.resolve(__dirname, "..");
-const publicDocs = [
-  "README.md",
-  "SECURITY.md",
-  "docs/getting-started.md",
-  "docs/architecture.md",
-  "docs/nodes.md",
-  "docs/environments.md",
-  "docs/artifacts.md",
-  "docs/debugging.md",
-  "docs/task-abi.md",
-  "docs/self-hosting.md",
-  "docs/security.md",
-];
-const contributorDocs = ["docs/contributing/releases.md"];
-const privateDocs = [
-  "private/docs/hosted-deployment.md",
-  "private/docs/authentik.md",
-  "private/docs/community-policy.md",
-  "private/docs/bandwidth-and-cost-controls.md",
-  "private/docs/publishing.md",
-];
-const internalDocs = ["internal/finish_mvp_2.md"];
-const filteredPublicTree =
-  process.env.CLUSTERFLUX_FILTERED_PUBLIC_TREE === "1" ||
-  fs.existsSync(path.join(root, "CLUSTERFLUX_PUBLIC_TREE.json"));
-
-const failures = [];
-const expectExactMarkdownSet = (directory, expected) => {
-  const actual = fs
-    .readdirSync(path.join(root, directory), { withFileTypes: true })
-    .filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
-    .map((entry) => path.posix.join(directory, entry.name))
-    .sort();
-  const wanted = [...expected].sort();
-  if (JSON.stringify(actual) !== JSON.stringify(wanted)) {
-    failures.push(`${directory} markdown set is ${actual.join(", ")}; expected ${wanted.join(", ")}`);
-  }
-};
-
-const requiredDocs = filteredPublicTree
-  ? [...publicDocs, ...contributorDocs]
-  : [...publicDocs, ...contributorDocs, ...privateDocs, ...internalDocs];
-for (const file of requiredDocs) {
-  if (!fs.existsSync(path.join(root, file))) failures.push(`missing canonical document: ${file}`);
-}
-expectExactMarkdownSet("docs", publicDocs.filter((file) => file.startsWith("docs/")));
-expectExactMarkdownSet("docs/contributing", contributorDocs);
-if (filteredPublicTree) {
-  for (const directory of ["private", "internal"]) {
-    if (fs.existsSync(path.join(root, directory))) {
-      failures.push(`${directory}/ must not exist in the filtered public tree`);
-    }
-  }
-}
-
-const forbidden = [
-  [/\bmvp\b/i, "internal milestone term"],
-  [/acceptance criteria/i, "internal gate language"],
-  [/release verification/i, "internal release language"],
-  [/public\/private source split/i, "source split narrative"],
-  [/founder|business decisions/i, "business planning narrative"],
-  [/hacker news|\bHN\b/, "launch-channel narrative"],
-  [/\busers can\b/i, "indirect reader wording"],
-  [/node\s+scripts\//i, "developer script instruction"],
-  [/scripts\/[^\s)]*smoke/i, "smoke script instruction"],
-  [/internal\/[^\s)]*/i, "internal tooling reference"],
-  [/private\/[^\s)]*/i, "private source reference"],
-];
-const topLevelCommands = new Set([
-  "doctor", "login", "logout", "auth", "agent", "key", "project", "inspect",
-  "build", "bundle", "run", "node", "process", "task", "logs", "artifact",
-  "dap", "debug", "quota", "admin",
-]);
-
-for (const file of publicDocs) {
-  const absolute = path.join(root, file);
-  const content = fs.readFileSync(absolute, "utf8");
-  for (const [pattern, description] of forbidden) {
-    if (pattern.test(content)) failures.push(`${file}: contains ${description}`);
-  }
-  for (const match of content.matchAll(/\]\(([^)#]+)(?:#[^)]+)?\)/g)) {
-    const target = match[1];
-    if (/^(?:https?:|mailto:)/.test(target)) continue;
-    const resolved = path.resolve(path.dirname(absolute), target);
-    if (!fs.existsSync(resolved)) failures.push(`${file}: broken link ${target}`);
-  }
-  for (const line of content.split(/\r?\n/)) {
-    const command = line.trim().match(/^clusterflux\s+([a-z][a-z-]*)\b/);
-    if (command && !topLevelCommands.has(command[1])) {
-      failures.push(`${file}: unknown top-level CLI command ${command[1]}`);
-    }
-  }
-}
-
-const rootMarkdown = fs
-  .readdirSync(root, { withFileTypes: true })
-  .filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
-  .map((entry) => entry.name)
-  .sort();
-if (JSON.stringify(rootMarkdown) !== JSON.stringify(["README.md", "SECURITY.md"])) {
-  failures.push(`top-level markdown set is ${rootMarkdown.join(", ")}; expected README.md, SECURITY.md`);
-}
-
-if (failures.length) {
-  for (const failure of failures) console.error(failure);
-  process.exit(1);
-}
-console.log("documentation checks passed");

scripts/check-old-name.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$repo_root"
-
-legacy_lower="$(printf '%s%s' 'disa' 'smer')"
-legacy_title="$(printf '%s%s' 'Disa' 'smer')"
-legacy_upper="$(printf '%s%s' 'DISA' 'SMER')"
-pattern="${legacy_lower}|${legacy_title}|${legacy_upper}"
-
-allowed_content='^\./scripts/migrate-clusterflux-state\.sh:'
-matches="$(
-  rg -n --hidden \
-    --glob '!**/.git/**' \
-    --glob '!**/target/**' \
-    --glob '!**/node_modules/**' \
-    --glob '!**/vendor/**' \
-    --glob '!**/.direnv/**' \
-    --glob '!**/.cache/**' \
-    --glob '!**/dist/**' \
-    --glob '!**/out/**' \
-    "$pattern" . 2>/dev/null | rg -v "$allowed_content" || true
-)"
-
-paths="$(
-  find . \
-    \( -type d \( -name .git -o -name target -o -name node_modules -o -name vendor -o -name .direnv -o -name .cache -o -name dist -o -name out \) -prune \) -o \
-    -iname "*${legacy_lower}*" -print | sort || true
-)"
-
-if [[ -n "$matches" || -n "$paths" ]]; then
-  [[ -z "$matches" ]] || printf '%s\n' "$matches" >&2
-  [[ -z "$paths" ]] || printf '%s\n' "$paths" >&2
-  printf 'unexpected legacy product name remains\n' >&2
-  exit 1
-fi
-
-printf 'old-name guard passed\n'

scripts/cli-browser-login-flow-smoke.js

@@ -1,222 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const fs = require("fs");
-const net = require("net");
-const path = require("path");
-
-const repo = path.resolve(__dirname, "..");
-const tmp = path.join(repo, "target", "acceptance", "tmp", "cli-browser-login-flow");
-const project = path.join(tmp, "project");
-fs.rmSync(tmp, { recursive: true, force: true });
-fs.mkdirSync(project, { recursive: true });
-
-function writeOpener() {
-  const opener = path.join(tmp, "browser-opener.js");
-  const trace = path.join(tmp, "browser-opener.log");
-  fs.writeFileSync(
-    opener,
-    `#!/usr/bin/env node
-const fs = require("fs");
-const trace = ${JSON.stringify(trace)};
-const loginUrl = new URL(process.argv[2]);
-const state = loginUrl.searchParams.get("state");
-const nonce = loginUrl.searchParams.get("nonce");
-const challenge = loginUrl.searchParams.get("code_challenge");
-if (loginUrl.protocol !== "https:" || !state || !nonce || !challenge) {
-  fs.appendFileSync(trace, "missing server-owned OIDC parameters\\n");
-  process.exit(1);
-}
-fs.appendFileSync(trace, "server-owned browser transaction\\n");
-// Model a real browser/opener that remains alive after the CLI transaction.
-// Its descriptors must not keep the invoking CLI process open.
-setTimeout(() => process.exit(0), 5000);
-`
-  );
-  fs.chmodSync(opener, 0o755);
-  return opener;
-}
-
-function startCoordinator() {
-  return new Promise((resolve, reject) => {
-    const requests = [];
-    const server = net.createServer((socket) => {
-      let buffered = "";
-      socket.on("data", (chunk) => {
-        buffered += chunk.toString("utf8");
-        while (buffered.includes("\n")) {
-          const newline = buffered.indexOf("\n");
-          const line = buffered.slice(0, newline);
-          buffered = buffered.slice(newline + 1);
-          const envelope = JSON.parse(line);
-          requests.push(envelope);
-          assert.strictEqual(envelope.type, "coordinator_request");
-          assert.strictEqual(envelope.protocol_version, 1);
-          assert.strictEqual(envelope.authentication.kind, "none");
-          const request = envelope.payload;
-
-          if (requests.length === 1) {
-            assert.strictEqual(envelope.request_id, "cli-1");
-            assert.strictEqual(envelope.operation, "begin_oidc_browser_login");
-            assert.deepStrictEqual(request, {
-              type: "begin_oidc_browser_login",
-            });
-            socket.write(
-              `${JSON.stringify({
-                type: "oidc_browser_login_started",
-                transaction_id: "login-transaction",
-                polling_secret: "opaque-polling-secret",
-                authorization_url:
-                  "https://auth.michelpaulissen.com/application/o/authorize/?state=server-state&nonce=server-nonce&code_challenge=server-pkce&code_challenge_method=S256&redirect_uri=https%3A%2F%2Fclusterflux.michelpaulissen.com%2Fauth%2Fcallback",
-                expires_at_epoch_seconds: 1800000000,
-              })}\n`
-            );
-            continue;
-          }
-
-          assert.strictEqual(envelope.request_id, "cli-2");
-          assert.strictEqual(envelope.operation, "poll_oidc_browser_login");
-          assert.deepStrictEqual(request, {
-            type: "poll_oidc_browser_login",
-            transaction_id: "login-transaction",
-            polling_secret: "opaque-polling-secret",
-          });
-          socket.end(
-            `${JSON.stringify({
-              type: "oidc_browser_session",
-              session: {
-                tenant: "tenant-smoke",
-                project: "project-smoke",
-                user: "user-smoke",
-                cli_session_credential_kind: "CliDeviceSession",
-                cli_session_secret: "scoped-cli-session-secret",
-                expires_at_epoch_seconds: 1800000000,
-                provider_tokens_sent_to_nodes: false,
-              },
-            })}\n`
-          );
-          server.close();
-        }
-      });
-    });
-    server.once("error", reject);
-    server.listen(0, "127.0.0.1", () => {
-      const address = server.address();
-      resolve({
-        url: `${address.address}:${address.port}`,
-        requests,
-        close: () =>
-          new Promise((closeResolve) => {
-            if (!server.listening) closeResolve();
-            else server.close(() => closeResolve());
-          }),
-      });
-    });
-  });
-}
-
-function runClusterflux(args, env) {
-  return new Promise((resolve, reject) => {
-    const child = cp.spawn(
-    "cargo",
-    [
-      "run",
-      "-q",
-      "--manifest-path",
-      path.join(repo, "Cargo.toml"),
-      "-p",
-      "clusterflux-cli",
-      "--bin",
-      "clusterflux",
-      "--",
-      ...args,
-    ],
-      { cwd: project, env, stdio: ["ignore", "pipe", "pipe"] }
-    );
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (chunk) => (stdout += chunk.toString("utf8")));
-    child.stderr.on("data", (chunk) => (stderr += chunk.toString("utf8")));
-    child.once("error", reject);
-    child.once("close", (code) => {
-      if (code === 0) resolve(stdout);
-      else reject(new Error(`clusterflux exited ${code}\n${stderr}\n${stdout}`));
-    });
-  });
-}
-
-(async () => {
-  const opener = writeOpener();
-  const coordinator = await startCoordinator();
-  try {
-    const loginStarted = Date.now();
-    const report = JSON.parse(
-      await runClusterflux(
-        [
-          "login",
-          "--browser",
-          "--json",
-          "--coordinator",
-          coordinator.url,
-          "--project-id",
-          "project-smoke",
-        ],
-        {
-          ...process.env,
-          CLUSTERFLUX_BROWSER_OPEN_COMMAND: opener,
-          CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS: "5",
-        }
-      )
-    );
-    assert(
-      Date.now() - loginStarted < 3000,
-      "a long-lived browser opener must not keep CLI output pipes or login completion open"
-    );
-    assert.strictEqual(report.plan.coordinator, coordinator.url);
-    assert.strictEqual(report.boundary.cli_contacted_coordinator, true);
-    assert.strictEqual(report.boundary.scoped_cli_session_received, true);
-    assert.strictEqual(report.boundary.local_cli_session_file_written, true);
-    assert.strictEqual(report.boundary.provider_tokens_persisted_locally, false);
-    assert.strictEqual(report.boundary.provider_tokens_exposed_to_cli, false);
-    assert.strictEqual(report.boundary.provider_tokens_sent_to_nodes, false);
-    assert.strictEqual(report.boundary.coordinator_session_requests, 2);
-    assert.strictEqual(coordinator.requests.length, 2);
-
-    const sessionFile = path.join(project, ".clusterflux", "session.json");
-    const sessionText = fs.readFileSync(sessionFile, "utf8");
-    const session = JSON.parse(sessionText);
-    assert.strictEqual(session.kind, "human");
-    assert.strictEqual(session.coordinator, coordinator.url);
-    assert.strictEqual(session.tenant, "tenant-smoke");
-    assert.strictEqual(session.project, "project-smoke");
-    assert.strictEqual(session.user, "user-smoke");
-    assert.strictEqual(session.cli_session_credential_kind, "CliDeviceSession");
-    assert.strictEqual(session.token_expiry_posture, "expires_at");
-    assert.strictEqual(session.expires_at, "1800000000");
-    assert.strictEqual(session.provider_tokens_exposed_to_cli, false);
-    assert.strictEqual(session.provider_tokens_sent_to_nodes, false);
-    assert.doesNotMatch(
-      sessionText,
-      /access_token|refresh_token|id_token|provider-secret|authorization_code|Bearer/
-    );
-
-    const authStatus = JSON.parse(
-      await runClusterflux(["auth", "status", "--json"], process.env)
-    );
-    assert.strictEqual(authStatus.active_coordinator, coordinator.url);
-    assert.strictEqual(authStatus.principal, "user-smoke");
-    assert.strictEqual(authStatus.tenant, "tenant-smoke");
-    assert.strictEqual(authStatus.project, "project-smoke");
-    assert.strictEqual(authStatus.session.kind, "human");
-    assert.strictEqual(authStatus.session.source, "session_file");
-    assert.strictEqual(authStatus.session.provider_tokens_exposed_to_cli, false);
-    assert.strictEqual(authStatus.session.provider_tokens_exposed_to_nodes, false);
-  } finally {
-    await coordinator.close();
-  }
-  console.log("CLI browser login flow smoke passed");
-})().catch((error) => {
-  console.error(error.stack || error.message);
-  process.exit(1);
-});

scripts/cli-error-exit-smoke.js

@@ -1,365 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const fs = require("fs");
-const http = require("http");
-const os = require("os");
-const path = require("path");
-
-const repo = path.resolve(__dirname, "..");
-const project = path.join(repo, "tests/fixtures/runtime-conformance");
-const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-error-"));
-
-function runClusterflux(args) {
-  return new Promise((resolve) => {
-    const child = cp.spawn(
-      "cargo",
-      ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
-      {
-        cwd: repo,
-        stdio: ["ignore", "pipe", "pipe"],
-      }
-    );
-    let stdout = "";
-    let stderr = "";
-    child.stdout.setEncoding("utf8");
-    child.stderr.setEncoding("utf8");
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk;
-    });
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk;
-    });
-    child.on("close", (code, signal) => {
-      resolve({ code, signal, stdout, stderr });
-    });
-  });
-}
-
-async function runWithOneCoordinatorResponse(buildArgs, response) {
-  let request = "";
-  const server = http.createServer((incoming, outgoing) => {
-    incoming.setEncoding("utf8");
-    incoming.on("data", (chunk) => {
-      request += chunk;
-    });
-    incoming.on("end", () => {
-      outgoing.writeHead(200, { "content-type": "application/json" });
-      outgoing.end(JSON.stringify(response));
-      server.close();
-    });
-  });
-
-  const address = await new Promise((resolve) => {
-    server.listen(0, "127.0.0.1", () => resolve(server.address()));
-  });
-  const coordinator = `http://${address.address}:${address.port}`;
-  const result = await runClusterflux(buildArgs(coordinator));
-  return { request, result };
-}
-
-async function main() {
-  const environmentProject = path.join(tempRoot, "missing-env-project");
-  fs.mkdirSync(path.join(environmentProject, "src"), { recursive: true });
-  fs.writeFileSync(
-    path.join(environmentProject, "Cargo.toml"),
-    "[package]\nname = \"missing-env-project\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
-  );
-  fs.writeFileSync(
-    path.join(environmentProject, "src", "main.rs"),
-    "fn main() { let _target = env!(\"linux\"); }\n"
-  );
-  const environmentFailure = await runClusterflux([
-    "build",
-    "--project",
-    environmentProject,
-    "--json",
-  ]);
-  assert.strictEqual(environmentFailure.signal, null, environmentFailure.stderr);
-  assert.strictEqual(environmentFailure.code, 26, environmentFailure.stderr);
-  const environmentReport = JSON.parse(environmentFailure.stdout);
-  assert.strictEqual(environmentReport.status, "blocked_before_schedule");
-  assert.strictEqual(environmentReport.scheduled_work, false);
-  assert.strictEqual(environmentReport.machine_error.category, "environment");
-  assert.strictEqual(
-    environmentReport.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    environmentReport.machine_error.next_actions.includes("clusterflux inspect")
-  );
-  assert.strictEqual(environmentReport.diagnostics[0].code, "missing_environment");
-
-  const nonInteractive = await runClusterflux([
-    "run",
-    "build",
-    "--project",
-    project,
-    "--non-interactive",
-    "--json",
-  ]);
-  assert.strictEqual(nonInteractive.signal, null, nonInteractive.stderr);
-  assert.strictEqual(nonInteractive.code, 20, nonInteractive.stderr);
-  assert.doesNotMatch(nonInteractive.stderr, /Opening Clusterflux browser login/);
-  const nonInteractiveReport = JSON.parse(nonInteractive.stdout);
-  assert.strictEqual(nonInteractiveReport.status, "authentication_required");
-  assert.strictEqual(nonInteractiveReport.non_interactive, true);
-  assert.strictEqual(nonInteractiveReport.browser_opened, false);
-  assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication");
-  assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20);
-  assert.strictEqual(
-    nonInteractiveReport.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    nonInteractiveReport.machine_error.next_actions.includes(
-      "pass --local to run against local services"
-    )
-  );
-
-  let request = "";
-  const server = http.createServer((incoming, outgoing) => {
-    incoming.setEncoding("utf8");
-    incoming.on("data", (chunk) => {
-      request += chunk;
-    });
-    incoming.on("end", () => {
-      outgoing.writeHead(200, { "content-type": "application/json" });
-      outgoing.end(
-        JSON.stringify({
-          type: "error",
-          message: "quota unavailable: resource limit exceeded for api_calls",
-        })
-      );
-      server.close();
-    });
-  });
-
-  const address = await new Promise((resolve) => {
-    server.listen(0, "127.0.0.1", () => resolve(server.address()));
-  });
-  const coordinator = `http://${address.address}:${address.port}`;
-
-  const result = await runClusterflux([
-    "run",
-    "build",
-    "--project",
-    project,
-    "--coordinator",
-    coordinator,
-    "--json",
-  ]);
-
-  assert.strictEqual(result.signal, null, result.stderr);
-  assert.strictEqual(result.code, 22, result.stderr);
-  assert.match(request, /"type":"start_process"/);
-  const report = JSON.parse(result.stdout);
-  assert.strictEqual(report.status, "coordinator_rejected");
-  assert.strictEqual(report.run_start.machine_error.category, "quota");
-  assert.strictEqual(report.run_start.machine_error.resource_category, "api_calls");
-  assert.strictEqual(report.run_start.machine_error.community_tier_language, true);
-  assert.strictEqual(
-    report.run_start.machine_error.community_tier_label,
-    "community tier"
-  );
-  assert.doesNotMatch(result.stdout, new RegExp(["free", "tier"].join(" "), "i"));
-  assert.strictEqual(
-    report.run_start.machine_error.private_abuse_heuristics_exposed,
-    false
-  );
-  assert.strictEqual(report.run_start.machine_error.stable_exit_code, 22);
-  assert.strictEqual(
-    report.run_start.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    report.run_start.machine_error.next_actions.includes("clusterflux quota status")
-  );
-
-  const capabilityFailure = await runWithOneCoordinatorResponse(
-    (coordinator) => [
-      "run",
-      "build",
-      "--project",
-      project,
-      "--coordinator",
-      coordinator,
-      "--json",
-    ],
-    {
-      type: "error",
-      message:
-        "scheduler placement failed: no capable node for placement: missing capability Command",
-    }
-  );
-  assert.strictEqual(capabilityFailure.result.signal, null, capabilityFailure.result.stderr);
-  assert.strictEqual(capabilityFailure.result.code, 24, capabilityFailure.result.stderr);
-  assert.match(capabilityFailure.request, /"type":"start_process"/);
-  const capabilityReport = JSON.parse(capabilityFailure.result.stdout);
-  assert.strictEqual(
-    capabilityReport.run_start.machine_error.category,
-    "capability"
-  );
-  assert.strictEqual(
-    capabilityReport.run_start.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    capabilityReport.run_start.machine_error.next_actions.includes(
-      "attach a node with the required capabilities"
-    )
-  );
-
-  const nodePolicyFailure = await runWithOneCoordinatorResponse(
-    (coordinator) => [
-      "run",
-      "build",
-      "--project",
-      project,
-      "--coordinator",
-      coordinator,
-      "--json",
-    ],
-    {
-      type: "error",
-      message: "node policy denied native command execution",
-    }
-  );
-  assert.strictEqual(nodePolicyFailure.result.signal, null, nodePolicyFailure.result.stderr);
-  assert.strictEqual(nodePolicyFailure.result.code, 23, nodePolicyFailure.result.stderr);
-  assert.match(nodePolicyFailure.request, /"type":"start_process"/);
-  const nodePolicyReport = JSON.parse(nodePolicyFailure.result.stdout);
-  assert.strictEqual(
-    nodePolicyReport.run_start.machine_error.category,
-    "policy"
-  );
-  assert.strictEqual(
-    nodePolicyReport.run_start.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    nodePolicyReport.run_start.machine_error.next_actions.includes(
-      "check coordinator policy for this action"
-    )
-  );
-
-  const programFailure = await runWithOneCoordinatorResponse(
-    (coordinator) => [
-      "task",
-      "list",
-      "--coordinator",
-      coordinator,
-      "--json",
-    ],
-    {
-      type: "task_events",
-      events: [
-        {
-          process: "vp-current",
-          task: "compile",
-          terminal_state: "failed",
-          environment: "linux",
-          node: "node-linux",
-          status_code: 1,
-          stderr_tail: "task exited with status 1",
-        },
-      ],
-    }
-  );
-  assert.strictEqual(programFailure.result.signal, null, programFailure.result.stderr);
-  assert.strictEqual(programFailure.result.code, 0, programFailure.result.stderr);
-  assert.match(programFailure.request, /"type":"list_task_events"/);
-  const programReport = JSON.parse(programFailure.result.stdout);
-  assert.strictEqual(programReport.tasks[0].machine_error.category, "program");
-  assert.strictEqual(programReport.tasks[0].machine_error.stable_exit_code, 27);
-  assert(
-    programReport.tasks[0].machine_error.next_actions.includes("clusterflux logs")
-  );
-
-  const artifactDownload = await runWithOneCoordinatorResponse(
-    (coordinator) => [
-      "artifact",
-      "download",
-      "app.txt",
-      "--coordinator",
-      coordinator,
-      "--json",
-    ],
-    {
-      type: "artifact_download_denied",
-      message: "artifact download unauthorized for project",
-    }
-  );
-  assert.strictEqual(artifactDownload.result.signal, null, artifactDownload.result.stderr);
-  assert.strictEqual(artifactDownload.result.code, 21, artifactDownload.result.stderr);
-  assert.match(artifactDownload.request, /"type":"create_artifact_download_link"/);
-  const artifactDownloadReport = JSON.parse(artifactDownload.result.stdout);
-  assert.strictEqual(
-    artifactDownloadReport.download_session.machine_error.category,
-    "authorization"
-  );
-  assert.strictEqual(
-    artifactDownloadReport.download_session.machine_error.process_exit_code_applied,
-    true
-  );
-
-  const artifactExport = await runWithOneCoordinatorResponse(
-    (coordinator) => [
-      "artifact",
-      "export",
-      "app.txt",
-      "--to",
-      path.join(project, "target", "blocked-artifact.txt"),
-      "--coordinator",
-      coordinator,
-      "--json",
-    ],
-    {
-      type: "artifact_export_unavailable",
-      message: "direct connectivity unavailable for artifact export",
-    }
-  );
-  assert.strictEqual(artifactExport.result.signal, null, artifactExport.result.stderr);
-  assert.strictEqual(artifactExport.result.code, 25, artifactExport.result.stderr);
-  assert.match(artifactExport.request, /"type":"export_artifact_to_node"/);
-  const artifactExportReport = JSON.parse(artifactExport.result.stdout);
-  assert.strictEqual(
-    artifactExportReport.export_plan.machine_error.category,
-    "connectivity"
-  );
-  assert.strictEqual(
-    artifactExportReport.export_plan.machine_error.process_exit_code_applied,
-    true
-  );
-
-  const confirmation = await runClusterflux([
-    "process",
-    "cancel",
-    "--coordinator",
-    "127.0.0.1:9",
-    "--json",
-  ]);
-  assert.strictEqual(confirmation.signal, null, confirmation.stderr);
-  assert.strictEqual(confirmation.code, 23, confirmation.stderr);
-  const confirmationReport = JSON.parse(confirmation.stdout);
-  assert.strictEqual(confirmationReport.status, "confirmation_required");
-  assert.strictEqual(confirmationReport.coordinator_request_sent, false);
-  assert.strictEqual(confirmationReport.machine_error.category, "policy");
-  assert.strictEqual(
-    confirmationReport.machine_error.process_exit_code_applied,
-    true
-  );
-  assert(
-    confirmationReport.next_actions.some((action) => action.includes("--yes"))
-  );
-}
-
-main()
-  .then(() => {
-    console.log("CLI error exit smoke passed");
-  })
-  .catch((error) => {
-    console.error(error);
-    process.exit(1);
-  });

scripts/cli-install-smoke.js

@@ -1,61 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-
-const repo = path.resolve(__dirname, "..");
-const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-install-"));
-const installRoot = path.join(temp, "install");
-const targetDir =
-  process.env.CLUSTERFLUX_CLI_INSTALL_CARGO_TARGET_DIR ||
-  process.env.CARGO_TARGET_DIR ||
-  path.join(repo, "target");
-const project = path.join(repo, "tests/fixtures/runtime-conformance");
-const binName = process.platform === "win32" ? "clusterflux.exe" : "clusterflux";
-const installedBin = path.join(installRoot, "bin", binName);
-
-try {
-  cp.execFileSync(
-    "cargo",
-    [
-      "install",
-      "--path",
-      "crates/clusterflux-cli",
-      "--bin",
-      "clusterflux",
-      "--root",
-      installRoot,
-      "--debug"
-    ],
-    {
-      cwd: repo,
-      env: {
-        ...process.env,
-        CARGO_TARGET_DIR: targetDir
-      },
-      stdio: "inherit"
-    }
-  );
-
-  assert(fs.existsSync(installedBin), "installed clusterflux binary must exist");
-
-  const inspection = JSON.parse(
-    cp.execFileSync(
-      installedBin,
-      ["bundle", "inspect", "--project", project, "--json"],
-      { cwd: repo, encoding: "utf8" }
-    )
-  );
-
-  assert.strictEqual(inspection.project, project);
-  assert.strictEqual(inspection.metadata.embeds_full_container_images, false);
-  assert(inspection.metadata.environments.some((env) => env.name === "linux"));
-  assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/lib.rs"));
-} finally {
-  fs.rmSync(temp, { recursive: true, force: true });
-}
-
-console.log("CLI install smoke passed");

scripts/cli-local-run-smoke.js

@@ -1,269 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const net = require("net");
-const path = require("path");
-const { coordinatorWireRequest } = require("./coordinator-wire");
-const { configurePodmanTestEnvironment } = require("./podman-test-env");
-
-const repo = path.resolve(__dirname, "..");
-configurePodmanTestEnvironment(repo);
-if (
-  !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL &&
-  cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 &&
-  cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0
-) {
-  cp.execFileSync(
-    "nix",
-    ["shell", "nixpkgs#podman", "--command", "node", __filename],
-    {
-      cwd: repo,
-      env: {
-        ...process.env,
-        CLUSTERFLUX_PODMAN_NIX_SHELL: "1",
-      },
-      stdio: "inherit",
-    }
-  );
-  process.exit(0);
-}
-const project = path.join(repo, "tests/fixtures/runtime-conformance");
-
-cp.execFileSync(
-  "cargo",
-  ["build", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node"],
-  { cwd: repo, stdio: "inherit" }
-);
-
-function waitForJsonLine(child) {
-  return new Promise((resolve, reject) => {
-    let buffer = "";
-    child.stdout.on("data", (chunk) => {
-      buffer += chunk.toString();
-      const newline = buffer.indexOf("\n");
-      if (newline < 0) return;
-      try {
-        resolve(JSON.parse(buffer.slice(0, newline).trim()));
-      } catch (error) {
-        reject(error);
-      }
-    });
-    child.once("exit", (code) => {
-      reject(new Error(`process exited before JSON line with code ${code}`));
-    });
-  });
-}
-
-function send(addr, message) {
-  return new Promise((resolve, reject) => {
-    const socket = net.connect(addr.port, addr.host, () => {
-      socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`);
-    });
-    let buffer = "";
-    socket.on("data", (chunk) => {
-      buffer += chunk.toString();
-      const newline = buffer.indexOf("\n");
-      if (newline < 0) return;
-      socket.end();
-      try {
-        resolve(JSON.parse(buffer.slice(0, newline)));
-      } catch (error) {
-        reject(error);
-      }
-    });
-    socket.on("error", reject);
-  });
-}
-
-function runCli(args, env = {}) {
-  return new Promise((resolve, reject) => {
-    const child = cp.spawn(
-      "cargo",
-      ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
-      {
-        cwd: repo,
-        env: {
-          ...process.env,
-          ...env
-        }
-      }
-    );
-    const cliPid = child.pid;
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk.toString();
-    });
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk.toString();
-    });
-    child.on("exit", (code) => {
-      if (code !== 0) {
-        reject(new Error(`CLI run failed with code ${code}\n${stderr}`));
-        return;
-      }
-      try {
-        resolve({ pid: cliPid, report: JSON.parse(stdout) });
-      } catch (error) {
-        reject(new Error(`CLI output was not JSON: ${stdout}\n${error.stack || error.message}`));
-      }
-    });
-  });
-}
-
-(async () => {
-  const coordinator = cp.spawn(
-    "cargo",
-    [
-      "run",
-      "-q",
-      "-p",
-      "clusterflux-coordinator",
-      "--bin",
-      "clusterflux-coordinator",
-      "--",
-      "--listen",
-      "127.0.0.1:0",
-      "--allow-local-trusted-loopback"
-    ],
-    { cwd: repo }
-  );
-  assert(Number.isInteger(coordinator.pid));
-
-  try {
-    const ready = await waitForJsonLine(coordinator);
-    const [host, portText] = ready.listen.split(":");
-    const addr = { host, port: Number(portText) };
-    assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
-
-    const { pid: cliPid, report } = await runCli([
-      "run",
-      "--coordinator",
-      `${addr.host}:${addr.port}`,
-      "--project",
-      project,
-      "--json",
-    ]);
-    assert(Number.isInteger(cliPid));
-    assert.notStrictEqual(cliPid, coordinator.pid);
-    assert.strictEqual(report.plan.entry, "build");
-    assert.deepStrictEqual(report.plan.session, "Anonymous");
-    assert.strictEqual(report.boundary.cli_process_started_node_process, true);
-    assert.strictEqual(report.boundary.cli_process_started_coordinator_process, false);
-    assert(Number.isInteger(report.boundary.spawned_node_process_id));
-    assert.notStrictEqual(report.boundary.spawned_node_process_id, cliPid);
-    assert.notStrictEqual(report.boundary.spawned_node_process_id, coordinator.pid);
-    assert.strictEqual(report.boundary.node_session_requests, 0);
-    assert.strictEqual(report.node_report.node_status, "completed");
-    assert.strictEqual(report.node_report.execution_substrate, "wasm");
-    assert.strictEqual(report.node_report.task_spawn_host_import, true);
-    assert.strictEqual(
-      report.node_report.pre_node_process_status.processes.length,
-      1
-    );
-    assert.deepStrictEqual(
-      report.node_report.pre_node_process_status.processes[0].connected_nodes,
-      []
-    );
-    assert.strictEqual(
-      report.node_report.pre_node_process_status.processes[0].main_state,
-      "running"
-    );
-    assert.strictEqual(
-      report.node_report.pre_node_process_status.processes[0].main_wait_state,
-      "waiting_for_node",
-      "the coordinator must expose that the capless main is parked on placement before a node exists"
-    );
-    assert.strictEqual(
-      report.node_report.pre_node_process_status.processes[0].main_task_instance,
-      report.node_report.run.task_instance
-    );
-    assert.strictEqual(report.node_report.run.status, "main_launched");
-    assert.strictEqual(report.node_report.join.type, "task_joined");
-    const process = report.node_report.run.process;
-    assert.strictEqual(process, "vp-current");
-
-    const events = await send(addr, {
-      type: "list_task_events",
-      tenant: "tenant",
-      project: "project",
-      actor_user: "user",
-      process
-    });
-    assert.strictEqual(events.type, "task_events");
-    assert(events.events.length >= 4);
-    assert(
-      events.events
-        .filter((event) => event.executor === "node")
-        .every((event) => event.node === "node-cli-local")
-    );
-    assert(
-      events.events.some(
-        (event) =>
-          event.executor === "coordinator_main" &&
-          event.node === "coordinator-main"
-      )
-    );
-    assert(events.events.every((event) => event.process === process));
-    assert.deepStrictEqual(
-      new Set(events.events.map((event) => event.task_definition)),
-      new Set([
-        report.node_report.run.task_definition,
-        "prepare_source",
-        "compile_linux",
-        "package_release",
-      ])
-    );
-    assert.strictEqual(
-      new Set(events.events.map((event) => event.task)).size,
-      events.events.length,
-      "every live task event must retain its unique instance identity"
-    );
-    assert(
-      events.events.some(
-        (event) => event.task === report.node_report.run.task_instance
-      )
-    );
-    assert(events.events.some((event) => event.task.endsWith(":child:1")));
-    assert(events.events.some((event) => event.task.endsWith(":child:2")));
-    assert(events.events.some((event) => event.task.endsWith(":child:3")));
-    assert(events.events.some((event) => event.artifact_path));
-  } finally {
-    coordinator.kill("SIGTERM");
-  }
-
-  const { pid: autoCliPid, report: autoReport } = await runCli([
-    "run",
-    "--local",
-    "--project",
-    project,
-    "--json",
-  ]);
-  assert(Number.isInteger(autoCliPid));
-  assert.strictEqual(autoReport.plan.entry, "build");
-  assert.deepStrictEqual(autoReport.plan.coordinator, "LocalOnly");
-  assert.deepStrictEqual(autoReport.plan.session, "Anonymous");
-  assert.strictEqual(autoReport.boundary.cli_process_started_node_process, true);
-  assert.strictEqual(autoReport.boundary.cli_process_started_coordinator_process, true);
-  assert.match(autoReport.boundary.coordinator_address, /^127\.0\.0\.1:\d+$/);
-  assert(Number.isInteger(autoReport.boundary.coordinator_process_id));
-  assert(Number.isInteger(autoReport.boundary.spawned_node_process_id));
-  assert.notStrictEqual(autoReport.boundary.coordinator_process_id, autoCliPid);
-  assert.notStrictEqual(autoReport.boundary.spawned_node_process_id, autoCliPid);
-  assert.notStrictEqual(
-    autoReport.boundary.spawned_node_process_id,
-    autoReport.boundary.coordinator_process_id
-  );
-  assert.strictEqual(autoReport.boundary.node_session_requests, 0);
-  assert.strictEqual(autoReport.node_report.node_status, "completed");
-  assert.strictEqual(autoReport.node_report.execution_substrate, "wasm");
-  assert.strictEqual(autoReport.node_report.task_spawn_host_import, true);
-  assert.strictEqual(autoReport.node_report.run.status, "main_launched");
-  assert.strictEqual(autoReport.node_report.join.type, "task_joined");
-
-  console.log("CLI local run smoke passed");
-})().catch((error) => {
-  console.error(error.stack || error.message);
-  process.exit(1);
-});

scripts/cli-login-smoke.js

@@ -1,72 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const path = require("path");
-
-const repo = path.resolve(__dirname, "..");
-const coordinator = "https://coord.example.test";
-const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com";
-
-function clusterflux(args) {
-  return JSON.parse(
-    cp.execFileSync(
-      "cargo",
-      ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
-      { cwd: repo, encoding: "utf8" }
-    )
-  );
-}
-
-function clusterfluxRaw(args, env = {}) {
-  return cp.spawnSync(
-    "cargo",
-    ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
-    {
-      cwd: repo,
-      encoding: "utf8",
-      env: {
-        ...process.env,
-        ...env,
-      },
-    }
-  );
-}
-
-const browser = clusterflux(["login", "--plan", "--coordinator", coordinator, "--json"]);
-assert.strictEqual(browser.coordinator, coordinator);
-assert(browser.human_flow.Browser, "browser login should be available for human users");
-assert.strictEqual(browser.human_flow.Browser.authorization_url, null);
-assert.strictEqual(browser.human_flow.Browser.server_owns_state, true);
-assert.strictEqual(browser.human_flow.Browser.server_owns_nonce, true);
-assert.strictEqual(browser.human_flow.Browser.pkce_required, true);
-assert.strictEqual(browser.human_flow.Browser.hosted_callback, true);
-assert.strictEqual(browser.human_flow.Browser.cli_receives_provider_authorization_code, false);
-assert.strictEqual(browser.human_flow.Browser.cli_submits_identity_claims, false);
-
-const defaultBrowser = clusterflux(["login", "--plan", "--json"]);
-assert.strictEqual(defaultBrowser.coordinator, defaultHostedCoordinatorEndpoint);
-assert(defaultBrowser.human_flow.Browser);
-
-const nonInteractiveBrowser = clusterfluxRaw(
-  ["login", "--browser", "--non-interactive", "--coordinator", coordinator, "--json"],
-  {
-    CLUSTERFLUX_BROWSER_OPEN_COMMAND:
-      "node -e 'require(\"fs\").writeFileSync(\"/tmp/clusterflux-browser-should-not-open\", \"opened\")'",
-  }
-);
-assert.strictEqual(nonInteractiveBrowser.status, 20, nonInteractiveBrowser.stderr);
-assert.doesNotMatch(nonInteractiveBrowser.stderr, /Opening Clusterflux browser login/);
-const nonInteractiveReport = JSON.parse(nonInteractiveBrowser.stdout);
-assert.strictEqual(nonInteractiveReport.status, "authentication_required");
-assert.strictEqual(nonInteractiveReport.non_interactive, true);
-assert.strictEqual(nonInteractiveReport.browser_opened, false);
-assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication");
-assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20);
-assert(
-  nonInteractiveReport.machine_error.next_actions.includes(
-    "rerun without --non-interactive to open the browser"
-  )
-);
-
-console.log("CLI login smoke passed");

scripts/cli-output-mode-smoke.js

@@ -1,191 +0,0 @@
-#!/usr/bin/env node
-
-const assert = require("assert");
-const cp = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-
-const repo = path.resolve(__dirname, "..");
-const project = path.join(repo, "tests/fixtures/runtime-conformance");
-const isolatedCwd = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-output-"));
-const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-home-"));
-
-function clusterflux(args, env = {}, cwd = isolatedCwd) {
-  return cp.execFileSync(
-    "cargo",
-    [
-      "run",
-      "-q",
-      "--manifest-path",
-      path.join(repo, "Cargo.toml"),
-      "-p",
-      "clusterflux-cli",
-      "--bin",
-      "clusterflux",
-      "--",
-      ...args,
-    ],
-    {
-      cwd,
-      encoding: "utf8",
-      env: {
-        ...process.env,
-        HOME: isolatedHome,
-        USERPROFILE: isolatedHome,
-        XDG_CONFIG_HOME: path.join(isolatedHome, ".config"),
-        XDG_DATA_HOME: path.join(isolatedHome, ".local", "share"),
-        XDG_STATE_HOME: path.join(isolatedHome, ".local", "state"),
-        ...env,
-      },
-    }
-  );
-}
-
-function json(args, env, cwd) {
-  return JSON.parse(clusterflux(args, env, cwd));
-}
-
-function assertHuman(name, output, requiredPatterns) {
-  assert(
-    !output.trimStart().startsWith("{"),
-    `${name} default output should be human-readable text, not JSON`
-  );
-  for (const pattern of requiredPatterns) {
-    assert.match(output, pattern, `${name} human output missing ${pattern}`);
-  }
-}
-
-const helpHuman = clusterflux(["help"]);
-assertHuman("help", helpHuman, [
-  /Primary workflow:/,
-  /clusterflux login --browser/,
-  /clusterflux project init/,
-  /clusterflux node attach; clusterflux-node --worker/,
-  /Clusterflux: Launch Virtual Process/,
-  /Hosted account creation happens in the browser login flow/,
-  /--json/,
-]);
-
-const loginHuman = clusterflux([
-  "login",
-  "--plan",
-  "--coordinator",
-  "https://coord.example.test",
-]);
-assertHuman("login", loginHuman, [
-  /Clusterflux login/,
-  /flow: browser/,
-]);
-
-const loginJson = json([
-  "login",
-  "--plan",
-  "--coordinator",
-  "https://coord.example.test",
-  "--json",
-]);
-assert.strictEqual(loginJson.coordinator, "https://coord.example.test");
-assert(loginJson.human_flow.Browser);
-assert.strictEqual(loginJson.human_flow.Browser.hosted_callback, true);
-assert.strictEqual(loginJson.human_flow.Browser.cli_submits_identity_claims, false);
-
-const doctorHuman = clusterflux(["doctor"]);
-assertHuman("doctor", doctorHuman, [
-  /Clusterflux doctor/,
-  /coordinator reachability: not_configured/,
-  /dependencies:/,
-  /auth:/,
-  /node capabilities:/,
-  /node readiness: (ready_to_attach|local_dependencies_missing|limited_capabilities)/,
-  /node next:/,
-]);
-
-const doctorJson = json(["doctor", "--json"]);
-assert.strictEqual(doctorJson.coordinator_reachability.checked, false);
-assert.strictEqual(doctorJson.coordinator_reachability.status, "not_configured");
-assert(
-  ["ready_to_attach", "local_dependencies_missing", "limited_capabilities"].includes(
-    doctorJson.node_readiness_summary.status
-  )
-);
-assert.strictEqual(doctorJson.node_readiness_summary.explicit_attach_required, true);
-assert.strictEqual(
-  doctorJson.node_readiness_summary.command_execution_capability,
-  true
-);
-assert(Array.isArray(doctorJson.node_readiness_summary.missing_local_dependencies));
-assert(doctorJson.node_readiness_summary.next_actions.length >= 2);
-
-const authJson = json(["auth", "status", "--json"], {
-  CLUSTERFLUX_TOKEN: "token",
-  CLUSTERFLUX_TOKEN_EXPIRES_AT: "2026-07-04T00:00:00Z",
-}, isolatedCwd);
-assert.strictEqual(authJson.session.kind, "human");
-assert.strictEqual(authJson.session.token_expiry_posture, "expires_at");
-assert.strictEqual(authJson.session.expires_at, "2026-07-04T00:00:00Z");
-assert.strictEqual(authJson.coordinator_account_status.checked, false);
-assert.strictEqual(authJson.coordinator_account_status.account_status, "unknown");
-assert.strictEqual(
-  authJson.coordinator_account_status.private_moderation_details_exposed,
-  false
-);
-
-const inspectHuman = clusterflux(["bundle", "inspect", "--project", project]);
-assertHuman("bundle inspect", inspectHuman, [
-  /Clusterflux bundle inspect/,
-  /bundle: sha256:/,
-  /environments:/,
-]);
-
-const inspectJson = json(["bundle", "inspect", "--project", project, "--json"]);
-assert.strictEqual(inspectJson.project, project);
-assert.match(inspectJson.metadata.identity, /^sha256:/);
-assert.match(inspectJson.metadata.wasm_code, /^sha256:/);
-assert.strictEqual(inspectJson.metadata.task_metadata.default_entrypoint, "build");
-assert.deepStrictEqual(inspectJson.metadata.task_metadata.entrypoints, [
-  "build",
-  "fail",
-  "identity",
-  "long-join",
-  "park-wake",
-  "restart",
-]);
-assert.strictEqual(
-  inspectJson.metadata.source_metadata.transfer_policy.coordinator_receives_source_bytes_by_default,
-  false
-);
-assert.strictEqual(
-  inspectJson.metadata.source_metadata.transfer_policy.default_full_repo_tarball,
-  false
-);
-assert.strictEqual(inspectJson.metadata.debug_metadata.dap_virtual_process, true);
-assert.strictEqual(
-  inspectJson.metadata.large_input_policy.selected_inputs_are_content_digests,
-  true
-);
-assert.strictEqual(inspectJson.metadata.large_input_policy.selected_input_bytes_included, false);
-assert.strictEqual(inspectJson.metadata.large_input_policy.full_repository_bytes_included, false);
-assert.strictEqual(
-  inspectJson.metadata.large_input_policy.silent_task_argument_serialization,
-  false
-);
-assert(inspectJson.metadata.large_input_policy.supported_handle_types.includes("SourceSnapshot"));
-assert.strictEqual(
-  inspectJson.metadata.restart_compatibility.source_edits_can_restart_from_clean_task_boundary,
-  true
-);
-assert.strictEqual(
-  inspectJson.metadata.restart_compatibility.requires_clean_checkpoint_boundary,
-  true
-);
-assert.strictEqual(
-  inspectJson.metadata.restart_compatibility.compares_task_abi,
-  inspectJson.metadata.task_metadata.task_abi
-);
-assert.strictEqual(
-  inspectJson.metadata.restart_compatibility.incompatible_changes_require_whole_process_restart,
-  true
-);
-
-console.log("CLI output mode smoke passed");

scripts/coordinator-wire.js

@@ -1,43 +0,0 @@
-let requestId = 0;
-
-function coordinatorWireRequest(payload, prefix = "acceptance") {
-  if (payload && payload.type === "coordinator_request") return payload;
-  if (!payload || typeof payload.type !== "string" || !payload.type.trim()) {
-    throw new Error("coordinator payload must have a non-empty type");
-  }
-  requestId += 1;
-  return {
-    type: "coordinator_request",
-    protocol_version: 1,
-    request_id: `${prefix}-${process.pid}-${requestId}`,
-    operation: payload.type,
-    authentication: authenticationMetadata(payload),
-    payload,
-  };
-}
-
-function authenticationMetadata(payload) {
-  if (payload.type === "authenticated") {
-    return {
-      kind: "cli_session",
-      session: true,
-      request_operation: payload.request?.type || "unknown",
-    };
-  }
-  if (payload.type === "signed_node" || payload.node_signature) {
-    return { kind: "node_signature", node: payload.node || null };
-  }
-  if (payload.agent_signature) {
-    return {
-      kind: "agent_signature",
-      agent: payload.actor_agent || null,
-      fingerprint: payload.agent_public_key_fingerprint || null,
-    };
-  }
-  if (payload.admin_token) {
-    return { kind: "admin_credential" };
-  }
-  return { kind: "none" };
-}
-
-module.exports = { coordinatorWireRequest };