Public release release-f8ee5fc25a1d

Source commit: f8ee5fc25a1d0fed1e59b3f34d534b395bf34dad

Public tree identity: sha256:a428db3c520f7c0779135e36a841a5102ed26070519b696a146dbfa54514bd52

crates/clusterflux-cli/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "clusterflux-cli"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[[bin]]
+name = "clusterflux"
+path = "src/main.rs"
+
+[dependencies]
+anyhow.workspace = true
+base64.workspace = true
+clap.workspace = true
+clusterflux-core = { path = "../clusterflux-core" }
+clusterflux-control = { path = "../clusterflux-control" }
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+wasmparser.workspace = true

crates/clusterflux-cli/src/admin.rs

@@ -0,0 +1,236 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clusterflux_core::admin_request_proof;
+use serde_json::{json, Value};
+
+use crate::client::JsonLineSession;
+use crate::project::project_init_report;
+use crate::tools::{command_nonce, unix_timestamp_seconds};
+use crate::{
+    confirmation_required_report, AdminBootstrapArgs, AdminStatusArgs, AdminSuspendTenantArgs,
+    ProjectInitArgs,
+};
+
+pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
+    if let Some(coordinator) = &args.scope.coordinator {
+        let tenant = args.scope.tenant.clone();
+        let user = args.scope.user.clone();
+        let admin_token = admin_token_for_request(args.admin_token.as_deref())?;
+        let admin_nonce = command_nonce("admin-status");
+        let issued_at_epoch_seconds = unix_timestamp_seconds();
+        let admin_proof = admin_request_proof(
+            &admin_token,
+            "admin_status",
+            &tenant,
+            &user,
+            &tenant,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        );
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(json!({
+            "type": "admin_status",
+            "tenant": tenant,
+            "actor_user": user,
+            "admin_proof": admin_proof,
+            "admin_nonce": admin_nonce,
+            "issued_at_epoch_seconds": issued_at_epoch_seconds,
+        }))?;
+        return Ok(json!({
+            "command": "admin status",
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "user": user,
+            "suspended": response
+                .get("suspended")
+                .cloned()
+                .unwrap_or(json!(false)),
+            "response": response,
+            "safe_default": "read_only",
+            "private_website_required": false,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "admin status",
+        "mode": "self_hosted_local",
+        "safe_default": "read_only",
+        "private_website_required": false,
+    }))
+}
+
+pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> Result<Value> {
+    let scope = args.scope.clone();
+    let new_project = args.scope.project.clone();
+    let project_init = project_init_report(
+        ProjectInitArgs {
+            scope: args.scope,
+            new_project,
+            name: args.name,
+            yes: args.yes,
+        },
+        cwd,
+    )?;
+    let coordinator = scope
+        .coordinator
+        .clone()
+        .unwrap_or_else(|| "<local-coordinator>".to_owned());
+    let tenant = scope.tenant.clone();
+    let project = scope.project.clone();
+    let user = scope.user.clone();
+    Ok(json!({
+        "command": "admin bootstrap",
+        "mode": if scope.coordinator.is_some() { "public_coordinator_api" } else { "self_hosted_local" },
+        "tenant": tenant.clone(),
+        "project": project.clone(),
+        "user": user,
+        "coordinator": scope.coordinator,
+        "private_website_required": false,
+        "self_hosted_cli_only": true,
+        "project_config_written": project_init
+            .get("project_config_written")
+            .cloned()
+            .unwrap_or(json!(false)),
+        "project_init": project_init,
+        "admin_surfaces": {
+            "coordinator": "clusterflux-coordinator",
+            "project": "clusterflux project init/status/list/select",
+            "node": "clusterflux node enroll/list/status/revoke",
+            "process": "clusterflux run/process status/process restart/process cancel",
+            "logs": "clusterflux logs",
+            "artifacts": "clusterflux artifact list/download/export",
+            "quota": "clusterflux quota status",
+            "policy": "clusterflux admin status/suspend-tenant",
+        },
+        "bootstrap_sequence": [
+            {
+                "step": "start_self_hosted_coordinator",
+                "command": "clusterflux-coordinator --listen 127.0.0.1:0",
+                "private_website_required": false,
+            },
+            {
+                "step": "create_or_link_project",
+                "command": "clusterflux project init --yes",
+                "completed": true,
+                "private_website_required": false,
+            },
+            {
+                "step": "create_node_enrollment_grant",
+                "command": format!(
+                    "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
+                    tenant, project
+                ),
+                "private_website_required": false,
+            },
+            {
+                "step": "attach_worker_node",
+                "command": format!(
+                    "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
+                    tenant, project
+                ),
+                "private_website_required": false,
+            },
+            {
+                "step": "run_process",
+                "command": format!(
+                    "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
+                    tenant, project
+                ),
+                "private_website_required": false,
+            },
+            {
+                "step": "inspect_status_logs_artifacts",
+                "commands": [
+                    "clusterflux process status",
+                    "clusterflux task list",
+                    "clusterflux logs",
+                    "clusterflux artifact list",
+                    "clusterflux quota status",
+                ],
+                "private_website_required": false,
+            },
+            {
+                "step": "revoke_access",
+                "command": "clusterflux admin revoke-node --node <node-id> --yes",
+                "private_website_required": false,
+            }
+        ],
+    }))
+}
+
+pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Result<Value> {
+    let tenant = args
+        .target_tenant
+        .unwrap_or_else(|| args.scope.tenant.clone());
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "admin suspend-tenant",
+            "suspend_tenant",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "actor_tenant": args.scope.tenant,
+                "actor_user": args.scope.user,
+                "target_tenant": tenant,
+            }),
+            "clusterflux admin suspend-tenant --yes".to_owned(),
+        ));
+    }
+    if let Some(coordinator) = &args.scope.coordinator {
+        let actor_tenant = args.scope.tenant.clone();
+        let actor_user = args.scope.user.clone();
+        let admin_token = admin_token_for_request(args.admin_token.as_deref())?;
+        let admin_nonce = command_nonce("admin-suspend-tenant");
+        let issued_at_epoch_seconds = unix_timestamp_seconds();
+        let admin_proof = admin_request_proof(
+            &admin_token,
+            "suspend_tenant",
+            &actor_tenant,
+            &actor_user,
+            &tenant,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        );
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(json!({
+            "type": "suspend_tenant",
+            "tenant": actor_tenant,
+            "actor_user": actor_user,
+            "target_tenant": tenant,
+            "admin_proof": admin_proof,
+            "admin_nonce": admin_nonce,
+            "issued_at_epoch_seconds": issued_at_epoch_seconds,
+        }))?;
+        return Ok(json!({
+            "command": "admin suspend-tenant",
+            "coordinator": coordinator,
+            "requires_confirmation": !args.yes,
+            "tenant": tenant,
+            "actor_tenant": actor_tenant,
+            "actor_user": actor_user,
+            "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
+            "private_website_required": false,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "admin suspend-tenant",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "tenant": tenant,
+        "private_website_required": false,
+    }))
+}
+
+fn admin_token_for_request(explicit: Option<&str>) -> Result<String> {
+    explicit
+        .map(str::to_owned)
+        .or_else(|| std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok())
+        .filter(|token| !token.trim().is_empty())
+        .ok_or_else(|| {
+            anyhow::anyhow!(
+                "admin command requires --admin-token or CLUSTERFLUX_ADMIN_TOKEN for coordinator requests"
+            )
+        })
+}

crates/clusterflux-cli/src/agent.rs

@@ -0,0 +1,17 @@
+use clusterflux_core::Digest;
+use serde::Serialize;
+
+use crate::AgentEnrollArgs;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct AgentEnrollmentPlan {
+    pub(crate) public_key_fingerprint: Digest,
+    pub(crate) browser_interaction_required_each_run: bool,
+}
+
+pub(crate) fn agent_enrollment_plan(args: AgentEnrollArgs) -> AgentEnrollmentPlan {
+    AgentEnrollmentPlan {
+        public_key_fingerprint: Digest::sha256(args.public_key),
+        browser_interaction_required_each_run: false,
+    }
+}

crates/clusterflux-cli/src/artifact.rs

@@ -0,0 +1,547 @@
+use anyhow::{Context, Result};
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use serde_json::{json, Value};
+use sha2::{Digest as _, Sha256};
+use std::io::Write;
+use std::path::Path;
+use std::time::{Duration, Instant};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
+    JsonLineSession,
+};
+use crate::config::StoredCliSession;
+use crate::errors::cli_error_summary_for_category;
+use crate::process_events::{
+    artifact_download_grant_disclosures, artifact_download_session_summary,
+    artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
+};
+use crate::{ArtifactDownloadArgs, ArtifactExportArgs, ArtifactListArgs};
+
+pub(crate) const DEFAULT_ARTIFACT_EXPORT_MAX_BYTES: u64 = 64 * 1024 * 1024;
+const ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES: u64 = 256 * 1024;
+
+#[cfg(test)]
+pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
+    artifact_list_report_with_session(args, None)
+}
+
+pub(crate) fn artifact_list_report_with_session(
+    args: ArtifactListArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let events = list_task_events_if_available_with_session(
+        args.scope.coordinator.as_deref(),
+        &args.scope,
+        args.process.clone(),
+        stored_session,
+    )?;
+    let artifacts = artifact_summaries(events.as_ref());
+    Ok(json!({
+        "command": "artifact list",
+        "process": args.process,
+        "source": "task_events",
+        "artifacts": artifacts,
+        "default_durable_store_assumed": false,
+        "events": events,
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Value> {
+    artifact_download_report_with_session(args, None)
+}
+
+pub(crate) fn artifact_download_report_with_session(
+    args: ArtifactDownloadArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    if let Some(coordinator) = &args.scope.coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "create_artifact_download_link",
+                "artifact": args.artifact,
+                "max_bytes": args.max_bytes,
+            }),
+            json!({
+                "type": "create_artifact_download_link",
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "actor_user": args.scope.user,
+                "artifact": args.artifact,
+                "max_bytes": args.max_bytes,
+            }),
+        )?)?;
+        let download_session = artifact_download_session_summary(&response);
+        let grant_disclosures = artifact_download_grant_disclosures(&response);
+        let local_download = match &args.to {
+            Some(path)
+                if response.get("type").and_then(Value::as_str)
+                    == Some("artifact_download_link") =>
+            {
+                download_link_to_path(
+                    &mut session,
+                    coordinator,
+                    &args.scope,
+                    &args.artifact,
+                    args.max_bytes,
+                    path,
+                    &response,
+                    stored_session,
+                )?
+            }
+            Some(path) => json!({
+                "status": "download_link_failed",
+                "local_path": path,
+                "local_bytes_written_by_cli": false,
+                "machine_error": artifact_response_machine_error(
+                    &response,
+                    "coordinator rejected artifact download",
+                    "connectivity"
+                ),
+            }),
+            None => Value::Null,
+        };
+        return Ok(json!({
+            "command": "artifact download",
+            "coordinator": coordinator,
+            "artifact": args.artifact,
+            "max_bytes": args.max_bytes,
+            "to": args.to,
+            "download_session": download_session,
+            "local_download": local_download,
+            "grant_disclosures": grant_disclosures,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "artifact download",
+        "status": "requires_coordinator",
+        "artifact": args.artifact,
+        "max_bytes": args.max_bytes,
+        "to": args.to,
+        "download_session": {
+            "status": "requires_coordinator",
+            "link_issued": false,
+            "explicit_user_action_required": true,
+            "machine_error": cli_error_summary_for_category(
+                "connectivity",
+                "artifact download requires a coordinator"
+            ),
+        },
+        "grant_disclosures": [],
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
+    artifact_export_report_with_session(args, None)
+}
+
+pub(crate) fn artifact_export_report_with_session(
+    args: ArtifactExportArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    if let Some(coordinator) = &args.scope.coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "export_artifact_to_node",
+                "artifact": args.artifact,
+                "receiver_node": args.receiver_node,
+                "direct_connectivity": true,
+                "failure_reason": "",
+            }),
+            json!({
+                "type": "export_artifact_to_node",
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "actor_user": args.scope.user,
+                "artifact": args.artifact,
+                "receiver_node": args.receiver_node,
+                "direct_connectivity": true,
+                "failure_reason": "",
+            }),
+        )?)?;
+        let mut export_plan = artifact_export_plan_summary(&response, &args.to);
+        let local_export = if response.get("type").and_then(Value::as_str)
+            == Some("artifact_export_plan")
+        {
+            artifact_export_local_write_followup(&mut session, coordinator, &args, stored_session)?
+        } else {
+            json!({
+                "status": "skipped",
+                "explicit_user_action": true,
+                "local_path": &args.to,
+                "local_bytes_written_by_cli": false,
+                "machine_error": artifact_response_machine_error(
+                    &response,
+                    "coordinator rejected artifact export",
+                    "connectivity"
+                ),
+            })
+        };
+        apply_local_export_summary(&mut export_plan, &local_export);
+        let grant_disclosures = local_export
+            .get("grant_disclosures")
+            .cloned()
+            .unwrap_or_else(|| json!([]));
+        return Ok(json!({
+            "command": "artifact export",
+            "coordinator": coordinator,
+            "artifact": args.artifact,
+            "to": args.to,
+            "receiver_node": args.receiver_node,
+            "export_plan": export_plan,
+            "local_export": local_export,
+            "grant_disclosures": grant_disclosures,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "artifact export",
+        "status": "requires_coordinator",
+        "artifact": args.artifact,
+        "to": args.to,
+        "receiver_node": args.receiver_node,
+        "export_plan": {
+            "status": "requires_coordinator",
+            "explicit_user_action": true,
+            "local_bytes_written_by_cli": false,
+            "default_durable_store_assumed": false,
+            "machine_error": cli_error_summary_for_category(
+                "connectivity",
+                "artifact export requires a coordinator"
+            ),
+        },
+        "grant_disclosures": [],
+    }))
+}
+
+fn artifact_export_local_write_followup(
+    session: &mut JsonLineSession,
+    coordinator: &str,
+    args: &ArtifactExportArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let link_response = session.request(authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({
+            "type": "create_artifact_download_link",
+            "artifact": args.artifact,
+            "max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
+        }),
+        json!({
+            "type": "create_artifact_download_link",
+            "tenant": args.scope.tenant,
+            "project": args.scope.project,
+            "actor_user": args.scope.user,
+            "artifact": args.artifact,
+            "max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
+        }),
+    )?)?;
+    if link_response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
+        return Ok(json!({
+            "status": "download_link_failed",
+            "explicit_user_action": true,
+            "local_path": &args.to,
+            "local_bytes_written_by_cli": false,
+            "content_bytes_available": false,
+            "download_session": artifact_download_session_summary(&link_response),
+            "grant_disclosures": [],
+            "machine_error": artifact_response_machine_error(
+                &link_response,
+                "coordinator rejected artifact download for local export",
+                "connectivity"
+            ),
+            "link_response": link_response,
+        }));
+    }
+    download_link_to_path(
+        session,
+        coordinator,
+        &args.scope,
+        &args.artifact,
+        DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
+        &args.to,
+        &link_response,
+        stored_session,
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn download_link_to_path(
+    session: &mut JsonLineSession,
+    coordinator: &str,
+    scope: &crate::CliScopeArgs,
+    artifact: &str,
+    max_bytes: u64,
+    destination: &Path,
+    link_response: &Value,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let download_session = artifact_download_session_summary(link_response);
+    let grant_disclosures = artifact_download_grant_disclosures(link_response);
+    let token_digest = link_response
+        .pointer("/link/scoped_token_digest")
+        .cloned()
+        .context("artifact download link did not include a scoped token digest")?;
+    let expected_digest: clusterflux_core::Digest = serde_json::from_value(
+        link_response
+            .pointer("/link/artifact_digest")
+            .cloned()
+            .context("artifact download link omitted the published digest")?,
+    )?;
+    let expected_size = link_response
+        .pointer("/link/artifact_size_bytes")
+        .and_then(Value::as_u64)
+        .context("artifact download link omitted the published size")?;
+    if expected_size > max_bytes {
+        return Ok(json!({
+            "status": "download_limit_failed",
+            "local_path": destination,
+            "local_bytes_written_by_cli": false,
+            "content_bytes_available": false,
+            "download_session": download_session,
+            "grant_disclosures": grant_disclosures,
+            "machine_error": cli_error_summary_for_category(
+                "quota",
+                &format!("artifact size {expected_size} exceeds requested limit {max_bytes}")
+            ),
+        }));
+    }
+
+    let parent = destination
+        .parent()
+        .filter(|parent| !parent.as_os_str().is_empty())
+        .unwrap_or_else(|| Path::new("."));
+    std::fs::create_dir_all(parent)
+        .with_context(|| format!("failed to create {}", parent.display()))?;
+    let mut temporary = tempfile::Builder::new()
+        .prefix(".clusterflux-download-")
+        .tempfile_in(parent)
+        .with_context(|| {
+            format!(
+                "failed to create a bounded download in {}",
+                parent.display()
+            )
+        })?;
+    let started = Instant::now();
+    let mut received_bytes = 0_u64;
+    let mut hasher = Sha256::new();
+    let stream_response = loop {
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "open_artifact_download_stream",
+                "artifact": artifact,
+                "max_bytes": max_bytes,
+                "token_digest": token_digest,
+                "chunk_bytes": expected_size.clamp(1, ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES),
+            }),
+            json!({
+                "type": "open_artifact_download_stream",
+                "tenant": scope.tenant,
+                "project": scope.project,
+                "actor_user": scope.user,
+                "artifact": artifact,
+                "max_bytes": max_bytes,
+                "token_digest": token_digest,
+                "chunk_bytes": expected_size.clamp(1, ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES),
+            }),
+        )?)?;
+        if response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
+            break response;
+        }
+        if response
+            .get("content_bytes_available")
+            .and_then(Value::as_bool)
+            == Some(true)
+        {
+            let offset = response
+                .get("content_offset")
+                .and_then(Value::as_u64)
+                .unwrap_or(u64::MAX);
+            if offset != received_bytes {
+                break json!({
+                    "type": "error",
+                    "message": format!(
+                        "artifact reverse stream returned offset {offset}, expected {received_bytes}"
+                    ),
+                });
+            }
+            let Some(content_base64) = response.get("content_base64").and_then(Value::as_str)
+            else {
+                break json!({
+                    "type": "error",
+                    "message": "artifact reverse stream marked bytes available without content",
+                });
+            };
+            let content = BASE64_STANDARD
+                .decode(content_base64)
+                .context("artifact download stream returned invalid base64 content")?;
+            received_bytes = received_bytes
+                .checked_add(content.len() as u64)
+                .context("artifact download byte count overflowed")?;
+            if received_bytes > expected_size || received_bytes > max_bytes {
+                break json!({
+                    "type": "error",
+                    "message": "artifact reverse stream exceeded its published size or CLI limit",
+                });
+            }
+            temporary
+                .write_all(&content)
+                .context("write bounded artifact download")?;
+            hasher.update(&content);
+            if response.get("content_eof").and_then(Value::as_bool) == Some(true) {
+                break response;
+            }
+            continue;
+        }
+        if started.elapsed() >= Duration::from_secs(120) {
+            break json!({
+                "type": "error",
+                "message": "timed out waiting for the retaining node to reverse-stream artifact bytes",
+            });
+        }
+        std::thread::sleep(Duration::from_millis(25));
+    };
+    if stream_response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
+        return Ok(json!({
+            "status": "download_stream_failed",
+            "explicit_user_action": true,
+            "local_path": destination,
+            "local_bytes_written_by_cli": false,
+            "content_bytes_available": false,
+            "download_session": download_session,
+            "grant_disclosures": grant_disclosures,
+            "machine_error": artifact_response_machine_error(
+                &stream_response,
+                "coordinator rejected artifact download stream",
+                "connectivity"
+            ),
+            "stream": artifact_stream_summary(&stream_response),
+        }));
+    }
+    let actual_digest_hex = format!("{:x}", hasher.finalize());
+    let actual_digest = clusterflux_core::Digest::from_sha256_hex(&actual_digest_hex)
+        .map_err(anyhow::Error::msg)?;
+    if received_bytes != expected_size || actual_digest != expected_digest {
+        return Ok(json!({
+            "status": "download_integrity_failed",
+            "explicit_user_action": true,
+            "local_path": destination,
+            "local_bytes_written_by_cli": false,
+            "content_bytes_available": false,
+            "download_session": download_session,
+            "grant_disclosures": grant_disclosures,
+            "machine_error": cli_error_summary_for_category(
+                "program",
+                &format!(
+                    "artifact download digest/size mismatch: received {received_bytes} bytes with {actual_digest}, expected {expected_size} bytes with {expected_digest}"
+                )
+            ),
+            "stream": artifact_stream_summary(&stream_response),
+        }));
+    }
+    temporary
+        .as_file()
+        .sync_all()
+        .context("sync verified artifact download")?;
+    temporary.persist_noclobber(destination).map_err(|error| {
+        anyhow::anyhow!(
+            "refusing to replace artifact download destination {}: {}",
+            destination.display(),
+            error.error
+        )
+    })?;
+
+    Ok(json!({
+        "status": "local_bytes_written",
+        "explicit_user_action": true,
+        "local_path": destination,
+        "local_bytes_written_by_cli": true,
+        "bytes_written": received_bytes,
+        "verified_digest": actual_digest,
+        "published_digest": expected_digest,
+        "content_bytes_available": true,
+        "content_source": stream_response.get("content_source").cloned().unwrap_or(Value::Null),
+        "download_session": download_session,
+        "grant_disclosures": grant_disclosures,
+        "stream": artifact_stream_summary(&stream_response),
+    }))
+}
+
+pub(crate) fn artifact_stream_summary(response: &Value) -> Value {
+    let status = response
+        .get("type")
+        .and_then(Value::as_str)
+        .unwrap_or("coordinator_response");
+    let mut summary = json!({
+        "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+        "streamed_bytes": response.get("streamed_bytes").cloned().unwrap_or_else(|| json!(0)),
+        "charged_download_bytes": response.get("charged_download_bytes").cloned().unwrap_or_else(|| json!(0)),
+        "content_bytes_available": response.get("content_bytes_available").cloned().unwrap_or(json!(false)),
+        "content_offset": response.get("content_offset").cloned().unwrap_or(Value::Null),
+        "content_eof": response.get("content_eof").cloned().unwrap_or(json!(false)),
+        "content_source": response.get("content_source").cloned().unwrap_or(Value::Null),
+        "content_material_returned_in_report": false,
+        "error": response.get("message").cloned().unwrap_or(Value::Null),
+    });
+    if status != "artifact_download_stream" {
+        if let Some(object) = summary.as_object_mut() {
+            object.insert(
+                "machine_error".to_owned(),
+                artifact_response_machine_error(
+                    response,
+                    "coordinator rejected artifact download stream",
+                    "connectivity",
+                ),
+            );
+        }
+    }
+    summary
+}
+
+fn apply_local_export_summary(export_plan: &mut Value, local_export: &Value) {
+    let Some(object) = export_plan.as_object_mut() else {
+        return;
+    };
+    object.insert(
+        "local_bytes_written_by_cli".to_owned(),
+        local_export
+            .get("local_bytes_written_by_cli")
+            .cloned()
+            .unwrap_or(json!(false)),
+    );
+    object.insert(
+        "local_export_status".to_owned(),
+        local_export
+            .get("status")
+            .cloned()
+            .unwrap_or_else(|| json!("unknown")),
+    );
+    object.insert(
+        "content_bytes_available".to_owned(),
+        local_export
+            .get("content_bytes_available")
+            .cloned()
+            .unwrap_or(json!(false)),
+    );
+    if let Some(bytes_written) = local_export.get("bytes_written") {
+        object.insert("bytes_written".to_owned(), bytes_written.clone());
+        object.insert(
+            "writes_require_data_plane_followup".to_owned(),
+            json!(false),
+        );
+    }
+}

crates/clusterflux-cli/src/auth.rs

@@ -0,0 +1,776 @@
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use anyhow::{Context, Result};
+use serde::Serialize;
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, stored_session_for_coordinator, JsonLineSession,
+};
+use crate::config::{
+    default_hosted_coordinator_endpoint, effective_scope_value, read_cli_session,
+    read_project_config, write_cli_session, write_project_config, ProjectConfig, StoredCliSession,
+};
+use crate::errors::cli_error_summary;
+use crate::run::{session_from_env, CliSession};
+use crate::ConnectSelfHostedArgs;
+use crate::{AuthStatusArgs, LoginArgs};
+
+const DEFAULT_BROWSER_LOGIN_TRANSACTION_TIMEOUT_SECONDS: u64 = 300;
+
+pub(crate) fn read_session_secret_from_stdin() -> Result<String> {
+    let mut secret = String::new();
+    std::io::stdin()
+        .read_line(&mut secret)
+        .context("failed to read self-hosted session secret from stdin")?;
+    let secret = secret.trim_end_matches(['\r', '\n']).to_owned();
+    if secret.trim().is_empty() {
+        anyhow::bail!("self-hosted session secret from stdin must not be empty");
+    }
+    Ok(secret)
+}
+
+pub(crate) fn connect_self_hosted_report(
+    args: ConnectSelfHostedArgs,
+    cwd: PathBuf,
+    session_secret: String,
+) -> Result<Value> {
+    if !args.session_secret_stdin {
+        anyhow::bail!("self-hosted session configuration requires --session-secret-stdin");
+    }
+    let coordinator = args
+        .scope
+        .coordinator
+        .as_deref()
+        .filter(|value| !value.trim().is_empty())
+        .context("self-hosted session configuration requires --coordinator <host:port>")?;
+    if session_secret.trim().is_empty() {
+        anyhow::bail!("self-hosted session secret from stdin must not be empty");
+    }
+
+    let mut connection = JsonLineSession::connect(coordinator)?;
+    let response = connection.request(json!({
+        "type": "authenticated",
+        "session_secret": session_secret,
+        "request": { "type": "auth_status" },
+    }))?;
+    for (field, expected) in [
+        ("tenant", args.scope.tenant.as_str()),
+        ("project", args.scope.project.as_str()),
+        ("actor", args.scope.user.as_str()),
+    ] {
+        let actual = response.get(field).and_then(Value::as_str).unwrap_or("");
+        if actual != expected {
+            anyhow::bail!(
+                "self-hosted session {field} mismatch: coordinator returned {actual:?}, expected {expected:?}"
+            );
+        }
+    }
+    if response.get("authenticated").and_then(Value::as_bool) != Some(true) {
+        anyhow::bail!("self-hosted coordinator did not confirm the CLI session");
+    }
+
+    let stored = StoredCliSession {
+        kind: "self_hosted".to_owned(),
+        coordinator: coordinator.to_owned(),
+        tenant: args.scope.tenant,
+        project: args.scope.project,
+        user: args.scope.user,
+        cli_session_credential_kind: "CliDeviceSession".to_owned(),
+        session_secret: Some(session_secret),
+        token_expiry_posture: "configured_by_self_hosted_operator".to_owned(),
+        expires_at: None,
+        provider_tokens_exposed_to_cli: false,
+        provider_tokens_sent_to_nodes: false,
+        created_at_unix_seconds: unix_timestamp_seconds(),
+    };
+    let session_file = write_cli_session(&cwd, &stored)?;
+    Ok(json!({
+        "command": "auth connect-self-hosted",
+        "status": "connected",
+        "coordinator": coordinator,
+        "tenant": stored.tenant,
+        "project": stored.project,
+        "user": stored.user,
+        "session_file": session_file,
+        "session_secret_read_from_stdin": true,
+        "session_secret_exposed_in_report": false,
+        "provider_tokens_exposed_to_cli": false,
+        "provider_tokens_sent_to_nodes": false,
+        "coordinator_response": response,
+        "coordinator_session_requests": connection.requests(),
+    }))
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct LoginPlan {
+    pub(crate) coordinator: String,
+    pub(crate) human_flow: LoginFlowPlan,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub(crate) struct LoginCompletionReport {
+    pub(crate) plan: LoginPlan,
+    pub(crate) boundary: LoginCompletionBoundaryEvidence,
+    pub(crate) coordinator_response: Value,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct LoginCompletionBoundaryEvidence {
+    pub(crate) cli_contacted_coordinator: bool,
+    pub(crate) coordinator_address: String,
+    pub(crate) scoped_cli_session_received: bool,
+    pub(crate) local_cli_session_file_written: bool,
+    pub(crate) provider_tokens_persisted_locally: bool,
+    pub(crate) provider_tokens_exposed_to_cli: bool,
+    pub(crate) provider_tokens_sent_to_nodes: bool,
+    pub(crate) coordinator_session_requests: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) enum LoginFlowPlan {
+    Browser(HostedBrowserLoginPlan),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct HostedBrowserLoginPlan {
+    pub(crate) authorization_url: Option<String>,
+    pub(crate) server_owns_state: bool,
+    pub(crate) server_owns_nonce: bool,
+    pub(crate) pkce_required: bool,
+    pub(crate) hosted_callback: bool,
+    pub(crate) cli_receives_provider_authorization_code: bool,
+    pub(crate) cli_submits_identity_claims: bool,
+}
+
+pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<Value> {
+    let config = read_project_config(&cwd)?;
+    let stored_session = read_cli_session(&cwd)?;
+    let configured_coordinator = args
+        .scope
+        .coordinator
+        .clone()
+        .or_else(|| {
+            config
+                .as_ref()
+                .and_then(|config| config.coordinator.clone())
+        })
+        .or_else(|| {
+            stored_session
+                .as_ref()
+                .map(|session| session.coordinator.clone())
+        });
+    let active_coordinator = configured_coordinator
+        .clone()
+        .unwrap_or_else(default_hosted_coordinator_endpoint);
+    let tenant = effective_scope_value(
+        &args.scope.tenant,
+        config
+            .as_ref()
+            .map(|config| config.tenant.as_str())
+            .or_else(|| {
+                stored_session
+                    .as_ref()
+                    .map(|session| session.tenant.as_str())
+            }),
+        "tenant",
+    );
+    let project = effective_scope_value(
+        &args.scope.project,
+        config
+            .as_ref()
+            .map(|config| config.project.as_str())
+            .or_else(|| {
+                stored_session
+                    .as_ref()
+                    .map(|session| session.project.as_str())
+            }),
+        "project",
+    );
+    let principal = effective_scope_value(
+        &args.scope.user,
+        config
+            .as_ref()
+            .map(|config| config.user.as_str())
+            .or_else(|| stored_session.as_ref().map(|session| session.user.as_str())),
+        "user",
+    );
+    let session_scope_mismatch = crate::auth_scope::session_scope_mismatch(
+        stored_session.as_ref(),
+        &active_coordinator,
+        &tenant,
+        &project,
+        &principal,
+    );
+    let session_matches_project = stored_session.is_some() && session_scope_mismatch.is_none();
+    let coordinator_account_status = configured_coordinator
+        .as_ref()
+        .filter(|_| session_scope_mismatch.is_none())
+        .map(|coordinator| {
+            coordinator_auth_status_summary(
+                coordinator,
+                &tenant,
+                &project,
+                &principal,
+                stored_session.as_ref(),
+            )
+        })
+        .unwrap_or_else(|| {
+            if let Some(fields) = &session_scope_mismatch {
+                return crate::auth_scope::session_scope_mismatch_status(fields);
+            }
+            json!({
+                "checked": false,
+                "reason": "no project or session coordinator configured",
+                "suspension_known": false,
+                "account_state_known": false,
+                "account_status": "unknown",
+                "private_moderation_details_exposed": false,
+                "signup_failure_details_exposed": false,
+            })
+        });
+    Ok(json!({
+        "command": "auth status",
+        "active_coordinator": active_coordinator,
+        "principal": principal,
+        "tenant": tenant,
+        "project": project,
+        "session": auth_state_value(&cwd)?,
+        "session_matches_current_project": session_matches_project,
+        "session_scope_mismatch": session_scope_mismatch,
+        "coordinator_account_status": coordinator_account_status,
+        "project_config": config,
+    }))
+}
+
+fn coordinator_auth_status_summary(
+    coordinator: &str,
+    tenant: &str,
+    project: &str,
+    principal: &str,
+    stored_session: Option<&StoredCliSession>,
+) -> Value {
+    let mut session = match JsonLineSession::connect(coordinator) {
+        Ok(session) => session,
+        Err(error) => {
+            let message = error.to_string();
+            return json!({
+                "checked": true,
+                "reachable": false,
+                "source": "public_coordinator_api",
+                "account_status": "unknown",
+                "suspension_known": false,
+                "account_state_known": false,
+                "private_moderation_details_exposed": false,
+                "signup_failure_details_exposed": false,
+                "machine_error": cli_error_summary(&message),
+                "error": message,
+                "next_actions": ["clusterflux doctor", "check coordinator status"],
+                "coordinator_session_requests": 0,
+            });
+        }
+    };
+    let request = match authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({
+            "type": "auth_status",
+        }),
+        json!({
+            "type": "auth_status",
+            "tenant": tenant,
+            "project": project,
+            "actor_user": principal,
+        }),
+    ) {
+        Ok(request) => request,
+        Err(error) => {
+            let message = error.to_string();
+            return json!({
+                "checked": false,
+                "reachable": "not_checked",
+                "source": "public_coordinator_api",
+                "authenticated_for_current_project": false,
+                "account_status": "unknown",
+                "suspension_known": false,
+                "account_state_known": false,
+                "private_moderation_details_exposed": false,
+                "signup_failure_details_exposed": false,
+                "machine_error": cli_error_summary(&message),
+                "error": message,
+                "next_actions": ["clusterflux login --browser"],
+                "coordinator_session_requests": 0,
+            });
+        }
+    };
+    let response = match session.request_allow_error(request) {
+        Ok(response) => response,
+        Err(error) => {
+            let message = error.to_string();
+            return json!({
+                "checked": true,
+                "reachable": false,
+                "source": "public_coordinator_api",
+                "account_status": "unknown",
+                "suspension_known": false,
+                "account_state_known": false,
+                "private_moderation_details_exposed": false,
+                "signup_failure_details_exposed": false,
+                "machine_error": cli_error_summary(&message),
+                "error": message,
+                "next_actions": ["clusterflux doctor", "check coordinator status"],
+                "coordinator_session_requests": session.requests(),
+            });
+        }
+    };
+    let coordinator_session_requests = session.requests();
+    if response.get("type").and_then(Value::as_str) == Some("error") {
+        let message = response
+            .get("message")
+            .and_then(Value::as_str)
+            .unwrap_or("coordinator rejected auth status");
+        return json!({
+            "checked": true,
+            "reachable": true,
+            "source": "public_coordinator_api",
+            "account_status": "unknown",
+            "suspension_known": false,
+            "account_state_known": false,
+            "private_moderation_details_exposed": false,
+            "signup_failure_details_exposed": false,
+            "machine_error": cli_error_summary(message),
+            "coordinator_response_type": "error",
+            "next_actions": ["clusterflux doctor", "clusterflux login --browser"],
+            "coordinator_session_requests": coordinator_session_requests,
+        });
+    }
+    let suspended = response
+        .get("suspended")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let disabled = response
+        .get("disabled")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let deleted = response
+        .get("deleted")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let manual_review = response
+        .get("manual_review")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let account_status = response
+        .get("account_status")
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .unwrap_or_else(|| {
+            if deleted {
+                "deleted"
+            } else if disabled {
+                "disabled"
+            } else if suspended {
+                "suspended"
+            } else if manual_review {
+                "manual_review"
+            } else {
+                "active"
+            }
+            .to_owned()
+        });
+    let sanitized_reason = response.get("sanitized_reason").and_then(Value::as_str);
+    let next_actions = response
+        .get("next_actions")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default()
+        .into_iter()
+        .filter_map(|value| value.as_str().map(str::to_owned))
+        .collect::<Vec<_>>();
+    json!({
+        "checked": true,
+        "reachable": true,
+        "source": "public_coordinator_api",
+        "used_cli_session_credential": stored_session_for_coordinator(coordinator, stored_session).is_some(),
+        "account_status": account_status,
+        "suspension_known": true,
+        "account_state_known": true,
+        "suspended": suspended,
+        "disabled": disabled,
+        "deleted": deleted,
+        "manual_review": manual_review,
+        "sanitized_reason": sanitized_reason,
+        "next_actions": next_actions,
+        "private_moderation_details_exposed": false,
+        "signup_failure_details_exposed": false,
+        "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
+        "coordinator_session_requests": coordinator_session_requests,
+    })
+}
+
+pub(crate) fn non_interactive_browser_login_report(args: &LoginArgs) -> Value {
+    let message =
+        "browser login requires an interactive browser, but non-interactive mode is enabled";
+    let next_actions = vec![
+        "rerun without --non-interactive to open the browser",
+        "clusterflux login --browser --plan",
+        "use CLUSTERFLUX_AGENT_PRIVATE_KEY for automation",
+    ];
+    json!({
+        "command": "login",
+        "status": "authentication_required",
+        "coordinator": args.coordinator,
+        "non_interactive": true,
+        "browser_requested": true,
+        "browser_opened": false,
+        "safe_failure": true,
+        "message": message,
+        "next_actions": next_actions,
+        "machine_error": crate::auth_scope::non_interactive_auth_machine_error(message, next_actions),
+    })
+}
+
+pub(crate) fn auth_state_value(cwd: &Path) -> Result<Value> {
+    match session_from_env()? {
+        CliSession::Anonymous => {
+            if let Some(session) = read_cli_session(cwd)? {
+                return Ok(json!({
+                    "kind": session.kind,
+                    "authenticated": true,
+                    "source": "session_file",
+                    "coordinator": session.coordinator,
+                    "tenant": session.tenant,
+                    "project": session.project,
+                    "principal": session.user,
+                    "cli_session_credential_kind": session.cli_session_credential_kind,
+                    "provider_tokens_exposed_to_cli": session.provider_tokens_exposed_to_cli,
+                    "provider_tokens_exposed_to_nodes": session.provider_tokens_sent_to_nodes,
+                    "expires_at": session.expires_at,
+                    "token_expiry_posture": session.token_expiry_posture,
+                }));
+            }
+            Ok(json!({
+                "kind": "anonymous",
+                "authenticated": false,
+                "source": "environment",
+                "token_expiry_posture": "no_session",
+            }))
+        }
+        CliSession::HumanSession => {
+            let expires_at = std::env::var("CLUSTERFLUX_TOKEN_EXPIRES_AT").ok();
+            Ok(json!({
+                "kind": "human",
+                "authenticated": true,
+                "source": "CLUSTERFLUX_TOKEN",
+                "provider_tokens_exposed_to_nodes": false,
+                "expires_at": expires_at,
+                "token_expiry_posture": if expires_at.is_some() { "expires_at" } else { "unknown_env_token" },
+            }))
+        }
+        CliSession::AgentPublicKey {
+            agent,
+            public_key_fingerprint,
+            browser_interaction_required,
+            ..
+        } => Ok(json!({
+            "kind": "agent_public_key",
+            "authenticated": true,
+            "agent": agent,
+            "source": "CLUSTERFLUX_AGENT_PRIVATE_KEY",
+            "public_key_fingerprint": public_key_fingerprint,
+            "browser_interaction_required": browser_interaction_required,
+            "token_expiry_posture": "not_applicable_public_key",
+        })),
+    }
+}
+
+pub(crate) fn login_plan(args: LoginArgs) -> LoginPlan {
+    let human_flow = LoginFlowPlan::Browser(HostedBrowserLoginPlan {
+        authorization_url: None,
+        server_owns_state: true,
+        server_owns_nonce: true,
+        pkce_required: true,
+        hosted_callback: true,
+        cli_receives_provider_authorization_code: false,
+        cli_submits_identity_claims: false,
+    });
+
+    LoginPlan {
+        coordinator: args.coordinator,
+        human_flow,
+    }
+}
+
+fn finalize_browser_login(
+    args: LoginArgs,
+    plan: LoginPlan,
+    coordinator_response: Value,
+    coordinator_session_requests: u64,
+) -> Result<LoginCompletionReport> {
+    let coordinator = args.coordinator.clone();
+    let scoped_cli_session_received = coordinator_response
+        .pointer("/session/cli_session_credential_kind")
+        .and_then(Value::as_str)
+        == Some("CliDeviceSession");
+    let provider_tokens_sent_to_nodes = coordinator_response
+        .pointer("/session/provider_tokens_sent_to_nodes")
+        .and_then(Value::as_bool)
+        .unwrap_or(true);
+    let provider_tokens_exposed_to_cli = contains_provider_token_field(&coordinator_response);
+    let local_cli_session_file_written = if scoped_cli_session_received {
+        let cwd = std::env::current_dir()?;
+        let stored_session = stored_cli_session_from_login_response(
+            &coordinator,
+            &coordinator_response,
+            provider_tokens_exposed_to_cli,
+            provider_tokens_sent_to_nodes,
+        )?;
+        write_cli_session(&cwd, &stored_session)?;
+        write_project_config(
+            &cwd,
+            &ProjectConfig {
+                tenant: stored_session.tenant.clone(),
+                project: stored_session.project.clone(),
+                user: stored_session.user.clone(),
+                coordinator: Some(stored_session.coordinator.clone()),
+            },
+        )?;
+        true
+    } else {
+        false
+    };
+
+    Ok(LoginCompletionReport {
+        plan,
+        boundary: LoginCompletionBoundaryEvidence {
+            cli_contacted_coordinator: true,
+            coordinator_address: coordinator,
+            scoped_cli_session_received,
+            local_cli_session_file_written,
+            provider_tokens_persisted_locally: false,
+            provider_tokens_exposed_to_cli,
+            provider_tokens_sent_to_nodes,
+            coordinator_session_requests,
+        },
+        coordinator_response,
+    })
+}
+
+pub(crate) fn stored_cli_session_from_login_response(
+    coordinator: &str,
+    coordinator_response: &Value,
+    provider_tokens_exposed_to_cli: bool,
+    provider_tokens_sent_to_nodes: bool,
+) -> Result<StoredCliSession> {
+    let session = coordinator_response.get("session").unwrap_or(&Value::Null);
+    let expires_at = session
+        .get("expires_at")
+        .or_else(|| session.get("token_expires_at"))
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .or_else(|| {
+            session
+                .get("expires_at_epoch_seconds")
+                .and_then(Value::as_u64)
+                .map(|value| value.to_string())
+        });
+    let tenant = session
+        .get("tenant")
+        .and_then(Value::as_str)
+        .context("hosted login session omitted tenant")?;
+    let project = session
+        .get("project")
+        .and_then(Value::as_str)
+        .context("hosted login session omitted project")?;
+    let user = session
+        .get("user")
+        .and_then(Value::as_str)
+        .context("hosted login session omitted user")?;
+    let session_secret = session
+        .get("cli_session_secret")
+        .or_else(|| session.get("session_secret"))
+        .and_then(Value::as_str)
+        .context("hosted login session omitted CLI session secret")?;
+    Ok(StoredCliSession {
+        kind: "human".to_owned(),
+        coordinator: coordinator.to_owned(),
+        tenant: tenant.to_owned(),
+        project: project.to_owned(),
+        user: user.to_owned(),
+        cli_session_credential_kind: session
+            .get("cli_session_credential_kind")
+            .and_then(Value::as_str)
+            .unwrap_or("CliDeviceSession")
+            .to_owned(),
+        session_secret: Some(session_secret.to_owned()),
+        token_expiry_posture: if expires_at.is_some() {
+            "expires_at".to_owned()
+        } else {
+            "unknown_coordinator_session".to_owned()
+        },
+        expires_at,
+        provider_tokens_exposed_to_cli,
+        provider_tokens_sent_to_nodes,
+        created_at_unix_seconds: unix_timestamp_seconds(),
+    })
+}
+
+pub(crate) fn execute_interactive_browser_login(args: LoginArgs) -> Result<LoginCompletionReport> {
+    let coordinator = args.coordinator.clone();
+    let mut session = JsonLineSession::connect_browser_login(&coordinator)?;
+    let started = session.request(json!({
+        "type": "begin_oidc_browser_login",
+    }))?;
+    if started.get("type").and_then(Value::as_str) != Some("oidc_browser_login_started") {
+        anyhow::bail!("coordinator did not start a hosted browser login transaction");
+    }
+    let transaction_id = started
+        .get("transaction_id")
+        .and_then(Value::as_str)
+        .context("hosted login transaction omitted transaction id")?
+        .to_owned();
+    let polling_secret = started
+        .get("polling_secret")
+        .and_then(Value::as_str)
+        .context("hosted login transaction omitted polling secret")?
+        .to_owned();
+    let authorization_url = started
+        .get("authorization_url")
+        .and_then(Value::as_str)
+        .context("hosted login transaction omitted authorization URL")?
+        .to_owned();
+    let loopback_test_flow = coordinator.starts_with("http://")
+        && crate::client::is_loopback_coordinator(&coordinator)
+        && authorization_url.starts_with("http://")
+        && crate::client::is_loopback_coordinator(&authorization_url);
+    if !authorization_url.starts_with("https://") && !loopback_test_flow {
+        anyhow::bail!(
+            "hosted login authorization URL must use HTTPS (plain HTTP is accepted only when both coordinator and identity provider are loopback test services)"
+        );
+    }
+    let plan = LoginPlan {
+        coordinator: coordinator.clone(),
+        human_flow: LoginFlowPlan::Browser(HostedBrowserLoginPlan {
+            authorization_url: Some(authorization_url.clone()),
+            server_owns_state: true,
+            server_owns_nonce: true,
+            pkce_required: true,
+            hosted_callback: true,
+            cli_receives_provider_authorization_code: false,
+            cli_submits_identity_claims: false,
+        }),
+    };
+
+    eprintln!("Opening Clusterflux browser login: {authorization_url}");
+    eprintln!("Waiting for the hosted login callback to complete.");
+    open_browser(&authorization_url)?;
+
+    let deadline = Instant::now() + browser_login_timeout();
+    loop {
+        let response = session.request(json!({
+            "type": "poll_oidc_browser_login",
+            "transaction_id": transaction_id,
+            "polling_secret": polling_secret,
+        }))?;
+        match response.get("type").and_then(Value::as_str) {
+            Some("oidc_browser_login_pending") => {
+                if Instant::now() >= deadline {
+                    anyhow::bail!("timed out waiting for hosted browser login completion");
+                }
+                std::thread::sleep(Duration::from_millis(500));
+            }
+            Some("oidc_browser_session") => {
+                return finalize_browser_login(args, plan, response, session.requests());
+            }
+            _ => anyhow::bail!("coordinator returned an invalid hosted login status"),
+        }
+    }
+}
+
+pub(crate) fn print_browser_login_success(report: &LoginCompletionReport) {
+    let session = report.coordinator_response.get("session");
+    let tenant = session
+        .and_then(|value| value.get("tenant"))
+        .and_then(Value::as_str)
+        .unwrap_or("tenant");
+    let project = session
+        .and_then(|value| value.get("project"))
+        .and_then(Value::as_str)
+        .unwrap_or("project");
+    let user = session
+        .and_then(|value| value.get("user"))
+        .and_then(Value::as_str)
+        .unwrap_or("user");
+    println!(
+        "Signed in to {} as {user} for {tenant}/{project}.",
+        report.plan.coordinator
+    );
+    if report.boundary.scoped_cli_session_received {
+        println!("Received a scoped CLI session from the coordinator.");
+    }
+}
+
+fn open_browser(url: &str) -> Result<()> {
+    let mut command = if let Some(command) = std::env::var_os("CLUSTERFLUX_BROWSER_OPEN_COMMAND") {
+        Command::new(command)
+    } else {
+        platform_browser_command()
+    };
+    command
+        .arg(url)
+        .stdin(Stdio::null())
+        .stdout(Stdio::null())
+        .stderr(Stdio::null());
+    command
+        .spawn()
+        .with_context(|| format!("failed to start browser opener for {url}"))?;
+    Ok(())
+}
+
+#[cfg(target_os = "macos")]
+fn platform_browser_command() -> Command {
+    Command::new("open")
+}
+
+#[cfg(target_os = "windows")]
+fn platform_browser_command() -> Command {
+    let mut command = Command::new("cmd");
+    command.args(["/C", "start", ""]);
+    command
+}
+
+#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
+fn platform_browser_command() -> Command {
+    Command::new("xdg-open")
+}
+
+fn browser_login_timeout() -> Duration {
+    let seconds = std::env::var("CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS")
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        .filter(|seconds| *seconds > 0)
+        .unwrap_or(DEFAULT_BROWSER_LOGIN_TRANSACTION_TIMEOUT_SECONDS);
+    Duration::from_secs(seconds)
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+pub(crate) fn contains_provider_token_field(value: &Value) -> bool {
+    match value {
+        Value::Object(object) => object.iter().any(|(key, value)| {
+            matches!(
+                key.as_str(),
+                "access_token" | "refresh_token" | "id_token" | "provider_token" | "oauth_token"
+            ) || contains_provider_token_field(value)
+        }),
+        Value::Array(items) => items.iter().any(contains_provider_token_field),
+        _ => false,
+    }
+}

crates/clusterflux-cli/src/auth_scope.rs

@@ -0,0 +1,79 @@
+use std::path::Path;
+
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::control_endpoint_identity;
+use crate::config::{
+    effective_scope_value, read_project_config, StoredCliSession,
+    DEFAULT_HOSTED_COORDINATOR_ENDPOINT,
+};
+use crate::errors::cli_error_summary_for_category;
+use crate::LoginArgs;
+
+pub(crate) fn login_args_for_project(mut args: LoginArgs, cwd: &Path) -> Result<LoginArgs> {
+    let Some(config) = read_project_config(cwd)? else {
+        return Ok(args);
+    };
+    args.project = effective_scope_value(&args.project, Some(&config.project), "project");
+    if args.coordinator == DEFAULT_HOSTED_COORDINATOR_ENDPOINT {
+        if let Some(coordinator) = config.coordinator {
+            args.coordinator = coordinator;
+        }
+    }
+    Ok(args)
+}
+
+pub(crate) fn session_scope_mismatch(
+    session: Option<&StoredCliSession>,
+    active_coordinator: &str,
+    tenant: &str,
+    project: &str,
+    user: &str,
+) -> Option<Vec<&'static str>> {
+    session.and_then(|session| {
+        let mut fields = Vec::new();
+        if control_endpoint_identity(&session.coordinator).ok()
+            != control_endpoint_identity(active_coordinator).ok()
+        {
+            fields.push("coordinator");
+        }
+        if session.tenant != tenant {
+            fields.push("tenant");
+        }
+        if session.project != project {
+            fields.push("project");
+        }
+        if session.user != user {
+            fields.push("user");
+        }
+        (!fields.is_empty()).then_some(fields)
+    })
+}
+
+pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
+    json!({
+        "checked": false,
+        "reason": "stored CLI session does not match the current project",
+        "mismatched_fields": fields,
+        "authenticated_for_current_project": false,
+        "account_status": "unknown",
+        "suspension_known": false,
+        "account_state_known": false,
+        "private_moderation_details_exposed": false,
+        "signup_failure_details_exposed": false,
+        "next_actions": ["clusterflux login --browser"],
+    })
+}
+
+pub(crate) fn non_interactive_auth_machine_error(
+    message: &str,
+    next_actions: Vec<&'static str>,
+) -> Value {
+    let mut machine_error = cli_error_summary_for_category("authentication", message);
+    if let Some(object) = machine_error.as_object_mut() {
+        object.insert("next_actions".to_owned(), json!(next_actions));
+        object.insert("browser_opened".to_owned(), json!(false));
+    }
+    machine_error
+}

crates/clusterflux-cli/src/build.rs

@@ -0,0 +1,322 @@
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use anyhow::{bail, Context, Result};
+use clusterflux_core::Digest;
+use serde_json::{json, Value};
+use wasmparser::{Parser, Payload};
+
+use crate::errors::cli_error_summary_for_category;
+use crate::{bundle_inspection, BuildArgs, BundleInspectArgs};
+
+pub(crate) fn build_report(args: BuildArgs, cwd: PathBuf) -> Result<Value> {
+    let inspection = bundle_inspection(
+        BundleInspectArgs {
+            project: args.project.clone(),
+            source_provider: args.source_provider.clone(),
+            disabled_source_providers: args.disabled_source_providers.clone(),
+            json: true,
+        },
+        cwd,
+    )?;
+    let diagnostics = inspection.pre_schedule_diagnostics.clone();
+    let blocking_diagnostic = diagnostics
+        .iter()
+        .find(|diagnostic| diagnostic.severity == "error")
+        .cloned();
+    if let Some(diagnostic) = blocking_diagnostic {
+        let machine_category = match diagnostic.category.as_str() {
+            "environment" => "environment",
+            "source_provider" | "capability" => "capability",
+            _ => "unknown",
+        };
+        return Ok(json!({
+            "command": "build",
+            "status": "blocked_before_schedule",
+            "bundle": inspection,
+            "diagnostics": diagnostics,
+            "contains_full_repository_upload": false,
+            "content_addressed": false,
+            "debug_metadata_available": false,
+            "scheduled_work": false,
+            "machine_error": cli_error_summary_for_category(machine_category, &diagnostic.message),
+        }));
+    }
+
+    let mut wasm = compile_project_wasm(&inspection.project)?;
+    let environment_manifest = serde_json::to_vec(&inspection.metadata.environments)?;
+    append_custom_section(
+        &mut wasm.bytes,
+        "clusterflux.environments",
+        &environment_manifest,
+    );
+    let bundle_digest = Digest::sha256(&wasm.bytes);
+    let task_descriptors = descriptor_records(&wasm.bytes, "clusterflux.tasks")?;
+    let entrypoint_descriptors = descriptor_records(&wasm.bytes, "clusterflux.entrypoints")?;
+    if task_descriptors.is_empty() {
+        bail!(
+            "compiled Wasm module contains no #[clusterflux::task] descriptors; annotate at least one exported task"
+        );
+    }
+    if entrypoint_descriptors.is_empty() {
+        bail!(
+            "compiled Wasm module contains no #[clusterflux::main] descriptors; annotate at least one entrypoint"
+        );
+    }
+    let output = args.output.unwrap_or_else(|| {
+        inspection.project.join(".clusterflux/build").join(
+            bundle_digest
+                .as_str()
+                .trim_start_matches("sha256:")
+                .get(..16)
+                .unwrap_or("bundle"),
+        )
+    });
+    let bundle_artifact = write_bundle(
+        &output,
+        &wasm,
+        &bundle_digest,
+        &inspection,
+        &task_descriptors,
+        &entrypoint_descriptors,
+    )?;
+
+    Ok(json!({
+        "command": "build",
+        "status": "built",
+        "bundle": inspection,
+        "bundle_artifact": bundle_artifact,
+        "diagnostics": diagnostics,
+        "contains_full_repository_upload": false,
+        "content_addressed": true,
+        "debug_metadata_available": true,
+        "scheduled_work": false,
+    }))
+}
+
+fn append_custom_section(module: &mut Vec<u8>, name: &str, data: &[u8]) {
+    let mut section = Vec::new();
+    encode_unsigned_leb(name.len() as u64, &mut section);
+    section.extend_from_slice(name.as_bytes());
+    section.extend_from_slice(data);
+    module.push(0);
+    encode_unsigned_leb(section.len() as u64, module);
+    module.extend_from_slice(&section);
+}
+
+fn encode_unsigned_leb(mut value: u64, output: &mut Vec<u8>) {
+    loop {
+        let mut byte = (value & 0x7f) as u8;
+        value >>= 7;
+        if value != 0 {
+            byte |= 0x80;
+        }
+        output.push(byte);
+        if value == 0 {
+            break;
+        }
+    }
+}
+
+struct CompiledWasm {
+    bytes: Vec<u8>,
+    package: String,
+    target: String,
+    source_path: PathBuf,
+}
+
+fn compile_project_wasm(project: &Path) -> Result<CompiledWasm> {
+    let manifest = project.join("Cargo.toml");
+    let metadata_output = Command::new("cargo")
+        .args([
+            "metadata",
+            "--format-version",
+            "1",
+            "--no-deps",
+            "--manifest-path",
+        ])
+        .arg(&manifest)
+        .output()
+        .with_context(|| format!("failed to run cargo metadata for {}", manifest.display()))?;
+    if !metadata_output.status.success() {
+        bail!(
+            "cargo metadata failed for {}: {}",
+            manifest.display(),
+            String::from_utf8_lossy(&metadata_output.stderr).trim()
+        );
+    }
+    let metadata: Value = serde_json::from_slice(&metadata_output.stdout)?;
+    let canonical_manifest = std::fs::canonicalize(&manifest)?;
+    let package = metadata["packages"]
+        .as_array()
+        .and_then(|packages| {
+            packages.iter().find(|package| {
+                package["manifest_path"]
+                    .as_str()
+                    .and_then(|path| std::fs::canonicalize(path).ok())
+                    .as_ref()
+                    == Some(&canonical_manifest)
+            })
+        })
+        .context("cargo metadata did not return the requested project package")?;
+    let package_name = package["name"]
+        .as_str()
+        .context("cargo package name missing")?;
+    let target = package["targets"]
+        .as_array()
+        .and_then(|targets| {
+            targets.iter().find(|target| {
+                target["crate_types"]
+                    .as_array()
+                    .is_some_and(|types| types.iter().any(|kind| kind == "cdylib"))
+            })
+        })
+        .context("Clusterflux project library must include crate-type = [\"cdylib\"]")?;
+    let target_name = target["name"]
+        .as_str()
+        .context("cargo target name missing")?;
+    let build = Command::new("cargo")
+        // The MVP transports one Wasm bundle in a bounded control frame. These are
+        // ordinary Cargo release-profile settings, applied only to the guest build,
+        // that keep the product SDK/runtime inside that accepted boundary.
+        .env("CARGO_PROFILE_RELEASE_OPT_LEVEL", "z")
+        .env("CARGO_PROFILE_RELEASE_LTO", "thin")
+        .env("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1")
+        .args([
+            "build",
+            "--quiet",
+            "--release",
+            "--target",
+            "wasm32-unknown-unknown",
+            "--lib",
+            "--manifest-path",
+        ])
+        .arg(&manifest)
+        .output()
+        .with_context(|| format!("failed to compile {} to Wasm", manifest.display()))?;
+    if !build.status.success() {
+        bail!(
+            "Wasm bundle compilation failed for {}: {}",
+            manifest.display(),
+            String::from_utf8_lossy(&build.stderr).trim()
+        );
+    }
+    let target_directory = metadata["target_directory"]
+        .as_str()
+        .context("cargo target_directory missing")?;
+    let source_path = Path::new(target_directory)
+        .join("wasm32-unknown-unknown/release")
+        .join(format!("{}.wasm", target_name.replace('-', "_")));
+    let bytes = std::fs::read(&source_path)
+        .with_context(|| format!("compiled Wasm module missing at {}", source_path.display()))?;
+    Ok(CompiledWasm {
+        bytes,
+        package: package_name.to_owned(),
+        target: target_name.to_owned(),
+        source_path,
+    })
+}
+
+fn descriptor_records(module: &[u8], section_name: &str) -> Result<Vec<Value>> {
+    let mut records: Vec<Value> = Vec::new();
+    for payload in Parser::new(0).parse_all(module) {
+        let Payload::CustomSection(section) = payload? else {
+            continue;
+        };
+        if section.name() != section_name {
+            continue;
+        }
+        for record in section
+            .data()
+            .split(|byte| *byte == b'\n' || *byte == 0)
+            .filter(|record| !record.is_empty())
+        {
+            records.push(serde_json::from_slice(record).with_context(|| {
+                format!("invalid descriptor record in Wasm custom section {section_name}")
+            })?);
+        }
+    }
+    records.sort_by(|left, right| left["name"].as_str().cmp(&right["name"].as_str()));
+    Ok(records)
+}
+
+fn write_bundle(
+    output: &Path,
+    wasm: &CompiledWasm,
+    bundle_digest: &Digest,
+    inspection: &crate::bundle::BundleInspection,
+    tasks: &[Value],
+    entrypoints: &[Value],
+) -> Result<Value> {
+    std::fs::create_dir_all(output)?;
+    let module_path = output.join("module.wasm");
+    let task_path = output.join("task-descriptors.json");
+    let entrypoint_path = output.join("entrypoints.json");
+    let environment_path = output.join("environments.json");
+    let source_path = output.join("source-provider.json");
+    let vfs_path = output.join("vfs-seed.json");
+    let debug_path = output.join("debug-metadata.json");
+    let manifest_path = output.join("manifest.json");
+    std::fs::write(&module_path, &wasm.bytes)?;
+    write_json(&task_path, &json!(tasks))?;
+    write_json(&entrypoint_path, &json!(entrypoints))?;
+    write_json(&environment_path, &json!(inspection.metadata.environments))?;
+    write_json(&source_path, &json!(inspection.source_provider_manifest))?;
+    write_json(
+        &vfs_path,
+        &json!({
+            "epoch": 0,
+            "mounts": ["/vfs/artifacts", "/vfs/sources", "/vfs/blobs"],
+            "large_bytes_embedded": false,
+        }),
+    )?;
+    write_json(&debug_path, &json!(inspection.metadata.debug_metadata))?;
+    let manifest = json!({
+        "kind": "clusterflux-bundle",
+        "format_version": 1,
+        "package": wasm.package,
+        "target": wasm.target,
+        "bundle_digest": bundle_digest,
+        "module": "module.wasm",
+        "module_size_bytes": wasm.bytes.len(),
+        "task_descriptors": "task-descriptors.json",
+        "entrypoints": "entrypoints.json",
+        "environments": "environments.json",
+        "source_provider": "source-provider.json",
+        "vfs_seed": "vfs-seed.json",
+        "debug_metadata": "debug-metadata.json",
+        "required_capabilities": tasks.iter().flat_map(|task| {
+            task["required_capabilities"].as_array().into_iter().flatten().cloned()
+        }).collect::<Vec<_>>(),
+        "metadata_identity": inspection.metadata.identity,
+        "coordinator_receives_source_bytes_by_default": false,
+        "embeds_full_repository": false,
+    });
+    write_json(&manifest_path, &manifest)?;
+    Ok(json!({
+        "directory": output,
+        "manifest": manifest_path,
+        "module": module_path,
+        "compiled_module_source": wasm.source_path,
+        "bundle_digest": bundle_digest,
+        "module_size_bytes": wasm.bytes.len(),
+        "task_descriptor_count": tasks.len(),
+        "entrypoint_count": entrypoints.len(),
+        "files": [
+            "manifest.json",
+            "module.wasm",
+            "task-descriptors.json",
+            "entrypoints.json",
+            "environments.json",
+            "source-provider.json",
+            "vfs-seed.json",
+            "debug-metadata.json",
+        ],
+    }))
+}
+
+fn write_json(path: &Path, value: &Value) -> Result<()> {
+    let mut bytes = serde_json::to_vec_pretty(value)?;
+    bytes.push(b'\n');
+    std::fs::write(path, bytes).with_context(|| format!("failed to write {}", path.display()))
+}

crates/clusterflux-cli/src/bundle.rs

@@ -0,0 +1,379 @@
+use std::collections::BTreeSet;
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use clusterflux_core::{
+    diagnose_environment_references, discover_source_debug_probes, BundleDebugProbe,
+    BundleIdentityInputs, BundleMetadata, Digest, EnvironmentReference, ProjectModel,
+    SelectedInput, SourceProviderKind, SourceProviderManifest,
+};
+use serde::Serialize;
+
+use crate::BundleInspectArgs;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct BundleInspection {
+    pub(crate) project: PathBuf,
+    pub(crate) default_source_providers: Vec<SourceProviderKind>,
+    pub(crate) source_provider_manifest: SourceProviderManifest,
+    pub(crate) source_provider_statuses: Vec<SourceProviderStatus>,
+    pub(crate) environment_diagnostics: Vec<EnvironmentDiagnosticReport>,
+    pub(crate) pre_schedule_diagnostics: Vec<CliDiagnostic>,
+    pub(crate) metadata: BundleMetadata,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct SourceProviderStatus {
+    pub(crate) provider: String,
+    pub(crate) status: String,
+    pub(crate) active: bool,
+    pub(crate) reason: String,
+    pub(crate) coordinator_checkout_required: bool,
+    pub(crate) coordinator_receives_source_bytes_by_default: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct EnvironmentDiagnosticReport {
+    pub(crate) path: String,
+    pub(crate) reference: EnvironmentReference,
+    pub(crate) message: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct CliDiagnostic {
+    pub(crate) severity: String,
+    pub(crate) category: String,
+    pub(crate) code: String,
+    pub(crate) message: String,
+    pub(crate) next_actions: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct SourceProviderSelection {
+    active_provider: SourceProviderKind,
+    statuses: Vec<SourceProviderStatus>,
+}
+
+pub(crate) fn discovered_environment_names(inspection: Option<&BundleInspection>) -> Vec<String> {
+    inspection
+        .map(|inspection| {
+            inspection
+                .metadata
+                .environments
+                .iter()
+                .map(|environment| environment.name.clone())
+                .collect()
+        })
+        .unwrap_or_default()
+}
+
+pub(crate) fn bundle_inspection(args: BundleInspectArgs, cwd: PathBuf) -> Result<BundleInspection> {
+    let project = args.project.unwrap_or(cwd);
+    let model = ProjectModel::discover_without_config(&project)?;
+    let selected_inputs = discover_selected_inputs(&project)?;
+    let debug_probes = discover_debug_probes(&project, &selected_inputs)?;
+    let provider_selection = source_provider_selection(
+        &project,
+        args.source_provider.as_deref(),
+        &args.disabled_source_providers,
+    );
+    let source_provider_manifest = source_provider_manifest(&provider_selection.active_provider);
+    source_provider_manifest.validate_public_mvp()?;
+    let environment_diagnostics =
+        environment_diagnostics_for_inputs(&project, &selected_inputs, &model.environments)?;
+    let identity_inputs = BundleIdentityInputs {
+        wasm_code: wasm_source_proxy_digest(&selected_inputs),
+        task_abi: task_abi_digest(&model),
+        entrypoints: model.entrypoints.keys().cloned().collect(),
+        default_entrypoint: model.default_entrypoint.clone(),
+        environments: model.environments.clone(),
+        source_provider_manifest: source_provider_manifest.digest.clone(),
+        source_transfer_policy: source_provider_manifest.transfer_policy.clone(),
+        selected_inputs,
+    };
+    let mut metadata = identity_inputs.inspectable_metadata();
+    metadata.debug_metadata.probes = debug_probes;
+    let pre_schedule_diagnostics = pre_schedule_diagnostics(
+        &provider_selection.statuses,
+        &environment_diagnostics,
+        &model.environments,
+    );
+
+    Ok(BundleInspection {
+        project,
+        default_source_providers: vec![SourceProviderKind::Filesystem, SourceProviderKind::Git],
+        source_provider_manifest,
+        source_provider_statuses: provider_selection.statuses,
+        environment_diagnostics,
+        pre_schedule_diagnostics,
+        metadata,
+    })
+}
+
+fn discover_selected_inputs(project: &Path) -> Result<Vec<SelectedInput>> {
+    let mut inputs = Vec::new();
+    for path in [
+        "Cargo.toml",
+        "Cargo.lock",
+        "src/main.rs",
+        "src/lib.rs",
+        "src/build.rs",
+    ] {
+        let absolute = project.join(path);
+        if !absolute.is_file() {
+            continue;
+        }
+        let bytes = std::fs::read(&absolute)?;
+        inputs.push(SelectedInput {
+            path: path.to_owned(),
+            digest: Digest::from_parts([
+                b"bundle-selected-input:v1".as_slice(),
+                path.as_bytes(),
+                bytes.as_slice(),
+            ]),
+        });
+    }
+    Ok(inputs)
+}
+
+fn source_provider_selection(
+    project: &Path,
+    requested_provider: Option<&str>,
+    disabled_providers: &[String],
+) -> SourceProviderSelection {
+    let has_git_checkout = project.join(".git").exists();
+    let disabled = disabled_providers
+        .iter()
+        .map(|provider| provider.trim().to_ascii_lowercase())
+        .collect::<BTreeSet<_>>();
+    let requested = requested_provider.map(source_provider_kind_from_id);
+    let active_provider = requested.clone().unwrap_or_else(|| {
+        if has_git_checkout && !disabled.contains("git") {
+            SourceProviderKind::Git
+        } else {
+            SourceProviderKind::Filesystem
+        }
+    });
+    let active_provider_id = active_provider.provider_id().to_owned();
+    let mut statuses = vec![
+        builtin_source_provider_status(
+            SourceProviderKind::Filesystem,
+            true,
+            disabled.contains("filesystem"),
+            active_provider_id == "filesystem",
+        ),
+        builtin_source_provider_status(
+            SourceProviderKind::Git,
+            has_git_checkout,
+            disabled.contains("git"),
+            active_provider_id == "git",
+        ),
+        SourceProviderStatus {
+            provider: "custom".to_owned(),
+            status: "disabled".to_owned(),
+            active: false,
+            reason: "no custom source-provider module was configured for this project".to_owned(),
+            coordinator_checkout_required: false,
+            coordinator_receives_source_bytes_by_default: false,
+        },
+    ];
+
+    if let Some(SourceProviderKind::Custom(provider)) = requested {
+        statuses.push(SourceProviderStatus {
+            provider,
+            status: "unsupported".to_owned(),
+            active: true,
+            reason: "custom source-provider modules must be supplied by public plugin/API code before use".to_owned(),
+            coordinator_checkout_required: false,
+            coordinator_receives_source_bytes_by_default: false,
+        });
+    }
+
+    SourceProviderSelection {
+        active_provider,
+        statuses,
+    }
+}
+
+fn source_provider_kind_from_id(provider: &str) -> SourceProviderKind {
+    match provider.trim().to_ascii_lowercase().as_str() {
+        "filesystem" => SourceProviderKind::Filesystem,
+        "git" => SourceProviderKind::Git,
+        other => SourceProviderKind::Custom(other.to_owned()),
+    }
+}
+
+fn builtin_source_provider_status(
+    kind: SourceProviderKind,
+    available: bool,
+    disabled: bool,
+    active: bool,
+) -> SourceProviderStatus {
+    let provider = kind.provider_id().to_owned();
+    let (status, reason) = if disabled {
+        (
+            "disabled",
+            format!("source provider `{provider}` was disabled by CLI override"),
+        )
+    } else if available {
+        (
+            "enabled",
+            format!("source provider `{provider}` is available in this checkout"),
+        )
+    } else {
+        (
+            "missing",
+            format!("source provider `{provider}` is not available for this checkout"),
+        )
+    };
+    SourceProviderStatus {
+        provider,
+        status: status.to_owned(),
+        active,
+        reason,
+        coordinator_checkout_required: false,
+        coordinator_receives_source_bytes_by_default: false,
+    }
+}
+
+fn source_provider_manifest(kind: &SourceProviderKind) -> SourceProviderManifest {
+    SourceProviderManifest::local_first(
+        kind.clone(),
+        "default source provider manifest; snapshot creation can be scheduled as a node task",
+    )
+}
+
+fn environment_diagnostics_for_inputs(
+    project: &Path,
+    selected_inputs: &[SelectedInput],
+    environments: &[clusterflux_core::EnvironmentResource],
+) -> Result<Vec<EnvironmentDiagnosticReport>> {
+    let mut diagnostics = Vec::new();
+    for input in selected_inputs {
+        if !input.path.ends_with(".rs") {
+            continue;
+        }
+        let source = std::fs::read_to_string(project.join(&input.path))
+            .with_context(|| format!("failed to read {}", project.join(&input.path).display()))?;
+        diagnostics.extend(
+            diagnose_environment_references(&source, environments)
+                .into_iter()
+                .map(|diagnostic| EnvironmentDiagnosticReport {
+                    path: input.path.clone(),
+                    reference: diagnostic.reference,
+                    message: diagnostic.message,
+                }),
+        );
+    }
+    Ok(diagnostics)
+}
+
+fn discover_debug_probes(
+    project: &Path,
+    selected_inputs: &[SelectedInput],
+) -> Result<Vec<BundleDebugProbe>> {
+    let mut probes = Vec::new();
+    for input in selected_inputs {
+        if !input.path.ends_with(".rs") {
+            continue;
+        }
+        let source = std::fs::read_to_string(project.join(&input.path))
+            .with_context(|| format!("failed to read {}", project.join(&input.path).display()))?;
+        probes.extend(discover_source_debug_probes(&input.path, &source));
+    }
+    Ok(probes)
+}
+
+fn pre_schedule_diagnostics(
+    source_provider_statuses: &[SourceProviderStatus],
+    environment_diagnostics: &[EnvironmentDiagnosticReport],
+    environments: &[clusterflux_core::EnvironmentResource],
+) -> Vec<CliDiagnostic> {
+    let mut diagnostics = Vec::new();
+    diagnostics.extend(
+        environment_diagnostics
+            .iter()
+            .map(|diagnostic| CliDiagnostic {
+                severity: "error".to_owned(),
+                category: "environment".to_owned(),
+                code: "missing_environment".to_owned(),
+                message: format!("{} at {}", diagnostic.message, diagnostic.path),
+                next_actions: vec![
+                    "create the missing envs/<name>/Containerfile or envs/<name>/Dockerfile"
+                        .to_owned(),
+                    "rerun clusterflux inspect".to_owned(),
+                ],
+            }),
+    );
+
+    diagnostics.extend(
+        source_provider_statuses
+            .iter()
+            .filter(|status| {
+                status.active
+                    && matches!(
+                        status.status.as_str(),
+                        "missing" | "disabled" | "unsupported"
+                    )
+            })
+            .map(|status| CliDiagnostic {
+                severity: "error".to_owned(),
+                category: "source_provider".to_owned(),
+                code: format!("source_provider_{}", status.status),
+                message: format!(
+                    "active source provider `{}` is {}: {}",
+                    status.provider, status.status, status.reason
+                ),
+                next_actions: vec![
+                    "choose an available source provider with --source-provider".to_owned(),
+                    "rerun clusterflux inspect --json".to_owned(),
+                ],
+            }),
+    );
+
+    for environment in environments {
+        if environment.requirements.capabilities.is_empty() {
+            continue;
+        }
+        let mut capabilities = environment
+            .requirements
+            .capabilities
+            .iter()
+            .map(|capability| format!("{capability:?}"))
+            .collect::<Vec<_>>();
+        capabilities.sort();
+        diagnostics.push(CliDiagnostic {
+            severity: "info".to_owned(),
+            category: "capability".to_owned(),
+            code: "environment_capability_requirements".to_owned(),
+            message: format!(
+                "environment `{}` requires node capabilities: {}",
+                environment.name,
+                capabilities.join(", ")
+            ),
+            next_actions: vec![
+                "attach a node that reports these capabilities before scheduling work".to_owned(),
+            ],
+        });
+    }
+
+    diagnostics
+}
+
+fn wasm_source_proxy_digest(selected_inputs: &[SelectedInput]) -> Digest {
+    let mut parts = vec![b"wasm-source-proxy:v1".to_vec()];
+    for input in selected_inputs {
+        parts.push(input.path.as_bytes().to_vec());
+        parts.push(input.digest.as_str().as_bytes().to_vec());
+    }
+    Digest::from_parts(parts)
+}
+
+pub(crate) fn task_abi_digest(model: &ProjectModel) -> Digest {
+    let mut parts = vec![b"task-abi:v1".to_vec()];
+    for entrypoint in model.entrypoints.values() {
+        parts.push(entrypoint.name.as_bytes().to_vec());
+        parts.push(entrypoint.function.as_bytes().to_vec());
+    }
+    Digest::from_parts(parts)
+}

crates/clusterflux-cli/src/client.rs

@@ -0,0 +1,191 @@
+use anyhow::{Context, Result};
+use clusterflux_control::{
+    endpoint_identity, endpoint_is_loopback, ControlSession, LOGIN_API_PATH,
+};
+use clusterflux_core::coordinator_wire_request;
+use serde_json::{json, Value};
+
+use crate::config::StoredCliSession;
+use crate::errors::cli_error_summary;
+use crate::CliScopeArgs;
+
+pub(crate) struct JsonLineSession {
+    inner: ControlSession,
+}
+
+impl JsonLineSession {
+    pub(crate) fn connect(addr: &str) -> Result<Self> {
+        let inner = ControlSession::connect(addr)
+            .with_context(|| format!("failed to connect to coordinator {addr}"))?;
+        Ok(Self { inner })
+    }
+
+    pub(crate) fn connect_browser_login(addr: &str) -> Result<Self> {
+        let inner = ControlSession::connect_to_api_path(addr, LOGIN_API_PATH)
+            .with_context(|| format!("failed to connect to hosted login endpoint {addr}"))?;
+        Ok(Self { inner })
+    }
+
+    pub(crate) fn request(&mut self, value: Value) -> Result<Value> {
+        let response = self.request_allow_error(value)?;
+        if response.get("type").and_then(Value::as_str) == Some("error") {
+            let message = response
+                .get("message")
+                .and_then(Value::as_str)
+                .map(str::to_owned)
+                .unwrap_or_else(|| response.to_string());
+            let machine_error = cli_error_summary(&message);
+            let category = machine_error
+                .get("category")
+                .and_then(Value::as_str)
+                .unwrap_or("unknown");
+            let exit_code = machine_error
+                .get("stable_exit_code")
+                .and_then(Value::as_i64)
+                .unwrap_or(1);
+            anyhow::bail!("coordinator error ({category}, exit {exit_code}): {response}");
+        }
+        Ok(response)
+    }
+
+    pub(crate) fn request_allow_error(&mut self, value: Value) -> Result<Value> {
+        let request_id = format!("cli-{}", self.inner.requests() + 1);
+        let wire_request = coordinator_wire_request(request_id, value);
+        self.inner
+            .request(&wire_request)
+            .map_err(anyhow::Error::from)
+    }
+
+    pub(crate) fn requests(&self) -> u64 {
+        self.inner.requests()
+    }
+}
+
+pub(crate) fn control_endpoint_identity(endpoint: &str) -> Result<String> {
+    endpoint_identity(endpoint).map_err(anyhow::Error::from)
+}
+
+pub(crate) fn authenticated_or_local_trusted_request(
+    coordinator: &str,
+    stored_session: Option<&StoredCliSession>,
+    authenticated_request: Value,
+    local_trusted_request: Value,
+) -> Result<Value> {
+    if let Some(session_secret) = stored_session_for_coordinator(coordinator, stored_session)
+        .and_then(|session| session.session_secret.as_ref())
+    {
+        Ok(json!({
+            "type": "authenticated",
+            "session_secret": session_secret,
+            "request": authenticated_request,
+        }))
+    } else if is_loopback_coordinator(coordinator) {
+        Ok(local_trusted_request)
+    } else {
+        anyhow::bail!(
+            "no authenticated CLI session matches coordinator {coordinator}; run `clusterflux login --browser` from the current project"
+        )
+    }
+}
+
+pub(crate) fn is_loopback_coordinator(coordinator: &str) -> bool {
+    endpoint_is_loopback(coordinator)
+}
+
+pub(crate) fn stored_session_for_coordinator<'a>(
+    coordinator: &str,
+    stored_session: Option<&'a StoredCliSession>,
+) -> Option<&'a StoredCliSession> {
+    stored_session.filter(|session| {
+        session.session_secret.is_some()
+            && control_endpoint_identity(&session.coordinator).ok()
+                == control_endpoint_identity(coordinator).ok()
+    })
+}
+
+pub(crate) fn list_task_events_if_available_with_session(
+    coordinator: Option<&str>,
+    scope: &CliScopeArgs,
+    process: Option<String>,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Option<Value>> {
+    let Some(coordinator) = coordinator else {
+        return Ok(None);
+    };
+    let mut session = JsonLineSession::connect(coordinator)?;
+    let response = session.request(authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({
+            "type": "list_task_events",
+            "process": process,
+        }),
+        json!({
+            "type": "list_task_events",
+            "tenant": scope.tenant,
+            "project": scope.project,
+            "actor_user": scope.user,
+            "process": process,
+        }),
+    )?)?;
+    Ok(Some(json!({
+        "coordinator": coordinator,
+        "response": response,
+        "coordinator_session_requests": session.requests(),
+    })))
+}
+
+pub(crate) fn list_attached_nodes_if_available_with_session(
+    coordinator: Option<&str>,
+    scope: &CliScopeArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let Some(coordinator) = coordinator else {
+        return Ok(json!({
+            "checked": false,
+            "source": "no_coordinator",
+            "count": 0,
+            "online": 0,
+            "response": null,
+        }));
+    };
+
+    let mut session = JsonLineSession::connect(coordinator)?;
+    let response = session.request(authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({
+            "type": "list_node_descriptors",
+        }),
+        json!({
+            "type": "list_node_descriptors",
+            "tenant": scope.tenant,
+            "project": scope.project,
+            "actor_user": scope.user,
+        }),
+    )?)?;
+    let descriptors = response
+        .get("descriptors")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    let online = descriptors
+        .iter()
+        .filter(|descriptor| {
+            descriptor
+                .get("online")
+                .and_then(Value::as_bool)
+                .unwrap_or(false)
+        })
+        .count();
+
+    Ok(json!({
+        "checked": true,
+        "source": "coordinator",
+        "coordinator": coordinator,
+        "count": descriptors.len(),
+        "online": online,
+        "response": response,
+        "coordinator_session_requests": session.requests(),
+    }))
+}

crates/clusterflux-cli/src/config.rs

@@ -0,0 +1,150 @@
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+
+use crate::CliScopeArgs;
+
+pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
+    "https://clusterflux.michelpaulissen.com";
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ProjectConfig {
+    pub(crate) tenant: String,
+    pub(crate) project: String,
+    pub(crate) user: String,
+    pub(crate) coordinator: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct StoredCliSession {
+    pub(crate) kind: String,
+    pub(crate) coordinator: String,
+    pub(crate) tenant: String,
+    pub(crate) project: String,
+    pub(crate) user: String,
+    pub(crate) cli_session_credential_kind: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) session_secret: Option<String>,
+    pub(crate) token_expiry_posture: String,
+    pub(crate) expires_at: Option<String>,
+    pub(crate) provider_tokens_exposed_to_cli: bool,
+    pub(crate) provider_tokens_sent_to_nodes: bool,
+    pub(crate) created_at_unix_seconds: u64,
+}
+
+pub(crate) fn default_hosted_coordinator_endpoint() -> String {
+    DEFAULT_HOSTED_COORDINATOR_ENDPOINT.to_owned()
+}
+
+pub(crate) fn project_config_file(project: &Path) -> PathBuf {
+    project.join(".clusterflux").join("project.json")
+}
+
+pub(crate) fn session_config_file(project: &Path) -> PathBuf {
+    project.join(".clusterflux").join("session.json")
+}
+
+pub(crate) fn read_project_config(project: &Path) -> Result<Option<ProjectConfig>> {
+    let file = project_config_file(project);
+    if !file.exists() {
+        return Ok(None);
+    }
+    let bytes =
+        std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
+    let config = serde_json::from_slice(&bytes)
+        .with_context(|| format!("failed to parse {}", file.display()))?;
+    Ok(Some(config))
+}
+
+pub(crate) fn write_project_config(project: &Path, config: &ProjectConfig) -> Result<()> {
+    let file = project_config_file(project);
+    if let Some(parent) = file.parent() {
+        std::fs::create_dir_all(parent)
+            .with_context(|| format!("failed to create {}", parent.display()))?;
+    }
+    std::fs::write(&file, serde_json::to_vec_pretty(config)?)
+        .with_context(|| format!("failed to write {}", file.display()))
+}
+
+pub(crate) fn read_cli_session(project: &Path) -> Result<Option<StoredCliSession>> {
+    let file = session_config_file(project);
+    if !file.exists() {
+        return Ok(None);
+    }
+    let bytes =
+        std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
+    let session = serde_json::from_slice(&bytes)
+        .with_context(|| format!("failed to parse {}", file.display()))?;
+    Ok(Some(session))
+}
+
+pub(crate) fn write_cli_session(project: &Path, session: &StoredCliSession) -> Result<PathBuf> {
+    let file = session_config_file(project);
+    if let Some(parent) = file.parent() {
+        std::fs::create_dir_all(parent)
+            .with_context(|| format!("failed to create {}", parent.display()))?;
+    }
+    let bytes = serde_json::to_vec_pretty(session)?;
+    let mut options = std::fs::OpenOptions::new();
+    options.create(true).truncate(true).write(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt;
+        options.mode(0o600);
+    }
+    use std::io::Write;
+    let mut output = options
+        .open(&file)
+        .with_context(|| format!("failed to open {}", file.display()))?;
+    output
+        .write_all(&bytes)
+        .with_context(|| format!("failed to write {}", file.display()))?;
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600))
+            .with_context(|| format!("failed to secure {}", file.display()))?;
+    }
+    Ok(file)
+}
+
+pub(crate) fn effective_project_scope(
+    scope: &CliScopeArgs,
+    config: Option<&ProjectConfig>,
+) -> CliScopeArgs {
+    CliScopeArgs {
+        coordinator: scope
+            .coordinator
+            .clone()
+            .or_else(|| config.and_then(|config| config.coordinator.clone())),
+        tenant: effective_scope_value(
+            &scope.tenant,
+            config.map(|config| config.tenant.as_str()),
+            "tenant",
+        ),
+        project: effective_scope_value(
+            &scope.project,
+            config.map(|config| config.project.as_str()),
+            "project",
+        ),
+        user: effective_scope_value(
+            &scope.user,
+            config.map(|config| config.user.as_str()),
+            "user",
+        ),
+        json: scope.json,
+    }
+}
+
+pub(crate) fn effective_scope_value(
+    cli_value: &str,
+    config_value: Option<&str>,
+    default_value: &str,
+) -> String {
+    if cli_value == default_value {
+        config_value.unwrap_or(cli_value).to_owned()
+    } else {
+        cli_value.to_owned()
+    }
+}

crates/clusterflux-cli/src/confirm.rs

@@ -0,0 +1,35 @@
+use serde_json::{json, Value};
+
+use crate::errors::cli_error_summary_for_category;
+
+pub(crate) fn confirmation_required_report(
+    command: &str,
+    operation: &str,
+    target: Value,
+    confirm_command: String,
+) -> Value {
+    let message = format!("{command} requires --yes before {operation}");
+    let next_actions = json!([
+        confirm_command,
+        "review the command target before confirming",
+        "rerun with --json to inspect the safe failure"
+    ]);
+    let mut machine_error = cli_error_summary_for_category("policy", &message);
+    if let Some(object) = machine_error.as_object_mut() {
+        object.insert("confirmation_required".to_owned(), json!(true));
+        object.insert("next_actions".to_owned(), next_actions.clone());
+    }
+    json!({
+        "command": command,
+        "status": "confirmation_required",
+        "operation": operation,
+        "target": target,
+        "requires_confirmation": true,
+        "confirmation_required": true,
+        "explicit_user_action_required": true,
+        "coordinator_request_sent": false,
+        "safe_failure": true,
+        "next_actions": next_actions,
+        "machine_error": machine_error,
+    })
+}

crates/clusterflux-cli/src/debug.rs

@@ -0,0 +1,120 @@
+use std::process::Command;
+
+use anyhow::{Context, Result};
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, stored_session_for_coordinator, JsonLineSession,
+};
+use crate::config::StoredCliSession;
+use crate::tools::dap_binary_path;
+use crate::{DapArgs, DebugAttachArgs};
+
+pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
+    Ok(json!({
+        "command": "dap",
+        "adapter": dap_binary_path()?.display().to_string(),
+        "args": args.args,
+        "private_website_required": false,
+    }))
+}
+
+pub(crate) fn exec_dap(args: DapArgs) -> Result<()> {
+    let status = Command::new(dap_binary_path()?)
+        .args(args.args)
+        .status()
+        .context("failed to launch clusterflux-debug-dap")?;
+    if !status.success() {
+        anyhow::bail!("clusterflux-debug-dap exited with {status}");
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+pub(crate) fn debug_attach_report_with_dap(args: DebugAttachArgs, dap: String) -> Result<Value> {
+    debug_attach_report_with_dap_and_session(args, dap, None)
+}
+
+pub(crate) fn debug_attach_report_with_dap_and_session(
+    mut args: DebugAttachArgs,
+    dap: String,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    if args.scope.coordinator.is_none() {
+        args.scope.coordinator = stored_session
+            .filter(|session| session.session_secret.is_some())
+            .map(|session| session.coordinator.clone());
+    }
+    if let Some(bound_session) = args
+        .scope
+        .coordinator
+        .as_deref()
+        .and_then(|coordinator| stored_session_for_coordinator(coordinator, stored_session))
+    {
+        args.scope.tenant = bound_session.tenant.clone();
+        args.scope.project = bound_session.project.clone();
+        args.scope.user = bound_session.user.clone();
+    }
+    if let Some(coordinator) = &args.scope.coordinator {
+        let tenant = args.scope.tenant.clone();
+        let project = args.scope.project.clone();
+        let user = args.scope.user.clone();
+        let process = args.process.clone();
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "debug_attach",
+                "process": process,
+            }),
+            json!({
+                "type": "debug_attach",
+                "tenant": tenant,
+                "project": project,
+                "actor_user": user,
+                "process": process,
+            }),
+        )?)?;
+        let authorization = response.get("authorization").cloned().unwrap_or_else(
+            || json!({"allowed": false, "reason": "missing authorization response"}),
+        );
+        return Ok(json!({
+            "command": "debug attach",
+            "process": process,
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "dap": dap,
+            "authorized": authorization
+                .get("allowed")
+                .cloned()
+                .unwrap_or(json!(false)),
+            "authorization": authorization,
+            "audit_event": response.get("audit_event").cloned().unwrap_or(Value::Null),
+            "charged_debug_read_bytes": response
+                .get("charged_debug_read_bytes")
+                .cloned()
+                .unwrap_or_else(|| json!(0)),
+            "used_debug_read_bytes": response
+                .get("used_debug_read_bytes")
+                .cloned()
+                .unwrap_or_else(|| json!(0)),
+            "debug_reads_quota_limited": true,
+            "private_website_required": false,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "debug attach",
+        "process": args.process,
+        "coordinator": args.scope.coordinator,
+        "tenant": args.scope.tenant,
+        "project": args.scope.project,
+        "dap": dap,
+        "authorized": "unknown_without_coordinator",
+        "debug_reads_quota_limited": "unknown_without_coordinator",
+        "private_website_required": false,
+    }))
+}

crates/clusterflux-cli/src/dispatch.rs

@@ -0,0 +1,343 @@
+use anyhow::Result;
+use clap::Parser;
+
+use super::*;
+
+pub(crate) fn run_cli() -> Result<()> {
+    let cli = Cli::parse();
+    match cli.command {
+        Commands::Doctor(args) => {
+            let json_output = args.scope.json;
+            let report = doctor_report(args, std::env::current_dir()?)?;
+            emit_report(&report, json_output)?;
+        }
+        Commands::Login(args) => {
+            let args = crate::auth_scope::login_args_for_project(args, &std::env::current_dir()?)?;
+            let json_output = args.json;
+            if args.non_interactive && !args.plan {
+                let report = non_interactive_browser_login_report(&args);
+                emit_report(&report, json_output)?;
+            } else if !args.plan {
+                let report = execute_interactive_browser_login(args)?;
+                if json_output {
+                    emit_report(&report, true)?;
+                } else {
+                    print_browser_login_success(&report);
+                }
+            } else {
+                let plan = login_plan(args);
+                emit_report(&plan, json_output)?;
+            }
+        }
+        Commands::Logout(args) => {
+            let json_output = args.scope.json;
+            let report = logout_report(args, std::env::current_dir()?, "logout")?;
+            emit_report(&report, json_output)?;
+        }
+        Commands::Auth { command } => {
+            let (report, json_output) = match command {
+                AuthCommands::Status(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        auth_status_report(args, std::env::current_dir()?)?,
+                        json_output,
+                    )
+                }
+                AuthCommands::ConnectSelfHosted(args) => {
+                    let json_output = args.scope.json;
+                    let secret = read_session_secret_from_stdin()?;
+                    (
+                        connect_self_hosted_report(args, std::env::current_dir()?, secret)?,
+                        json_output,
+                    )
+                }
+                AuthCommands::Logout(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        auth_logout_report(args, std::env::current_dir()?)?,
+                        json_output,
+                    )
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Agent {
+            command: AgentCommands::Enroll(args),
+        } => {
+            let json_output = args.json;
+            let plan = agent_enrollment_plan(args);
+            emit_report(&plan, json_output)?;
+        }
+        Commands::Key { command } => {
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            let (report, json_output) = match command {
+                KeyCommands::Add(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        key_add_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                KeyCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        key_list_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                KeyCommands::Revoke(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        key_revoke_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Project { command } => {
+            let cwd = std::env::current_dir()?;
+            let (report, json_output) = match command {
+                ProjectCommands::Init(args) => {
+                    let json_output = args.scope.json;
+                    (project_init_report(args, cwd)?, json_output)
+                }
+                ProjectCommands::Status(args) => {
+                    let json_output = args.scope.json;
+                    (project_status_report(args, cwd)?, json_output)
+                }
+                ProjectCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (project_list_report(args, cwd)?, json_output)
+                }
+                ProjectCommands::Select(args) => {
+                    let json_output = args.scope.json;
+                    (project_select_report(args, cwd)?, json_output)
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Inspect(args) => {
+            let json_output = args.json;
+            let inspection = bundle_inspection(args, std::env::current_dir()?)?;
+            emit_report(&inspection, json_output)?;
+        }
+        Commands::Build(args) => {
+            let json_output = args.json;
+            let report = build_report(args, std::env::current_dir()?)?;
+            emit_report(&report, json_output)?;
+        }
+        Commands::Bundle {
+            command: BundleCommands::Inspect(args),
+        } => {
+            let json_output = args.json;
+            let inspection = bundle_inspection(args, std::env::current_dir()?)?;
+            emit_report(&inspection, json_output)?;
+        }
+        Commands::Run(args) => {
+            let json_output = args.json;
+            let cwd = std::env::current_dir()?;
+            let session_project = args.project.clone().unwrap_or_else(|| cwd.clone());
+            let session = session_from_sources(&session_project)?;
+            let report = run_report(args, cwd, session)?;
+            emit_report(&report, json_output)?;
+        }
+        Commands::Node {
+            command: NodeCommands::Attach(args),
+        } => {
+            let json_output = args.json;
+            if args.coordinator.is_some() {
+                let report = execute_node_attach(args)?;
+                emit_report(&report, json_output)?;
+            } else {
+                let plan = attach_plan(args);
+                emit_report(&plan, json_output)?;
+            }
+        }
+        Commands::Node { command } => {
+            let cwd = std::env::current_dir()?;
+            let (report, json_output) = match command {
+                NodeCommands::Enroll(args) => {
+                    let json_output = args.scope.json;
+                    (node_enroll_report(args, cwd.clone())?, json_output)
+                }
+                NodeCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (node_list_report(args, cwd.clone())?, json_output)
+                }
+                NodeCommands::Status(args) => {
+                    let json_output = args.scope.json;
+                    (node_status_report(args, cwd.clone())?, json_output)
+                }
+                NodeCommands::Revoke(args) => {
+                    let json_output = args.scope.json;
+                    (node_revoke_report(args, cwd)?, json_output)
+                }
+                NodeCommands::Attach(_) => unreachable!("node attach is handled above"),
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Process { command } => {
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            let (report, json_output) = match command {
+                ProcessCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        process_list_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ProcessCommands::Status(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        process_status_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ProcessCommands::Restart(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        process_restart_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ProcessCommands::Cancel(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        process_cancel_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ProcessCommands::Abort(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        process_abort_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Task { command } => {
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            let (report, json_output) = match command {
+                TaskCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        task_list_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                TaskCommands::Restart(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        task_restart_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Logs(args) => {
+            let json_output = args.scope.json;
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            emit_report(
+                &logs_report_with_session(args, stored_session.as_ref())?,
+                json_output,
+            )?;
+        }
+        Commands::Artifact { command } => {
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            let (report, json_output) = match command {
+                ArtifactCommands::List(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        artifact_list_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ArtifactCommands::Download(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        artifact_download_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+                ArtifactCommands::Export(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        artifact_export_report_with_session(args, stored_session.as_ref())?,
+                        json_output,
+                    )
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+        Commands::Dap(args) => {
+            let json_output = args.json;
+            if args.plan {
+                emit_report(&dap_plan(args)?, json_output)?;
+            } else {
+                return exec_dap(args);
+            }
+        }
+        Commands::Debug {
+            command: DebugCommands::Attach(args),
+        } => {
+            let json_output = args.scope.json;
+            let cwd = std::env::current_dir()?;
+            let stored_session = read_cli_session(&cwd)?;
+            let dap = dap_binary_path()?.display().to_string();
+            emit_report(
+                &debug_attach_report_with_dap_and_session(args, dap, stored_session.as_ref())?,
+                json_output,
+            )?;
+        }
+        Commands::Quota {
+            command: QuotaCommands::Status(args),
+        } => {
+            let json_output = args.scope.json;
+            emit_report(
+                &quota_status_report(args, std::env::current_dir()?)?,
+                json_output,
+            )?;
+        }
+        Commands::Admin { command } => {
+            let (report, json_output) = match command {
+                AdminCommands::Status(args) => {
+                    let json_output = args.scope.json;
+                    (admin_status_report(args)?, json_output)
+                }
+                AdminCommands::Bootstrap(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        admin_bootstrap_report(args, std::env::current_dir()?)?,
+                        json_output,
+                    )
+                }
+                AdminCommands::RevokeNode(args) => {
+                    let json_output = args.scope.json;
+                    (
+                        node_revoke_report(args, std::env::current_dir()?)?,
+                        json_output,
+                    )
+                }
+                AdminCommands::StopProcess(args) => {
+                    let json_output = args.scope.json;
+                    (process_cancel_report(args)?, json_output)
+                }
+                AdminCommands::SuspendTenant(args) => {
+                    let json_output = args.scope.json;
+                    (admin_suspend_tenant_report(args)?, json_output)
+                }
+            };
+            emit_report(&report, json_output)?;
+        }
+    }
+    Ok(())
+}

crates/clusterflux-cli/src/doctor.rs

@@ -0,0 +1,163 @@
+use std::path::PathBuf;
+use std::time::Duration;
+
+use anyhow::{Context, Result};
+use clusterflux_control::ControlSession;
+use clusterflux_core::{coordinator_wire_request, Capability, NodeCapabilities};
+use serde_json::{json, Value};
+
+use crate::auth::auth_state_value;
+use crate::config::read_project_config;
+use crate::tools::{command_available, sibling_binary};
+use crate::DoctorArgs;
+
+const DOCTOR_COORDINATOR_TIMEOUT: Duration = Duration::from_millis(500);
+
+pub(crate) fn doctor_report(args: DoctorArgs, cwd: PathBuf) -> Result<Value> {
+    let config = read_project_config(&cwd)?;
+    let coordinator = args.scope.coordinator.or_else(|| {
+        config
+            .as_ref()
+            .and_then(|config| config.coordinator.clone())
+    });
+    let coordinator_reachability = coordinator_reachability(coordinator.as_deref());
+    let dependencies = json!({
+        "cargo": command_available("cargo"),
+        "git": command_available("git"),
+        "podman": command_available("podman"),
+        "clusterflux-node": command_available("clusterflux-node") || sibling_binary("clusterflux-node").is_some(),
+        "clusterflux-coordinator": command_available("clusterflux-coordinator") || sibling_binary("clusterflux-coordinator").is_some(),
+        "clusterflux-debug-dap": command_available("clusterflux-debug-dap") || sibling_binary("clusterflux-debug-dap").is_some(),
+    });
+    let node_readiness = NodeCapabilities::detect_current();
+    let node_readiness_summary = node_readiness_summary(&node_readiness, &dependencies);
+    Ok(json!({
+        "command": "doctor",
+        "cwd": cwd,
+        "coordinator": coordinator,
+        "coordinator_reachability": coordinator_reachability,
+        "auth": auth_state_value(&cwd)?,
+        "project": config,
+        "dependencies": dependencies,
+        "node_readiness": node_readiness,
+        "node_readiness_summary": node_readiness_summary,
+        "next_actions": [
+            "clusterflux login --browser",
+            "clusterflux project init",
+            "clusterflux node attach",
+            "clusterflux run"
+        ]
+    }))
+}
+
+fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value) -> Value {
+    let has_command = capabilities.capabilities.contains(&Capability::Command);
+    let has_vfs_artifacts = capabilities
+        .capabilities
+        .contains(&Capability::VfsArtifacts);
+    let has_source_filesystem = capabilities
+        .capabilities
+        .contains(&Capability::SourceFilesystem);
+    let has_container_backend = capabilities
+        .environment_backends
+        .contains(&clusterflux_core::EnvironmentBackend::Container);
+    let podman_available = dependencies
+        .get("podman")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let git_available = dependencies
+        .get("git")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let clusterflux_node_available = dependencies
+        .get("clusterflux-node")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+
+    let mut missing = Vec::new();
+    if has_container_backend && !podman_available {
+        missing.push("podman");
+    }
+    if capabilities.source_providers.contains("git") && !git_available {
+        missing.push("git");
+    }
+    if !clusterflux_node_available {
+        missing.push("clusterflux-node");
+    }
+
+    let basic_runtime_ready = true;
+    let local_dependencies_ready = missing.is_empty();
+    let status = if basic_runtime_ready && local_dependencies_ready {
+        "ready_to_attach"
+    } else if basic_runtime_ready {
+        "local_dependencies_missing"
+    } else {
+        "limited_capabilities"
+    };
+    let next_actions = if status == "ready_to_attach" {
+        vec![
+            "clusterflux node enroll",
+            "clusterflux node attach",
+            "clusterflux-node --worker",
+        ]
+    } else {
+        vec![
+            "install missing local dependencies",
+            "rerun clusterflux doctor",
+        ]
+    };
+
+    json!({
+        "status": status,
+        "explicit_attach_required": true,
+        "command_execution_capability": has_command,
+        "wasm_runtime": "wasmtime",
+        "wasm_runtime_is_baseline": true,
+        "artifact_capability": has_vfs_artifacts,
+        "source_filesystem_capability": has_source_filesystem,
+        "container_backend_reported": has_container_backend,
+        "podman_binary_available": podman_available,
+        "git_binary_available": git_available,
+        "node_binary_available": clusterflux_node_available,
+        "missing_local_dependencies": missing,
+        "source_providers": capabilities.source_providers.iter().collect::<Vec<_>>(),
+        "next_actions": next_actions,
+    })
+}
+
+fn coordinator_reachability(coordinator: Option<&str>) -> Value {
+    let Some(coordinator) = coordinator else {
+        return json!({
+            "checked": false,
+            "status": "not_configured",
+            "next_action": "run clusterflux login --browser or pass --coordinator"
+        });
+    };
+
+    match ping_coordinator(coordinator, DOCTOR_COORDINATOR_TIMEOUT) {
+        Ok(response) => json!({
+            "checked": true,
+            "status": "reachable",
+            "coordinator": coordinator,
+            "response": response
+        }),
+        Err(err) => json!({
+            "checked": true,
+            "status": "unreachable",
+            "coordinator": coordinator,
+            "error": err.to_string(),
+            "next_action": "check the coordinator URL, network, and service status"
+        }),
+    }
+}
+
+fn ping_coordinator(coordinator: &str, timeout: Duration) -> Result<Value> {
+    let mut session = ControlSession::connect_with_timeouts(coordinator, timeout, timeout)
+        .with_context(|| format!("failed to connect to coordinator {coordinator}"))?;
+    session
+        .request(&coordinator_wire_request(
+            "doctor-1",
+            json!({ "type": "ping" }),
+        ))
+        .with_context(|| format!("coordinator ping failed for {coordinator}"))
+}

crates/clusterflux-cli/src/errors.rs

@@ -0,0 +1,258 @@
+use serde_json::{json, Value};
+
+pub(crate) fn cli_error_summary(message: &str) -> Value {
+    let category = classify_cli_error_message(message);
+    cli_error_summary_for_category(category, message)
+}
+
+pub(crate) fn cli_error_summary_with_default(
+    message: &str,
+    default_category: &'static str,
+) -> Value {
+    let category = classify_cli_error_message(message);
+    let category = if category == "unknown" {
+        default_category
+    } else {
+        category
+    };
+    cli_error_summary_for_category(category, message)
+}
+
+pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &str) -> Value {
+    let mut summary = json!({
+        "category": category,
+        "stable_exit_code": cli_error_exit_code(category),
+        "process_exit_code_applied": false,
+        "retryable_after_user_action": cli_error_retryable_after_user_action(category),
+        "message": message,
+        "safe_failure": true,
+        "next_actions": cli_error_next_actions(category),
+    });
+    if category == "quota" {
+        if let Some(object) = summary.as_object_mut() {
+            object.insert(
+                "resource_category".to_owned(),
+                json!(
+                    quota_error_resource_category(message).unwrap_or_else(|| "unknown".to_owned())
+                ),
+            );
+            object.insert("community_tier_language".to_owned(), json!(true));
+            object.insert("community_tier_label".to_owned(), json!("community tier"));
+            object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
+        }
+    }
+    summary
+}
+
+fn quota_error_resource_category(message: &str) -> Option<String> {
+    let lower = message.to_ascii_lowercase();
+    for marker in [
+        "resource limit exceeded for ",
+        "quota unavailable for ",
+        "quota exceeded for ",
+        "limit exceeded for ",
+        "metering limit exceeded for ",
+        "quota limit for ",
+        "resource category ",
+        "resource=",
+        "resource:",
+    ] {
+        if let Some(index) = lower.find(marker) {
+            let start = index + marker.len();
+            let rest = &message[start..];
+            let value: String = rest
+                .chars()
+                .skip_while(|ch| ch.is_ascii_whitespace())
+                .take_while(|ch| {
+                    ch.is_ascii_alphanumeric()
+                        || *ch == '_'
+                        || *ch == '-'
+                        || *ch == '.'
+                        || *ch == ':'
+                })
+                .collect();
+            let value = value.trim_matches(|ch: char| ch == '.' || ch == ':' || ch == ',');
+            if !value.is_empty() {
+                return Some(value.to_owned());
+            }
+        }
+    }
+    None
+}
+
+fn classify_cli_error_message(message: &str) -> &'static str {
+    let message = message.to_ascii_lowercase();
+    if message.contains("already has active virtual process") || message.contains("active process")
+    {
+        return "active_process";
+    }
+    if message_mentions_locality_failure(&message) {
+        return "connectivity";
+    }
+    if message.contains("no capable node")
+        || message.contains("missing capability")
+        || message.contains("capability")
+        || message.contains("placement failed")
+        || message.contains("cmd.run")
+        || message.contains("node required")
+    {
+        return "capability";
+    }
+    if message.contains("quota")
+        || message.contains("resource limit")
+        || message.contains("limit exceeded")
+        || message.contains("metering")
+    {
+        return "quota";
+    }
+    if message.contains("policy")
+        || message.contains("disallowed")
+        || message.contains("not allowed")
+        || message.contains("suspended")
+        || message.contains("blocked by")
+        || message.contains("denied")
+    {
+        return "policy";
+    }
+    if message.contains("unauthenticated")
+        || message.contains("not authenticated")
+        || message.contains("login required")
+        || message.contains("browser login required")
+        || message.contains("missing session")
+        || message.contains("no cli session")
+        || message.contains("session credential has expired")
+        || message.contains("session credential has been revoked")
+        || message.contains("401")
+    {
+        return "authentication";
+    }
+    if message.contains("unauthorized")
+        || message.contains("not authorized")
+        || message.contains("forbidden")
+        || message.contains("permission")
+        || message.contains("tenant mismatch")
+        || message.contains("project mismatch")
+    {
+        return "authorization";
+    }
+    if message.contains("failed to connect")
+        || message.contains("connection refused")
+        || message.contains("closed session")
+        || message.contains("timed out")
+        || message.contains("timeout")
+        || message.contains("name resolution")
+        || message.contains("network unreachable")
+        || message.contains("dns")
+    {
+        return "connectivity";
+    }
+    if message.contains("missing environment")
+        || message.contains("environment mismatch")
+        || message.contains("incompatible environment")
+        || message.contains("containerfile")
+        || message.contains("dockerfile")
+        || message.contains("envs/")
+    {
+        return "environment";
+    }
+    if message.contains("task exited")
+        || message.contains("exit status")
+        || message.contains("status code")
+        || message.contains("stderr")
+        || message.contains("panic")
+        || message.contains("program error")
+        || message.contains("command failed")
+    {
+        return "program";
+    }
+    "unknown"
+}
+
+pub(crate) fn message_mentions_locality_failure(message: &str) -> bool {
+    message.contains("direct connectivity unavailable")
+        || message.contains("direct transfer")
+        || message.contains("source snapshot unavailable")
+        || (message.contains("required artifact")
+            && message.contains("unavailable")
+            && message.contains("direct connectivity"))
+        || message.contains("locality assumption")
+}
+
+fn cli_error_exit_code(category: &str) -> i64 {
+    match category {
+        "authentication" => 20,
+        "authorization" => 21,
+        "quota" => 22,
+        "policy" => 23,
+        "capability" => 24,
+        "connectivity" => 25,
+        "environment" => 26,
+        "program" => 27,
+        "active_process" => 28,
+        _ => 1,
+    }
+}
+
+fn cli_error_retryable_after_user_action(category: &str) -> bool {
+    matches!(
+        category,
+        "authentication"
+            | "authorization"
+            | "quota"
+            | "policy"
+            | "capability"
+            | "connectivity"
+            | "environment"
+            | "program"
+            | "active_process"
+    )
+}
+
+fn cli_error_next_actions(category: &str) -> Vec<&'static str> {
+    match category {
+        "authentication" => vec!["clusterflux login --browser", "clusterflux auth status"],
+        "authorization" => vec![
+            "clusterflux auth status",
+            "check tenant/project selection",
+            "ask an admin to grant access",
+        ],
+        "quota" => vec![
+            "clusterflux quota status",
+            "reduce concurrent work or wait for usage to fall",
+        ],
+        "policy" => vec![
+            "clusterflux doctor",
+            "check coordinator policy for this action",
+        ],
+        "capability" => vec![
+            "clusterflux node list",
+            "attach a node with the required capabilities",
+            "check tenant/project on the attached node",
+        ],
+        "connectivity" => vec![
+            "clusterflux doctor",
+            "check the coordinator endpoint and network reachability",
+        ],
+        "environment" => vec![
+            "clusterflux inspect",
+            "check envs/<name>/Containerfile or envs/<name>/Dockerfile",
+        ],
+        "program" => vec![
+            "clusterflux logs",
+            "fix the program or task command and rerun",
+        ],
+        "active_process" => vec![
+            "clusterflux process list",
+            "clusterflux process status",
+            "clusterflux debug attach",
+            "clusterflux process restart --yes",
+            "clusterflux process cancel --yes",
+            "clusterflux process abort --yes",
+            "use another Coordinator Project",
+        ],
+        _ => vec![
+            "clusterflux doctor",
+            "rerun with --json for machine-readable details",
+        ],
+    }
+}

crates/clusterflux-cli/src/key.rs

@@ -0,0 +1,271 @@
+use anyhow::Result;
+use clusterflux_core::Digest;
+use serde_json::{json, Value};
+
+use crate::client::{authenticated_or_local_trusted_request, JsonLineSession};
+use crate::config::StoredCliSession;
+use crate::{confirmation_required_report, KeyAddArgs, KeyListArgs, KeyRevokeArgs};
+
+#[cfg(test)]
+pub(crate) fn key_add_report(args: KeyAddArgs) -> Result<Value> {
+    key_add_report_with_session(args, None)
+}
+
+pub(crate) fn key_add_report_with_session(
+    args: KeyAddArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
+    if let Some(coordinator) = &coordinator {
+        let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
+            &session.tenant
+        });
+        let project = effective_session_value(stored_session, &args.scope.project, |session| {
+            &session.project
+        });
+        let user =
+            effective_session_value(stored_session, &args.scope.user, |session| &session.user);
+        let agent = args.agent.clone();
+        let public_key_fingerprint = Digest::sha256(&args.public_key);
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "register_agent_public_key",
+                "agent": agent,
+                "public_key": args.public_key,
+            }),
+            json!({
+                "type": "register_agent_public_key",
+                "tenant": tenant,
+                "project": project,
+                "user": user,
+                "agent": agent,
+                "public_key": args.public_key,
+            }),
+        )?)?;
+        let record = response.get("record").cloned().unwrap_or_else(|| {
+            json!({
+                "tenant": tenant,
+                "project": project,
+                "user": user,
+                "agent": agent,
+                "public_key_fingerprint": public_key_fingerprint,
+                "scopes": ["project:read", "project:run"],
+                "human_account_creation_privilege": false,
+                "browser_interaction_required_each_run": false,
+            })
+        });
+        return Ok(json!({
+            "command": "key add",
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "agent": agent,
+            "public_key_fingerprint": record
+                .get("public_key_fingerprint")
+                .cloned()
+                .unwrap_or_else(|| json!(public_key_fingerprint)),
+            "credential_scope": {
+                "tenant": tenant,
+                "project": project,
+                "actions": record
+                    .get("scopes")
+                    .cloned()
+                    .unwrap_or_else(|| json!(["project:read", "project:run"])),
+                "human_account_creation_privilege": record
+                    .get("human_account_creation_privilege")
+                    .cloned()
+                    .unwrap_or(json!(false)),
+            },
+            "browser_interaction_required_each_run": record
+                .get("browser_interaction_required_each_run")
+                .cloned()
+                .unwrap_or(json!(false)),
+            "attribution": {
+                "registered_by_user": user,
+                "agent": agent,
+                "credential_kind": "public_key",
+            },
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "key add",
+        "status": "planned_without_coordinator",
+        "agent": args.agent,
+        "public_key_fingerprint": Digest::sha256(args.public_key),
+        "browser_interaction_required_each_run": false,
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn key_list_report(args: KeyListArgs) -> Result<Value> {
+    key_list_report_with_session(args, None)
+}
+
+pub(crate) fn key_list_report_with_session(
+    args: KeyListArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
+    if let Some(coordinator) = &coordinator {
+        let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
+            &session.tenant
+        });
+        let project = effective_session_value(stored_session, &args.scope.project, |session| {
+            &session.project
+        });
+        let user =
+            effective_session_value(stored_session, &args.scope.user, |session| &session.user);
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "list_agent_public_keys",
+            }),
+            json!({
+                "type": "list_agent_public_keys",
+                "tenant": tenant,
+                "project": project,
+                "user": user,
+            }),
+        )?)?;
+        return Ok(json!({
+            "command": "key list",
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "records": response
+                .get("records")
+                .cloned()
+                .unwrap_or_else(|| json!([])),
+            "credential_scope": {
+                "tenant": tenant,
+                "project": project,
+                "listed_for_user": user,
+                "human_account_creation_privilege": false,
+            },
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "key list",
+        "status": "requires_coordinator",
+        "records": [],
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn key_revoke_report(args: KeyRevokeArgs) -> Result<Value> {
+    key_revoke_report_with_session(args, None)
+}
+
+pub(crate) fn key_revoke_report_with_session(
+    args: KeyRevokeArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "key revoke",
+            "revoke_agent_public_key",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "user": args.scope.user,
+                "agent": args.agent,
+            }),
+            format!("clusterflux key revoke --agent {} --yes", args.agent),
+        ));
+    }
+    let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
+    if let Some(coordinator) = &coordinator {
+        let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
+            &session.tenant
+        });
+        let project = effective_session_value(stored_session, &args.scope.project, |session| {
+            &session.project
+        });
+        let user =
+            effective_session_value(stored_session, &args.scope.user, |session| &session.user);
+        let agent = args.agent.clone();
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "revoke_agent_public_key",
+                "agent": agent,
+            }),
+            json!({
+                "type": "revoke_agent_public_key",
+                "tenant": tenant,
+                "project": project,
+                "user": user,
+                "agent": agent,
+            }),
+        )?)?;
+        return Ok(json!({
+            "command": "key revoke",
+            "coordinator": coordinator,
+            "requires_confirmation": !args.yes,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "agent": agent,
+            "revoked": response
+                .pointer("/record/revoked")
+                .cloned()
+                .unwrap_or(json!(true)),
+            "credential_scope": {
+                "tenant": tenant,
+                "project": project,
+                "revoked_for_user": user,
+                "human_account_creation_privilege": false,
+            },
+            "attribution": {
+                "revoked_by_user": user,
+                "agent": agent,
+                "credential_kind": "public_key",
+            },
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "key revoke",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "agent": args.agent,
+    }))
+}
+
+fn effective_coordinator(
+    requested: &Option<String>,
+    stored_session: Option<&StoredCliSession>,
+) -> Option<String> {
+    requested.clone().or_else(|| {
+        stored_session
+            .filter(|session| session.session_secret.is_some())
+            .map(|session| session.coordinator.clone())
+    })
+}
+
+fn effective_session_value(
+    stored_session: Option<&StoredCliSession>,
+    requested: &str,
+    value: impl FnOnce(&StoredCliSession) -> &String,
+) -> String {
+    stored_session
+        .filter(|session| session.session_secret.is_some())
+        .map(value)
+        .cloned()
+        .unwrap_or_else(|| requested.to_owned())
+}

crates/clusterflux-cli/src/logout.rs

@@ -0,0 +1,127 @@
+use std::path::PathBuf;
+
+use anyhow::{Context, Result};
+use serde_json::{json, Value};
+
+use crate::client::JsonLineSession;
+use crate::config::{read_cli_session, session_config_file, StoredCliSession};
+use crate::confirm::confirmation_required_report;
+use crate::errors::cli_error_summary;
+use crate::AuthLogoutArgs;
+
+pub(crate) fn auth_logout_report(args: AuthLogoutArgs, cwd: PathBuf) -> Result<Value> {
+    logout_report(args, cwd, "auth logout")
+}
+
+pub(crate) fn logout_report(args: AuthLogoutArgs, cwd: PathBuf, command: &str) -> Result<Value> {
+    let session_file = session_config_file(&cwd);
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            command,
+            "delete_cli_session",
+            json!({
+                "session_file": session_file,
+                "node_credentials_untouched": true,
+            }),
+            format!("clusterflux {command} --yes"),
+        ));
+    }
+    let stored_session = read_cli_session(&cwd).ok().flatten();
+    let coordinator_revocation = revoke_stored_cli_session_if_possible(stored_session.as_ref());
+    let existed = session_file.exists();
+    if existed {
+        std::fs::remove_file(&session_file)
+            .with_context(|| format!("failed to remove {}", session_file.display()))?;
+    }
+    Ok(json!({
+        "command": command,
+        "requires_confirmation": !args.yes,
+        "removed_cli_session_file": existed,
+        "server_session_revocation": coordinator_revocation,
+        "node_credentials_untouched": true,
+        "session_file": session_file,
+    }))
+}
+
+fn revoke_stored_cli_session_if_possible(stored_session: Option<&StoredCliSession>) -> Value {
+    let Some(session) = stored_session else {
+        return json!({
+            "attempted": false,
+            "revoked": false,
+            "reason": "no_parseable_cli_session",
+        });
+    };
+    let Some(session_secret) = session.session_secret.as_deref() else {
+        return json!({
+            "attempted": false,
+            "revoked": false,
+            "reason": "no_cli_session_secret",
+            "coordinator": session.coordinator,
+        });
+    };
+    // A CLI session credential is authority issued by one coordinator. Never
+    // redirect it to a command-line endpoint override.
+    let coordinator = session.coordinator.as_str();
+    let mut coordinator_session = match JsonLineSession::connect(coordinator) {
+        Ok(session) => session,
+        Err(error) => {
+            let message = error.to_string();
+            return json!({
+                "attempted": true,
+                "revoked": false,
+                "reachable": false,
+                "coordinator": coordinator,
+                "error": message,
+                "machine_error": cli_error_summary(&message),
+                "next_actions": ["clusterflux login --browser"],
+            });
+        }
+    };
+    let response = match coordinator_session.request_allow_error(json!({
+        "type": "authenticated",
+        "session_secret": session_secret,
+        "request": {
+            "type": "revoke_cli_session",
+        },
+    })) {
+        Ok(response) => response,
+        Err(error) => {
+            let message = error.to_string();
+            return json!({
+                "attempted": true,
+                "revoked": false,
+                "reachable": false,
+                "coordinator": coordinator,
+                "error": message,
+                "machine_error": cli_error_summary(&message),
+                "coordinator_session_requests": coordinator_session.requests(),
+                "next_actions": ["clusterflux login --browser"],
+            });
+        }
+    };
+    let revoked = response.get("type").and_then(Value::as_str) == Some("cli_session_revoked");
+    if !revoked {
+        let message = response
+            .get("message")
+            .and_then(Value::as_str)
+            .unwrap_or("coordinator did not revoke CLI session");
+        return json!({
+            "attempted": true,
+            "revoked": false,
+            "reachable": true,
+            "coordinator": coordinator,
+            "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("unknown"),
+            "machine_error": cli_error_summary(message),
+            "coordinator_session_requests": coordinator_session.requests(),
+            "next_actions": ["clusterflux login --browser"],
+        });
+    }
+    json!({
+        "attempted": true,
+        "revoked": true,
+        "reachable": true,
+        "coordinator": coordinator,
+        "coordinator_response_type": "cli_session_revoked",
+        "coordinator_session_requests": coordinator_session.requests(),
+    })
+}

crates/clusterflux-cli/src/logs.rs

@@ -0,0 +1,34 @@
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::list_task_events_if_available_with_session;
+use crate::config::StoredCliSession;
+use crate::process_events::log_entries;
+use crate::LogsArgs;
+
+#[cfg(test)]
+pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
+    logs_report_with_session(args, None)
+}
+
+pub(crate) fn logs_report_with_session(
+    args: LogsArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let events = list_task_events_if_available_with_session(
+        args.scope.coordinator.as_deref(),
+        &args.scope,
+        args.process.clone(),
+        stored_session,
+    )?;
+    let log_entries = log_entries(events.as_ref(), args.task.as_deref());
+    Ok(json!({
+        "command": "logs",
+        "process": args.process,
+        "task": args.task,
+        "log_entries": log_entries,
+        "logs_are_capped": true,
+        "secret_redaction_policy": "configured-redaction-boundary",
+        "events": events,
+    }))
+}

crates/clusterflux-cli/src/main.rs

@@ -0,0 +1,689 @@
+#[cfg(test)]
+use std::io::{BufRead, BufReader, Write};
+#[cfg(test)]
+use std::net::TcpListener;
+#[cfg(test)]
+use std::path::Path;
+use std::path::PathBuf;
+
+use clap::{Args, Parser, Subcommand};
+#[cfg(test)]
+use clusterflux_core::{Capability, Digest, ProjectModel, SourceProviderKind};
+#[cfg(test)]
+use serde_json::json;
+use serde_json::Value;
+
+mod admin;
+mod agent;
+mod artifact;
+mod auth;
+mod auth_scope;
+mod build;
+mod bundle;
+mod client;
+mod config;
+mod confirm;
+mod debug;
+mod dispatch;
+mod doctor;
+mod errors;
+mod key;
+mod logout;
+mod logs;
+mod node;
+mod output;
+mod process;
+mod process_events;
+mod project;
+mod quota;
+mod run;
+mod task;
+mod tools;
+
+use admin::{admin_bootstrap_report, admin_status_report, admin_suspend_tenant_report};
+use agent::agent_enrollment_plan;
+#[cfg(test)]
+use artifact::{
+    artifact_download_report, artifact_export_report, artifact_list_report,
+    artifact_stream_summary, DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
+};
+use artifact::{
+    artifact_download_report_with_session, artifact_export_report_with_session,
+    artifact_list_report_with_session,
+};
+use auth::{
+    auth_status_report, connect_self_hosted_report, execute_interactive_browser_login, login_plan,
+    non_interactive_browser_login_report, print_browser_login_success,
+    read_session_secret_from_stdin,
+};
+#[cfg(test)]
+use auth::{contains_provider_token_field, stored_cli_session_from_login_response, LoginFlowPlan};
+#[cfg(test)]
+use auth_scope::login_args_for_project;
+use build::build_report;
+#[cfg(test)]
+use bundle::task_abi_digest;
+pub(crate) use bundle::{bundle_inspection, discovered_environment_names};
+#[cfg(test)]
+use client::control_endpoint_identity;
+use config::{default_hosted_coordinator_endpoint, read_cli_session};
+#[cfg(test)]
+use config::{
+    read_project_config, session_config_file, write_cli_session, write_project_config,
+    ProjectConfig, StoredCliSession, DEFAULT_HOSTED_COORDINATOR_ENDPOINT,
+};
+pub(crate) use confirm::confirmation_required_report;
+#[cfg(test)]
+use debug::debug_attach_report_with_dap;
+use debug::{dap_plan, debug_attach_report_with_dap_and_session, exec_dap};
+use doctor::doctor_report;
+use errors::cli_error_summary;
+#[cfg(test)]
+use errors::cli_error_summary_for_category;
+#[cfg(test)]
+use key::{key_add_report, key_list_report, key_revoke_report};
+use key::{
+    key_add_report_with_session, key_list_report_with_session, key_revoke_report_with_session,
+};
+use logout::{auth_logout_report, logout_report};
+#[cfg(test)]
+use logs::logs_report;
+use logs::logs_report_with_session;
+use node::{
+    attach_plan, execute_node_attach, node_enroll_report, node_list_report, node_revoke_report,
+    node_status_report,
+};
+use output::emit_report;
+#[cfg(test)]
+use output::{apply_command_report_exit_code, human_report};
+use process::{
+    process_abort_report_with_session, process_cancel_report, process_cancel_report_with_session,
+    process_list_report_with_session, process_restart_report_with_session,
+    process_status_report_with_session,
+};
+#[cfg(test)]
+use process::{process_restart_report, process_status_report};
+#[cfg(test)]
+use process_events::task_summaries;
+#[cfg(test)]
+use process_events::{
+    artifact_download_session_summary, artifact_export_plan_summary, log_entries,
+};
+use project::{
+    project_init_report, project_list_report, project_select_report, project_status_report,
+};
+use quota::quota_status_report;
+#[cfg(test)]
+use run::{
+    agent_session_from_keys, run_plan, run_start_summary, should_execute_local_node, CliSession,
+    CoordinatorSelection,
+};
+use run::{run_report, session_from_sources};
+#[cfg(test)]
+use task::{task_list_report, task_restart_report};
+use task::{task_list_report_with_session, task_restart_report_with_session};
+use tools::dap_binary_path;
+
+#[derive(Clone, Debug, Parser)]
+#[command(
+    name = "clusterflux",
+    version,
+    arg_required_else_help = true,
+    about = "Clusterflux distributed Wasm runtime CLI.",
+    after_help = "Primary workflow:
+  1. clusterflux login --browser
+  2. clusterflux project init
+  3. clusterflux node enroll; clusterflux node attach; clusterflux-node --worker
+  4. clusterflux run [entry] --project <path>
+  5. Debug with VS Code \"Clusterflux: Launch Virtual Process\" or clusterflux dap
+  6. Inspect with clusterflux process list, clusterflux process status, task list, logs, and artifact list
+  7. Request cooperative shutdown with process cancel; force termination with process abort
+
+Use --json on primary commands for scriptable output. Hosted account creation happens in the browser login flow."
+)]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum Commands {
+    Doctor(DoctorArgs),
+    Login(LoginArgs),
+    Logout(AuthLogoutArgs),
+    Auth {
+        #[command(subcommand)]
+        command: AuthCommands,
+    },
+    Agent {
+        #[command(subcommand)]
+        command: AgentCommands,
+    },
+    Key {
+        #[command(subcommand)]
+        command: KeyCommands,
+    },
+    Project {
+        #[command(subcommand)]
+        command: ProjectCommands,
+    },
+    Inspect(BundleInspectArgs),
+    Build(BuildArgs),
+    Bundle {
+        #[command(subcommand)]
+        command: BundleCommands,
+    },
+    Run(RunArgs),
+    Node {
+        #[command(subcommand)]
+        command: NodeCommands,
+    },
+    Process {
+        #[command(subcommand)]
+        command: ProcessCommands,
+    },
+    Task {
+        #[command(subcommand)]
+        command: TaskCommands,
+    },
+    Logs(LogsArgs),
+    Artifact {
+        #[command(subcommand)]
+        command: ArtifactCommands,
+    },
+    Dap(DapArgs),
+    Debug {
+        #[command(subcommand)]
+        command: DebugCommands,
+    },
+    Quota {
+        #[command(subcommand)]
+        command: QuotaCommands,
+    },
+    Admin {
+        #[command(subcommand)]
+        command: AdminCommands,
+    },
+}
+
+#[derive(Clone, Debug, Parser)]
+struct DoctorArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct LoginArgs {
+    #[arg(long = "browser")]
+    _browser: bool,
+    #[arg(long)]
+    non_interactive: bool,
+    #[arg(long)]
+    plan: bool,
+    #[arg(long)]
+    json: bool,
+    #[arg(long, default_value_t = default_hosted_coordinator_endpoint())]
+    coordinator: String,
+    #[arg(long = "project-id", default_value = "project")]
+    project: String,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum AuthCommands {
+    Status(AuthStatusArgs),
+    ConnectSelfHosted(ConnectSelfHostedArgs),
+    Logout(AuthLogoutArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AuthStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ConnectSelfHostedArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, required = true)]
+    session_secret_stdin: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AuthLogoutArgs {
+    #[arg(long)]
+    yes: bool,
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum AgentCommands {
+    Enroll(AgentEnrollArgs),
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum KeyCommands {
+    Add(KeyAddArgs),
+    List(KeyListArgs),
+    Revoke(KeyRevokeArgs),
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum BundleCommands {
+    Inspect(BundleInspectArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AgentEnrollArgs {
+    #[arg(long)]
+    json: bool,
+    #[arg(long = "public-key")]
+    public_key: String,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct KeyAddArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "agent")]
+    agent: String,
+    #[arg(long = "public-key")]
+    public_key: String,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct KeyListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct KeyRevokeArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "agent")]
+    agent: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum ProjectCommands {
+    Init(ProjectInitArgs),
+    Status(ProjectStatusArgs),
+    List(ProjectListArgs),
+    Select(ProjectSelectArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProjectInitArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "project")]
+    new_project: String,
+    #[arg(long, default_value = "Clusterflux Project")]
+    name: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProjectStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProjectListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProjectSelectArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(value_name = "PROJECT")]
+    selected_project: String,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct BundleInspectArgs {
+    #[arg(long)]
+    project: Option<PathBuf>,
+    #[arg(long = "source-provider")]
+    source_provider: Option<String>,
+    #[arg(long = "disable-source-provider")]
+    disabled_source_providers: Vec<String>,
+    #[arg(long)]
+    json: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct BuildArgs {
+    #[arg(long)]
+    project: Option<PathBuf>,
+    #[arg(long = "source-provider")]
+    source_provider: Option<String>,
+    #[arg(long = "disable-source-provider")]
+    disabled_source_providers: Vec<String>,
+    #[arg(long)]
+    output: Option<PathBuf>,
+    #[arg(long)]
+    json: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct RunArgs {
+    entry: Option<String>,
+    #[arg(long)]
+    project: Option<PathBuf>,
+    #[arg(long)]
+    coordinator: Option<String>,
+    #[arg(long)]
+    local: bool,
+    #[arg(long)]
+    non_interactive: bool,
+    #[arg(long)]
+    json: bool,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum NodeCommands {
+    Attach(AttachArgs),
+    Enroll(NodeEnrollArgs),
+    List(NodeListArgs),
+    Status(NodeStatusArgs),
+    Revoke(NodeRevokeArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AttachArgs {
+    #[arg(long)]
+    coordinator: Option<String>,
+    #[arg(long, default_value = "tenant")]
+    tenant: String,
+    #[arg(long = "project-id", default_value = "project")]
+    project: String,
+    #[arg(long)]
+    node: Option<String>,
+    #[arg(long = "cap")]
+    caps: Vec<String>,
+    #[arg(long = "enrollment-grant")]
+    enrollment_grant: Option<String>,
+    #[arg(long = "public-key")]
+    public_key: Option<String>,
+    #[arg(long)]
+    json: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct NodeEnrollArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value_t = 900)]
+    ttl_seconds: u64,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct NodeListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct NodeStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    node: Option<String>,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct NodeRevokeArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    node: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum ProcessCommands {
+    List(ProcessListArgs),
+    Status(ProcessStatusArgs),
+    Restart(ProcessRestartArgs),
+    Cancel(ProcessCancelArgs),
+    Abort(ProcessAbortArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProcessListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProcessStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProcessRestartArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProcessCancelArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+    #[arg(long)]
+    node: Option<String>,
+    #[arg(long)]
+    task: Option<String>,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ProcessAbortArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum TaskCommands {
+    List(TaskListArgs),
+    Restart(TaskRestartArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct TaskListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    process: Option<String>,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct TaskRestartArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    task: String,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct LogsArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    process: Option<String>,
+    #[arg(long)]
+    task: Option<String>,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum ArtifactCommands {
+    List(ArtifactListArgs),
+    Download(ArtifactDownloadArgs),
+    Export(ArtifactExportArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ArtifactListArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    process: Option<String>,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ArtifactDownloadArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    artifact: String,
+    /// Write the verified artifact bytes to this local path.
+    #[arg(long)]
+    to: Option<PathBuf>,
+    #[arg(long, default_value_t = 64 * 1024 * 1024)]
+    max_bytes: u64,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct ArtifactExportArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    artifact: String,
+    #[arg(long)]
+    to: PathBuf,
+    #[arg(long, default_value = "node-local")]
+    receiver_node: String,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct DapArgs {
+    #[arg(long)]
+    plan: bool,
+    #[arg(long)]
+    json: bool,
+    #[arg(last = true)]
+    args: Vec<String>,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum DebugCommands {
+    Attach(DebugAttachArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct DebugAttachArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "vp-current")]
+    process: String,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum QuotaCommands {
+    Status(QuotaStatusArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct QuotaStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+}
+
+#[derive(Clone, Debug, Subcommand)]
+enum AdminCommands {
+    Status(AdminStatusArgs),
+    Bootstrap(AdminBootstrapArgs),
+    RevokeNode(NodeRevokeArgs),
+    StopProcess(ProcessCancelArgs),
+    SuspendTenant(AdminSuspendTenantArgs),
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AdminStatusArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long)]
+    admin_token: Option<String>,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AdminBootstrapArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long, default_value = "Clusterflux Project")]
+    name: String,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Parser)]
+struct AdminSuspendTenantArgs {
+    #[command(flatten)]
+    scope: CliScopeArgs,
+    #[arg(long = "target-tenant")]
+    target_tenant: Option<String>,
+    #[arg(long)]
+    admin_token: Option<String>,
+    #[arg(long)]
+    yes: bool,
+}
+
+#[derive(Clone, Debug, Args)]
+struct CliScopeArgs {
+    #[arg(long)]
+    coordinator: Option<String>,
+    #[arg(long, default_value = "tenant")]
+    tenant: String,
+    #[arg(long = "project-id", default_value = "project")]
+    project: String,
+    #[arg(long, default_value = "user")]
+    user: String,
+    #[arg(long)]
+    json: bool,
+}
+
+fn main() {
+    if let Err(error) = dispatch::run_cli() {
+        let message = error.to_string();
+        let machine_error = cli_error_summary(&message);
+        let category = machine_error
+            .get("category")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        let exit_code = machine_error
+            .get("stable_exit_code")
+            .and_then(Value::as_i64)
+            .and_then(|code| i32::try_from(code).ok())
+            .unwrap_or(1);
+        eprintln!("Error ({category}, exit {exit_code}): {error:#}");
+        std::process::exit(exit_code);
+    }
+}
+
+#[cfg(test)]
+mod tests;

crates/clusterflux-cli/src/node.rs

@@ -0,0 +1,837 @@
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use clusterflux_core::{
+    generate_ed25519_private_key, node_ed25519_public_key_from_private_key, sign_node_request,
+    signed_request_payload_digest, Capability, Digest, EnvironmentBackend, NodeCapabilities,
+    NodeId,
+};
+use serde::Serialize;
+use serde_json::{json, Value};
+
+use crate::client::{authenticated_or_local_trusted_request, JsonLineSession};
+use crate::config::{effective_scope_value, read_cli_session, StoredCliSession};
+use crate::tools::{command_available, command_nonce, unix_timestamp_seconds};
+use crate::{
+    confirmation_required_report, AttachArgs, CliScopeArgs, NodeEnrollArgs, NodeListArgs,
+    NodeRevokeArgs, NodeStatusArgs,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct NodeAttachPlan {
+    pub(crate) node: String,
+    pub(crate) coordinator: Option<String>,
+    pub(crate) capabilities: NodeCapabilities,
+    pub(crate) detection: NodeAttachDetectionEvidence,
+    pub(crate) grant_disclosures: Vec<CapabilityGrantDisclosure>,
+    pub(crate) enrollment: Option<NodeEnrollmentPlan>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub(crate) struct NodeAttachReport {
+    pub(crate) command: String,
+    pub(crate) node: String,
+    pub(crate) plan: NodeAttachPlan,
+    pub(crate) grant_disclosures: Vec<CapabilityGrantDisclosure>,
+    pub(crate) boundary: NodeAttachBoundaryEvidence,
+    pub(crate) coordinator_response: Value,
+    pub(crate) heartbeat_response: Value,
+    pub(crate) capability_response: Value,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct NodeAttachBoundaryEvidence {
+    pub(crate) cli_contacted_coordinator: bool,
+    pub(crate) coordinator_address: String,
+    pub(crate) used_enrollment_exchange: bool,
+    pub(crate) coordinator_session_requests: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct CapabilityGrantDisclosure {
+    pub(crate) capability: Capability,
+    pub(crate) grant: String,
+    pub(crate) description: String,
+    pub(crate) risk: String,
+    pub(crate) coordinator_policy_limited: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct NodeAttachDetectionEvidence {
+    pub(crate) auto_detected: bool,
+    pub(crate) os: clusterflux_core::Os,
+    pub(crate) arch: String,
+    pub(crate) command_backend: String,
+    pub(crate) command_backend_available: bool,
+    pub(crate) container_backend: Option<String>,
+    pub(crate) container_backend_reported: bool,
+    pub(crate) container_backend_available: bool,
+    pub(crate) source_provider_backends: Vec<SourceProviderBackendStatus>,
+    pub(crate) manual_capability_overrides_allowed: bool,
+    pub(crate) manual_capability_overrides: Vec<String>,
+    pub(crate) recognized_capability_overrides: Vec<Capability>,
+    pub(crate) unrecognized_capability_overrides: Vec<String>,
+    pub(crate) os_arch_capabilities_require_manual_flags: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct SourceProviderBackendStatus {
+    pub(crate) provider: String,
+    pub(crate) detected: bool,
+    pub(crate) available: bool,
+    pub(crate) reason: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct NodeEnrollmentPlan {
+    pub(crate) grant: String,
+    pub(crate) public_key_fingerprint: Digest,
+    pub(crate) exchanges_short_lived_grant_for_long_lived_node_identity: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+struct StoredNodeCredential {
+    kind: String,
+    node: String,
+    private_key: String,
+    public_key: String,
+    credential_scope: String,
+}
+
+pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<Value> {
+    let stored_session = read_cli_session(&cwd)?;
+    let coordinator = args.scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    if let Some(coordinator) = &coordinator {
+        let tenant = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.tenant,
+            |session| session.tenant.as_str(),
+            "tenant",
+        );
+        let project = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.project,
+            |session| session.project.as_str(),
+            "project",
+        );
+        let user = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.user,
+            |session| session.user.as_str(),
+            "user",
+        );
+        let ttl_seconds = args.ttl_seconds;
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "create_node_enrollment_grant",
+                "ttl_seconds": ttl_seconds,
+            }),
+            json!({
+                "type": "create_node_enrollment_grant",
+                "tenant": tenant.clone(),
+                "project": project.clone(),
+                "actor_user": user.clone(),
+                "ttl_seconds": ttl_seconds,
+            }),
+        )?;
+        let response = session.request(request)?;
+        let enrollment_grant =
+            node_enrollment_grant_summary(&response, &tenant, &project, &user, ttl_seconds)?;
+        return Ok(json!({
+            "command": "node enroll",
+            "status": "created",
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "private_website_required": false,
+            "enrollment_grant": enrollment_grant,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "node enroll",
+        "status": "requires_coordinator",
+        "private_website_required": false,
+        "requested_ttl_seconds": args.ttl_seconds,
+        "enrollment_grant": null,
+        "reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
+    }))
+}
+
+fn node_enrollment_grant_summary(
+    response: &Value,
+    tenant: &str,
+    project: &str,
+    user: &str,
+    ttl_seconds: u64,
+) -> Result<Value> {
+    let grant = response
+        .get("grant")
+        .and_then(Value::as_str)
+        .context("coordinator did not return its generated enrollment grant")?;
+    Ok(json!({
+        "grant": grant,
+        "tenant": response
+            .get("tenant")
+            .cloned()
+            .unwrap_or_else(|| json!(tenant)),
+        "project": response
+            .get("project")
+            .cloned()
+            .unwrap_or_else(|| json!(project)),
+        "user": user,
+        "scope": response
+            .get("scope")
+            .cloned()
+            .unwrap_or_else(|| json!("node:attach")),
+        "ttl_seconds": ttl_seconds,
+        "expires_at_epoch_seconds": response
+            .get("expires_at_epoch_seconds")
+            .cloned()
+            .unwrap_or(Value::Null),
+        "short_lived": true,
+        "exchange_for_persistent_node_identity": true,
+        "node_credentials_separate_from_user_session": true,
+    }))
+}
+
+pub(crate) fn node_list_report(args: NodeListArgs, cwd: PathBuf) -> Result<Value> {
+    node_descriptors_report("node list", args.scope, None, cwd)
+}
+
+pub(crate) fn node_status_report(args: NodeStatusArgs, cwd: PathBuf) -> Result<Value> {
+    node_descriptors_report("node status", args.scope, args.node, cwd)
+}
+
+fn node_descriptors_report(
+    command: &str,
+    scope: CliScopeArgs,
+    node: Option<String>,
+    cwd: PathBuf,
+) -> Result<Value> {
+    let stored_session = read_cli_session(&cwd)?;
+    let coordinator = scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    if let Some(coordinator) = &coordinator {
+        let tenant = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &scope.tenant,
+            |session| session.tenant.as_str(),
+            "tenant",
+        );
+        let project = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &scope.project,
+            |session| session.project.as_str(),
+            "project",
+        );
+        let user = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &scope.user,
+            |session| session.user.as_str(),
+            "user",
+        );
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "list_node_descriptors",
+            }),
+            json!({
+                "type": "list_node_descriptors",
+                "tenant": tenant.clone(),
+                "project": project.clone(),
+                "actor_user": user.clone(),
+            }),
+        )?;
+        let response = session.request(request)?;
+        return Ok(json!({
+            "command": command,
+            "coordinator": coordinator,
+            "node": node,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": command,
+        "status": "local_capability_snapshot",
+        "node": node.unwrap_or_else(default_node_id),
+        "capabilities": NodeCapabilities::detect_current(),
+    }))
+}
+
+pub(crate) fn node_revoke_report(args: NodeRevokeArgs, cwd: PathBuf) -> Result<Value> {
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "node revoke",
+            "revoke_node_credential",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "user": args.scope.user,
+                "node": args.node,
+            }),
+            format!("clusterflux node revoke --node {} --yes", args.node),
+        ));
+    }
+    let stored_session = read_cli_session(&cwd)?;
+    let coordinator = args.scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    if let Some(coordinator) = &coordinator {
+        let tenant = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.tenant,
+            |session| session.tenant.as_str(),
+            "tenant",
+        );
+        let project = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.project,
+            |session| session.project.as_str(),
+            "project",
+        );
+        let user = session_or_effective_scope_value(
+            stored_session.as_ref(),
+            &args.scope.user,
+            |session| session.user.as_str(),
+            "user",
+        );
+        let node = args.node.clone();
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "revoke_node_credential",
+                "node": node.clone(),
+            }),
+            json!({
+                "type": "revoke_node_credential",
+                "tenant": tenant.clone(),
+                "project": project.clone(),
+                "actor_user": user.clone(),
+                "node": node.clone(),
+            }),
+        )?;
+        let response = session.request(request)?;
+        return Ok(json!({
+            "command": "node revoke",
+            "coordinator": coordinator,
+            "requires_confirmation": !args.yes,
+            "tenant": tenant,
+            "project": project,
+            "user": user,
+            "node": node,
+            "credential_revoked": response.get("type").and_then(Value::as_str) == Some("node_credential_revoked"),
+            "descriptor_removed": response
+                .get("descriptor_removed")
+                .cloned()
+                .unwrap_or(json!(false)),
+            "queued_assignments_removed": response
+                .get("queued_assignments_removed")
+                .cloned()
+                .unwrap_or_else(|| json!(0)),
+            "node_credentials_separate_from_user_session": true,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "node revoke",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "node": args.node,
+    }))
+}
+
+fn session_or_effective_scope_value(
+    stored_session: Option<&StoredCliSession>,
+    cli_value: &str,
+    session_value: impl FnOnce(&StoredCliSession) -> &str,
+    default_value: &str,
+) -> String {
+    if let Some(session) = stored_session.filter(|session| session.session_secret.is_some()) {
+        session_value(session).to_owned()
+    } else {
+        effective_scope_value(cli_value, stored_session.map(session_value), default_value)
+    }
+}
+
+pub(crate) fn attach_plan(args: AttachArgs) -> NodeAttachPlan {
+    let mut capabilities = NodeCapabilities::detect_current();
+    let mut recognized_capability_overrides = Vec::new();
+    let mut unrecognized_capability_overrides = Vec::new();
+    for cap in &args.caps {
+        if let Some(parsed) = parse_capability(cap) {
+            recognized_capability_overrides.push(parsed.clone());
+            capabilities.capabilities.insert(parsed);
+        } else {
+            unrecognized_capability_overrides.push(cap.clone());
+        }
+    }
+    recognized_capability_overrides.sort();
+    recognized_capability_overrides.dedup();
+    let node = args.node.unwrap_or_else(default_node_id);
+    let public_key = args
+        .public_key
+        .unwrap_or_else(|| default_node_public_key_for_plan(&node));
+    let enrollment = args.enrollment_grant.map(|grant| NodeEnrollmentPlan {
+        grant,
+        public_key_fingerprint: Digest::sha256(public_key),
+        exchanges_short_lived_grant_for_long_lived_node_identity: true,
+    });
+    let detection = node_attach_detection_evidence(
+        &capabilities,
+        args.caps,
+        recognized_capability_overrides,
+        unrecognized_capability_overrides,
+    );
+    let grant_disclosures = capability_grant_disclosures(&capabilities);
+
+    NodeAttachPlan {
+        node,
+        coordinator: args.coordinator,
+        capabilities,
+        detection,
+        grant_disclosures,
+        enrollment,
+    }
+}
+
+fn node_attach_detection_evidence(
+    capabilities: &NodeCapabilities,
+    manual_capability_overrides: Vec<String>,
+    recognized_capability_overrides: Vec<Capability>,
+    unrecognized_capability_overrides: Vec<String>,
+) -> NodeAttachDetectionEvidence {
+    let command_backend_available = capabilities.capabilities.contains(&Capability::Command);
+    let container_backend_reported = capabilities
+        .environment_backends
+        .contains(&EnvironmentBackend::Container);
+    let container_backend = container_backend_reported.then(|| "rootless-podman".to_owned());
+    let container_backend_available = container_backend_reported
+        && capabilities
+            .capabilities
+            .contains(&Capability::RootlessPodman)
+        && command_available("podman");
+
+    NodeAttachDetectionEvidence {
+        auto_detected: true,
+        os: capabilities.os.clone(),
+        arch: capabilities.arch.clone(),
+        command_backend: if command_backend_available {
+            "native-command".to_owned()
+        } else {
+            "unavailable".to_owned()
+        },
+        command_backend_available,
+        container_backend,
+        container_backend_reported,
+        container_backend_available,
+        source_provider_backends: source_provider_backend_statuses(capabilities),
+        manual_capability_overrides_allowed: true,
+        manual_capability_overrides,
+        recognized_capability_overrides,
+        unrecognized_capability_overrides,
+        os_arch_capabilities_require_manual_flags: false,
+    }
+}
+
+fn source_provider_backend_statuses(
+    capabilities: &NodeCapabilities,
+) -> Vec<SourceProviderBackendStatus> {
+    let mut statuses = BTreeMap::new();
+    for provider in &capabilities.source_providers {
+        let available = match provider.as_str() {
+            "filesystem" => capabilities
+                .capabilities
+                .contains(&Capability::SourceFilesystem),
+            "git" => command_available("git"),
+            _ => true,
+        };
+        statuses.insert(
+            provider.clone(),
+            SourceProviderBackendStatus {
+                provider: provider.clone(),
+                detected: true,
+                available,
+                reason: if available {
+                    "detected by local node capability probe".to_owned()
+                } else {
+                    format!(
+                        "source provider `{provider}` was detected but its local helper is missing"
+                    )
+                },
+            },
+        );
+    }
+    statuses.into_values().collect()
+}
+
+fn capability_grant_disclosures(capabilities: &NodeCapabilities) -> Vec<CapabilityGrantDisclosure> {
+    let mut disclosures = Vec::new();
+    let mut push = |capability: Capability, grant: &str, description: &str, risk: &str| {
+        if capabilities.capabilities.contains(&capability) {
+            disclosures.push(CapabilityGrantDisclosure {
+                capability,
+                grant: grant.to_owned(),
+                description: description.to_owned(),
+                risk: risk.to_owned(),
+                coordinator_policy_limited: true,
+            });
+        }
+    };
+
+    push(
+        Capability::Command,
+        "native_command_execution",
+        "placed tasks may run native commands on this node",
+        "local process execution under the node account",
+    );
+    push(
+        Capability::WindowsCommandDev,
+        "native_command_execution",
+        "placed tasks may run Windows developer commands on this node",
+        "local process execution under the node account",
+    );
+    push(
+        Capability::SourceFilesystem,
+        "source_access",
+        "placed tasks may read the local project/source checkout exposed by this node",
+        "broad local source visibility",
+    );
+    push(
+        Capability::SourceGit,
+        "source_access",
+        "placed tasks may use Git-backed source access exposed by this node",
+        "source-provider visibility",
+    );
+    push(
+        Capability::Network,
+        "network_access",
+        "placed tasks may use outbound network access from this node",
+        "network egress from the node environment",
+    );
+    push(
+        Capability::HostFilesystem,
+        "host_filesystem_access",
+        "placed tasks may access configured host filesystem mounts",
+        "host file visibility outside the project checkout",
+    );
+    push(
+        Capability::Secrets,
+        "secret_access",
+        "placed tasks may receive configured secret material",
+        "secret exposure to authorized task code",
+    );
+    push(
+        Capability::InboundPorts,
+        "inbound_ports",
+        "placed tasks may expose inbound ports from this node",
+        "network service exposure from the node environment",
+    );
+    push(
+        Capability::ArbitrarySyscalls,
+        "arbitrary_syscalls",
+        "placed tasks may use broader host syscall surface",
+        "reduced host isolation",
+    );
+
+    disclosures.sort_by(|left, right| {
+        left.grant
+            .cmp(&right.grant)
+            .then_with(|| left.capability.cmp(&right.capability))
+    });
+    disclosures
+}
+
+pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport> {
+    let coordinator = args
+        .coordinator
+        .clone()
+        .context("node attach execution requires --coordinator")?;
+    let tenant = args.tenant.clone();
+    let project = args.project.clone();
+    let node = args.node.clone().unwrap_or_else(default_node_id);
+    let node_private_key = node_private_key_for_attach(&node)?;
+    let derived_public_key =
+        node_ed25519_public_key_from_private_key(&node_private_key).map_err(anyhow::Error::msg)?;
+    let public_key = args
+        .public_key
+        .clone()
+        .unwrap_or(derived_public_key.clone());
+    if public_key != derived_public_key {
+        anyhow::bail!(
+            "node attach --public-key must match CLUSTERFLUX_NODE_PRIVATE_KEY or the stored local node credential"
+        );
+    }
+    let mut plan = attach_plan(args);
+    if let Some(enrollment) = &mut plan.enrollment {
+        enrollment.public_key_fingerprint = Digest::sha256(&public_key);
+    }
+
+    let mut session = JsonLineSession::connect(&coordinator)?;
+    let used_enrollment_exchange = plan.enrollment.is_some();
+    let coordinator_response = if let Some(enrollment) = &plan.enrollment {
+        session.request(json!({
+            "type": "exchange_node_enrollment_grant",
+            "tenant": &tenant,
+            "project": &project,
+            "node": &node,
+            "public_key": &public_key,
+            "enrollment_grant": enrollment.grant,
+        }))?
+    } else {
+        session.request(json!({
+            "type": "attach_node",
+            "tenant": &tenant,
+            "project": &project,
+            "node": &node,
+            "public_key": &public_key,
+        }))?
+    };
+    let heartbeat_request = json!({
+        "type": "node_heartbeat",
+        "node": &plan.node,
+    });
+    let heartbeat_signature = sign_node_request(
+        &node_private_key,
+        &NodeId::from(plan.node.as_str()),
+        "node_heartbeat",
+        &signed_request_payload_digest(&heartbeat_request),
+        command_nonce("node-heartbeat"),
+        unix_timestamp_seconds(),
+    )
+    .map_err(anyhow::Error::msg)?;
+    let mut heartbeat_request = heartbeat_request;
+    heartbeat_request["node_signature"] = json!(heartbeat_signature);
+    let heartbeat_response = session.request(heartbeat_request)?;
+    let capability_response = session.request(signed_node_request_json(
+        &node_private_key,
+        &plan.node,
+        "report_node_capabilities",
+        json!({
+        "type": "report_node_capabilities",
+        "tenant": &tenant,
+        "project": &project,
+        "node": &plan.node,
+        "capabilities": &plan.capabilities,
+        "cached_environment_digests": [],
+        "dependency_cache_digests": [],
+        "source_snapshots": [],
+        "artifact_locations": [],
+        "direct_connectivity": true,
+        "online": false,
+        }),
+    )?)?;
+
+    Ok(NodeAttachReport {
+        command: "node attach".to_owned(),
+        node: plan.node.clone(),
+        grant_disclosures: plan.grant_disclosures.clone(),
+        plan,
+        boundary: NodeAttachBoundaryEvidence {
+            cli_contacted_coordinator: true,
+            coordinator_address: coordinator,
+            used_enrollment_exchange,
+            coordinator_session_requests: session.requests(),
+        },
+        coordinator_response,
+        heartbeat_response,
+        capability_response,
+    })
+}
+
+fn node_private_key_for_attach(node: &str) -> Result<String> {
+    if let Ok(private_key) = std::env::var("CLUSTERFLUX_NODE_PRIVATE_KEY") {
+        return Ok(private_key);
+    }
+    load_or_create_local_node_credential(&std::env::current_dir()?, node)
+}
+
+pub(crate) fn load_or_create_local_node_credential(project: &Path, node: &str) -> Result<String> {
+    let file = local_node_credential_file(project, node);
+    if credential_file_exists_without_symlink(&file)? {
+        let bytes =
+            std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
+        let credential: StoredNodeCredential = serde_json::from_slice(&bytes)
+            .with_context(|| format!("failed to parse {}", file.display()))?;
+        if credential.node != node {
+            anyhow::bail!(
+                "stored node credential {} belongs to node `{}` instead of `{}`",
+                file.display(),
+                credential.node,
+                node
+            );
+        }
+        let public_key = node_ed25519_public_key_from_private_key(&credential.private_key)
+            .map_err(anyhow::Error::msg)?;
+        if public_key != credential.public_key {
+            anyhow::bail!(
+                "stored node credential {} has a public key that does not match its private key",
+                file.display()
+            );
+        }
+        return Ok(credential.private_key);
+    }
+
+    let private_key = generate_ed25519_private_key().map_err(anyhow::Error::msg)?;
+    let public_key =
+        node_ed25519_public_key_from_private_key(&private_key).map_err(anyhow::Error::msg)?;
+    let credential = StoredNodeCredential {
+        kind: "clusterflux_node_credential".to_owned(),
+        node: node.to_owned(),
+        private_key: private_key.clone(),
+        public_key,
+        credential_scope: "local_project_node_identity".to_owned(),
+    };
+    persist_node_credential(&file, &credential)?;
+    Ok(private_key)
+}
+
+fn credential_file_exists_without_symlink(file: &Path) -> Result<bool> {
+    match std::fs::symlink_metadata(file) {
+        Ok(metadata) if metadata.file_type().is_symlink() => anyhow::bail!(
+            "refusing to read node credential through symbolic link {}",
+            file.display()
+        ),
+        Ok(metadata) if !metadata.is_file() => anyhow::bail!(
+            "node credential path {} is not a regular file",
+            file.display()
+        ),
+        Ok(_) => Ok(true),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
+        Err(error) => Err(error).with_context(|| format!("failed to inspect {}", file.display())),
+    }
+}
+
+fn persist_node_credential(file: &Path, credential: &StoredNodeCredential) -> Result<()> {
+    use std::io::Write;
+
+    let parent = file
+        .parent()
+        .with_context(|| format!("node credential path {} has no parent", file.display()))?;
+    std::fs::create_dir_all(parent)
+        .with_context(|| format!("failed to create {}", parent.display()))?;
+    if std::fs::symlink_metadata(parent)?.file_type().is_symlink() {
+        anyhow::bail!(
+            "refusing to store node credential through symbolic-link directory {}",
+            parent.display()
+        );
+    }
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
+            .with_context(|| format!("failed to secure {}", parent.display()))?;
+    }
+
+    let mut temporary = tempfile::NamedTempFile::new_in(parent).with_context(|| {
+        format!(
+            "failed to create temporary credential in {}",
+            parent.display()
+        )
+    })?;
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        temporary
+            .as_file()
+            .set_permissions(std::fs::Permissions::from_mode(0o600))?;
+    }
+    temporary.write_all(&serde_json::to_vec_pretty(credential)?)?;
+    temporary.as_file().sync_all()?;
+    temporary.persist_noclobber(file).map_err(|error| {
+        anyhow::anyhow!(
+            "refusing to overwrite node credential {}: {}",
+            file.display(),
+            error.error
+        )
+    })?;
+    Ok(())
+}
+
+pub(crate) fn local_node_credential_file(project: &Path, node: &str) -> PathBuf {
+    let digest = Digest::sha256(node);
+    let file_stem = digest.as_str().trim_start_matches("sha256:");
+    project
+        .join(".clusterflux")
+        .join("nodes")
+        .join(format!("{file_stem}.json"))
+}
+
+fn default_node_public_key_for_plan(node: &str) -> String {
+    let private_key = generate_ed25519_private_key()
+        .unwrap_or_else(|_| format!("unavailable-random-node-plan-key:{node}"));
+    node_ed25519_public_key_from_private_key(&private_key)
+        .unwrap_or_else(|_| format!("{node}-public-key"))
+}
+
+fn signed_node_request_json(
+    node_private_key: &str,
+    node: &str,
+    request_kind: &str,
+    request: Value,
+) -> Result<Value> {
+    let payload_digest = signed_request_payload_digest(&request);
+    let node_signature = sign_node_request(
+        node_private_key,
+        &NodeId::from(node),
+        request_kind,
+        &payload_digest,
+        command_nonce(request_kind),
+        unix_timestamp_seconds(),
+    )
+    .map_err(anyhow::Error::msg)?;
+    Ok(json!({
+        "type": "signed_node",
+        "node": node,
+        "node_signature": node_signature,
+        "request": request,
+    }))
+}
+
+pub(crate) fn default_node_id() -> String {
+    std::env::var("CLUSTERFLUX_NODE_ID")
+        .or_else(|_| std::env::var("HOSTNAME"))
+        .or_else(|_| std::env::var("COMPUTERNAME"))
+        .unwrap_or_else(|_| "node-local".to_owned())
+}
+
+fn parse_capability(cap: &str) -> Option<Capability> {
+    match cap {
+        "command" => Some(Capability::Command),
+        "containers" => Some(Capability::Containers),
+        "rootless-podman" => Some(Capability::RootlessPodman),
+        "source-filesystem" => Some(Capability::SourceFilesystem),
+        "source-git" => Some(Capability::SourceGit),
+        "host-filesystem" => Some(Capability::HostFilesystem),
+        "network" => Some(Capability::Network),
+        "secrets" => Some(Capability::Secrets),
+        "inbound-ports" => Some(Capability::InboundPorts),
+        "arbitrary-syscalls" => Some(Capability::ArbitrarySyscalls),
+        "vfs-artifacts" => Some(Capability::VfsArtifacts),
+        "windows-command-dev" => Some(Capability::WindowsCommandDev),
+        "quic-direct" => Some(Capability::QuicDirect),
+        _ => None,
+    }
+}

crates/clusterflux-cli/src/output.rs

@@ -0,0 +1,638 @@
+use anyhow::Result;
+use serde::Serialize;
+use serde_json::{json, Value};
+
+pub(crate) fn emit_report<T: Serialize>(report: &T, json_output: bool) -> Result<()> {
+    let mut value = serde_json::to_value(report)?;
+    let exit_code = apply_command_report_exit_code(&mut value);
+    if json_output {
+        println!("{}", serde_json::to_string_pretty(&value)?);
+    } else {
+        println!("{}", human_report(&value));
+    }
+    if let Some(exit_code) = exit_code {
+        std::process::exit(exit_code);
+    }
+    Ok(())
+}
+
+pub(crate) fn human_report(value: &Value) -> String {
+    let mut lines = Vec::new();
+    let title = value
+        .get("command")
+        .and_then(Value::as_str)
+        .map(|command| format!("Clusterflux {command}"))
+        .or_else(|| {
+            if value.get("human_flow").is_some() {
+                Some("Clusterflux login".to_owned())
+            } else if value.get("metadata").is_some()
+                && value.get("source_provider_manifest").is_some()
+            {
+                Some("Clusterflux bundle inspect".to_owned())
+            } else if value.get("entry").is_some() && value.get("session").is_some() {
+                Some("Clusterflux run".to_owned())
+            } else if value.get("capabilities").is_some() && value.get("node").is_some() {
+                Some("Clusterflux node attach".to_owned())
+            } else if value.get("public_key_fingerprint").is_some() {
+                Some("Clusterflux agent enroll".to_owned())
+            } else {
+                None
+            }
+        })
+        .unwrap_or_else(|| "Clusterflux report".to_owned());
+    lines.push(title);
+
+    push_string_field(&mut lines, value, "status", "status");
+    push_string_field(&mut lines, value, "coordinator", "coordinator");
+    push_string_field(&mut lines, value, "active_coordinator", "coordinator");
+    push_string_field(&mut lines, value, "tenant", "tenant");
+    push_string_field(&mut lines, value, "project", "project");
+    push_string_field(&mut lines, value, "process", "process");
+    push_string_field(&mut lines, value, "active_process", "active process");
+    push_string_field(&mut lines, value, "node", "node");
+    push_string_field(&mut lines, value, "artifact", "artifact");
+    push_string_field(&mut lines, value, "entry", "entry");
+    push_string_field(&mut lines, value, "quota_tier", "quota tier");
+    push_string_field(&mut lines, value, "config_file", "config");
+    push_string_field(&mut lines, value, "adapter", "adapter");
+
+    if let Some(project) = value.get("project").and_then(Value::as_str) {
+        if !lines
+            .iter()
+            .any(|line| line == &format!("project: {project}"))
+        {
+            lines.push(format!("project: {project}"));
+        }
+    }
+    if let Some(project_config) = value.get("project_config").or_else(|| value.get("project")) {
+        if project_config.is_object() {
+            push_nested_string_field(&mut lines, project_config, "tenant", "tenant");
+            push_nested_string_field(&mut lines, project_config, "project", "project");
+            push_nested_string_field(&mut lines, project_config, "coordinator", "coordinator");
+        }
+    }
+    if let Some(link) = value.get("current_directory_link") {
+        if link
+            .get("links_current_directory")
+            .and_then(Value::as_bool)
+            .unwrap_or(false)
+        {
+            lines.push("current directory linked: true".to_owned());
+        }
+        push_nested_string_field(&mut lines, link, "config_file", "current directory config");
+    }
+    if let Some(metadata) = value.get("metadata") {
+        push_nested_string_field(&mut lines, metadata, "identity", "bundle");
+        if let Some(environments) = metadata.get("environments").and_then(Value::as_array) {
+            let names = environments
+                .iter()
+                .filter_map(|env| env.get("name").and_then(Value::as_str))
+                .collect::<Vec<_>>();
+            if !names.is_empty() {
+                lines.push(format!("environments: {}", names.join(", ")));
+            }
+        }
+    }
+    if let Some(bundle) = value.get("bundle") {
+        if let Some(metadata) = bundle.get("metadata") {
+            push_nested_string_field(&mut lines, metadata, "identity", "bundle");
+        }
+    }
+    if let Some(environments) = value
+        .get("discovered_environments")
+        .and_then(Value::as_array)
+    {
+        let names = environments
+            .iter()
+            .filter_map(Value::as_str)
+            .collect::<Vec<_>>();
+        if !names.is_empty() {
+            lines.push(format!("environments: {}", names.join(", ")));
+        }
+    }
+    if let Some(attached_nodes) = value.get("attached_nodes") {
+        if let Some(count) = attached_nodes.get("count").and_then(Value::as_u64) {
+            let online = attached_nodes
+                .get("online")
+                .and_then(Value::as_u64)
+                .unwrap_or(0);
+            lines.push(format!("attached nodes: {count} ({online} online)"));
+        }
+    }
+    if let Some(current_usage) = value.pointer("/quota_posture/current_usage") {
+        lines.push(format!("quota usage: {}", compact_json(current_usage)));
+    }
+    if let Some(current_task_count) = value.get("current_task_count").and_then(Value::as_u64) {
+        lines.push(format!("tasks: {current_task_count}"));
+    }
+    if let Some(tasks) = value.get("current_tasks").and_then(Value::as_array) {
+        push_task_placement_reasons(&mut lines, tasks);
+        push_task_locality_failures(&mut lines, tasks);
+    }
+    if let Some(tasks) = value.get("tasks").and_then(Value::as_array) {
+        lines.push(format!("tasks: {}", tasks.len()));
+        push_task_placement_reasons(&mut lines, tasks);
+        push_task_locality_failures(&mut lines, tasks);
+    }
+    if let Some(log_entries) = value.get("log_entries").and_then(Value::as_array) {
+        lines.push(format!("log entries: {}", log_entries.len()));
+    }
+    if let Some(artifacts) = value.get("artifacts").and_then(Value::as_array) {
+        lines.push(format!("artifacts: {}", artifacts.len()));
+    }
+    if let Some(statuses) = value
+        .get("source_provider_statuses")
+        .and_then(Value::as_array)
+    {
+        push_source_provider_statuses(&mut lines, statuses);
+    }
+    if let Some(bundle_statuses) = value
+        .pointer("/bundle/source_provider_statuses")
+        .and_then(Value::as_array)
+    {
+        push_source_provider_statuses(&mut lines, bundle_statuses);
+    }
+    if let Some(diagnostics) = value
+        .get("pre_schedule_diagnostics")
+        .or_else(|| value.get("diagnostics"))
+        .and_then(Value::as_array)
+    {
+        push_cli_diagnostics(&mut lines, diagnostics);
+    }
+    if let Some(bundle_diagnostics) = value
+        .pointer("/bundle/pre_schedule_diagnostics")
+        .and_then(Value::as_array)
+    {
+        push_cli_diagnostics(&mut lines, bundle_diagnostics);
+    }
+    if let Some(current_usage) = value.get("current_usage") {
+        lines.push(format!("quota usage: {}", compact_json(current_usage)));
+    }
+    if let Some(next_blocked_action) = value.get("next_blocked_action") {
+        if !next_blocked_action.is_null() {
+            lines.push(format!(
+                "next blocked action: {}",
+                compact_json(next_blocked_action)
+            ));
+        }
+    }
+    push_machine_error_line(&mut lines, value.get("machine_error"));
+    push_machine_error_line(&mut lines, value.pointer("/run_start/machine_error"));
+    push_machine_error_line(&mut lines, value.pointer("/download_session/machine_error"));
+    push_machine_error_line(&mut lines, value.pointer("/export_plan/machine_error"));
+    push_machine_error_line(&mut lines, value.pointer("/local_export/machine_error"));
+    push_machine_error_line(
+        &mut lines,
+        value.pointer("/local_export/download_session/machine_error"),
+    );
+    push_machine_error_line(
+        &mut lines,
+        value.pointer("/local_export/stream/machine_error"),
+    );
+    if let Some(flow) = value.get("human_flow") {
+        if let Some(browser) = flow.get("Browser") {
+            lines.push("flow: browser".to_owned());
+            push_nested_string_field(&mut lines, browser, "authorization_url", "open");
+        }
+    }
+    if let Some(session) = value.get("session") {
+        lines.push(format!("session: {}", compact_json(session)));
+    }
+    if let Some(account) = value.get("coordinator_account_status") {
+        if let Some(checked) = account.get("checked").and_then(Value::as_bool) {
+            lines.push(format!("account status checked: {checked}"));
+        }
+        push_nested_string_field(&mut lines, account, "account_status", "account status");
+        if let Some(suspended) = account.get("suspended").and_then(Value::as_bool) {
+            lines.push(format!("account suspended: {suspended}"));
+        }
+        if let Some(disabled) = account.get("disabled").and_then(Value::as_bool) {
+            lines.push(format!("account disabled: {disabled}"));
+        }
+        if let Some(deleted) = account.get("deleted").and_then(Value::as_bool) {
+            lines.push(format!("account deleted: {deleted}"));
+        }
+        if let Some(manual_review) = account.get("manual_review").and_then(Value::as_bool) {
+            lines.push(format!("account manual review: {manual_review}"));
+        }
+        push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
+        if let Some(exposed) = account
+            .get("private_moderation_details_exposed")
+            .and_then(Value::as_bool)
+        {
+            lines.push(format!("private moderation details exposed: {exposed}"));
+        }
+    }
+    if let Some(coordinator_selection) = value.get("coordinator") {
+        if coordinator_selection.is_object() {
+            lines.push(format!(
+                "coordinator: {}",
+                compact_json(coordinator_selection)
+            ));
+        }
+    }
+    if let Some(reachability) = value.get("coordinator_reachability") {
+        if let Some(status) = reachability.get("status").and_then(Value::as_str) {
+            lines.push(format!("coordinator reachability: {status}"));
+        }
+        if let Some(error) = reachability.get("error").and_then(Value::as_str) {
+            lines.push(format!("coordinator error: {error}"));
+        }
+        if let Some(response_type) = reachability
+            .pointer("/response/type")
+            .and_then(Value::as_str)
+        {
+            lines.push(format!("coordinator ping: {response_type}"));
+        }
+    }
+    if let Some(response) = value
+        .get("response")
+        .or_else(|| value.get("coordinator_response"))
+    {
+        if let Some(response_type) = response.get("type").and_then(Value::as_str) {
+            lines.push(format!("coordinator response: {response_type}"));
+        }
+    }
+    if let Some(events) = value
+        .pointer("/events/response/events")
+        .and_then(Value::as_array)
+    {
+        lines.push(format!("events: {}", events.len()));
+    }
+    if let Some(events) = value
+        .pointer("/coordinator_response/response/events")
+        .and_then(Value::as_array)
+    {
+        lines.push(format!("events: {}", events.len()));
+    }
+    if let Some(requires_confirmation) = value.get("requires_confirmation").and_then(Value::as_bool)
+    {
+        lines.push(format!(
+            "confirmation: {}",
+            if requires_confirmation {
+                "required"
+            } else {
+                "confirmed"
+            }
+        ));
+    }
+    if let Some(flag) = value
+        .get("private_website_required")
+        .and_then(Value::as_bool)
+    {
+        lines.push(format!("private website required: {flag}"));
+    }
+    if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
+        let actions = next_actions
+            .iter()
+            .filter_map(Value::as_str)
+            .collect::<Vec<_>>();
+        if !actions.is_empty() {
+            lines.push(format!("next: {}", actions.join("; ")));
+        }
+    }
+    if let Some(auth) = value.get("auth").or_else(|| value.get("session")) {
+        if let Some(kind) = auth.get("kind").and_then(Value::as_str) {
+            lines.push(format!("auth: {kind}"));
+        }
+        if let Some(authenticated) = auth.get("authenticated").and_then(Value::as_bool) {
+            lines.push(format!("authenticated: {authenticated}"));
+        }
+        if let Some(expires) = auth.get("expires_at").and_then(Value::as_str) {
+            lines.push(format!("token expiry: {expires}"));
+        } else if let Some(posture) = auth.get("token_expiry_posture").and_then(Value::as_str) {
+            lines.push(format!("token expiry: {posture}"));
+        } else if auth.get("authenticated").is_some() {
+            lines.push("token expiry: unavailable".to_owned());
+        }
+    }
+    if let Some(dependencies) = value.get("dependencies").and_then(Value::as_object) {
+        let mut entries = dependencies
+            .iter()
+            .map(|(name, available)| {
+                format!(
+                    "{name}={}",
+                    if available.as_bool().unwrap_or(false) {
+                        "ok"
+                    } else {
+                        "missing"
+                    }
+                )
+            })
+            .collect::<Vec<_>>();
+        entries.sort();
+        if !entries.is_empty() {
+            lines.push(format!("dependencies: {}", entries.join(", ")));
+        }
+    }
+    if let Some(readiness) = value.get("node_readiness") {
+        push_nested_string_field(&mut lines, readiness, "os", "node os");
+        push_nested_string_field(&mut lines, readiness, "arch", "node arch");
+        if let Some(capabilities) = readiness.get("capabilities").and_then(Value::as_array) {
+            let caps = capabilities
+                .iter()
+                .filter_map(Value::as_str)
+                .collect::<Vec<_>>();
+            if !caps.is_empty() {
+                lines.push(format!("node capabilities: {}", caps.join(", ")));
+            }
+        }
+    }
+    if let Some(summary) = value.get("node_readiness_summary") {
+        push_nested_string_field(&mut lines, summary, "status", "node readiness");
+        if let Some(missing) = summary
+            .get("missing_local_dependencies")
+            .and_then(Value::as_array)
+        {
+            let missing = missing.iter().filter_map(Value::as_str).collect::<Vec<_>>();
+            if !missing.is_empty() {
+                lines.push(format!("node missing dependencies: {}", missing.join(", ")));
+            }
+        }
+        if let Some(actions) = summary.get("next_actions").and_then(Value::as_array) {
+            let actions = actions.iter().filter_map(Value::as_str).collect::<Vec<_>>();
+            if !actions.is_empty() {
+                lines.push(format!("node next: {}", actions.join("; ")));
+            }
+        }
+    }
+    if let Some(detection) = value
+        .pointer("/plan/detection")
+        .or_else(|| value.get("detection"))
+    {
+        push_node_attach_detection(&mut lines, detection);
+    }
+    if let Some(disclosures) = value
+        .get("grant_disclosures")
+        .and_then(Value::as_array)
+        .filter(|disclosures| !disclosures.is_empty())
+    {
+        for disclosure in disclosures {
+            let grant = disclosure
+                .get("grant")
+                .and_then(Value::as_str)
+                .unwrap_or("capability");
+            let description = disclosure
+                .get("description")
+                .and_then(Value::as_str)
+                .unwrap_or("capability grant");
+            let policy = if disclosure
+                .get("coordinator_policy_limited")
+                .and_then(Value::as_bool)
+                .unwrap_or(false)
+            {
+                "policy-limited"
+            } else {
+                "unbounded"
+            };
+            lines.push(format!("grant {grant}: {description} ({policy})"));
+        }
+    }
+
+    lines.dedup();
+    lines.join("\n")
+}
+
+fn push_node_attach_detection(lines: &mut Vec<String>, detection: &Value) {
+    push_nested_string_field(lines, detection, "os", "node os");
+    push_nested_string_field(lines, detection, "arch", "node arch");
+    push_nested_string_field(lines, detection, "command_backend", "command backend");
+    if let Some(backend) = detection.get("container_backend").and_then(Value::as_str) {
+        let available = detection
+            .get("container_backend_available")
+            .and_then(Value::as_bool)
+            .unwrap_or(false);
+        lines.push(format!(
+            "container backend: {backend} ({})",
+            if available { "available" } else { "reported" }
+        ));
+    } else if detection
+        .get("container_backend_reported")
+        .and_then(Value::as_bool)
+        .unwrap_or(false)
+    {
+        lines.push("container backend: reported".to_owned());
+    }
+    if let Some(providers) = detection
+        .get("source_provider_backends")
+        .and_then(Value::as_array)
+    {
+        let entries = providers
+            .iter()
+            .filter_map(|provider| {
+                let name = provider.get("provider").and_then(Value::as_str)?;
+                let state = if provider
+                    .get("available")
+                    .and_then(Value::as_bool)
+                    .unwrap_or(false)
+                {
+                    "available"
+                } else {
+                    "detected"
+                };
+                Some(format!("{name}={state}"))
+            })
+            .collect::<Vec<_>>();
+        if !entries.is_empty() {
+            lines.push(format!("source providers: {}", entries.join(", ")));
+        }
+    }
+    if let Some(overrides) = detection
+        .get("manual_capability_overrides")
+        .and_then(Value::as_array)
+    {
+        let overrides = overrides
+            .iter()
+            .filter_map(Value::as_str)
+            .collect::<Vec<_>>();
+        if !overrides.is_empty() {
+            lines.push(format!("capability overrides: {}", overrides.join(", ")));
+        }
+    }
+}
+
+fn push_string_field(lines: &mut Vec<String>, value: &Value, key: &str, label: &str) {
+    if let Some(text) = value.get(key).and_then(Value::as_str) {
+        lines.push(format!("{label}: {text}"));
+    } else if let Some(path) = value.get(key).and_then(|value| {
+        value
+            .as_object()
+            .and_then(|object| object.get("display"))
+            .and_then(Value::as_str)
+    }) {
+        lines.push(format!("{label}: {path}"));
+    }
+}
+
+fn push_task_placement_reasons(lines: &mut Vec<String>, tasks: &[Value]) {
+    for task in tasks {
+        let Some(placement) = task.get("node_placement") else {
+            continue;
+        };
+        let Some(reasons) = placement.get("reasons").and_then(Value::as_array) else {
+            continue;
+        };
+        let reasons = reasons.iter().filter_map(Value::as_str).collect::<Vec<_>>();
+        if reasons.is_empty() {
+            continue;
+        }
+        let task_name = task
+            .get("task")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        let node = placement
+            .get("node")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        lines.push(format!(
+            "placement {task_name}: {node} ({})",
+            reasons.join(", ")
+        ));
+    }
+}
+
+fn push_task_locality_failures(lines: &mut Vec<String>, tasks: &[Value]) {
+    for task in tasks {
+        let Some(locality) = task.get("locality_failure") else {
+            continue;
+        };
+        if locality.is_null() {
+            continue;
+        }
+        let task_name = task
+            .get("task")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        let affected = locality
+            .get("affected_data")
+            .and_then(Value::as_str)
+            .unwrap_or("direct_transfer");
+        let reason = locality
+            .get("reason")
+            .and_then(Value::as_str)
+            .unwrap_or("direct transfer or locality failed");
+        lines.push(format!("locality {task_name}: {affected} ({reason})"));
+        let actions = locality
+            .get("safe_next_actions")
+            .and_then(Value::as_array)
+            .map(|actions| actions.iter().filter_map(Value::as_str).collect::<Vec<_>>())
+            .unwrap_or_default();
+        if !actions.is_empty() {
+            lines.push(format!("locality next {task_name}: {}", actions.join("; ")));
+        }
+    }
+}
+
+fn push_nested_string_field(lines: &mut Vec<String>, value: &Value, key: &str, label: &str) {
+    if let Some(text) = value.get(key).and_then(Value::as_str) {
+        lines.push(format!("{label}: {text}"));
+    }
+}
+
+fn push_machine_error_line(lines: &mut Vec<String>, machine_error: Option<&Value>) {
+    let Some(machine_error) = machine_error else {
+        return;
+    };
+    if machine_error.is_null() {
+        return;
+    }
+    let category = machine_error
+        .get("category")
+        .and_then(Value::as_str)
+        .unwrap_or("unknown");
+    let exit_code = machine_error
+        .get("stable_exit_code")
+        .and_then(Value::as_i64)
+        .unwrap_or(1);
+    lines.push(format!("error category: {category} (exit {exit_code})"));
+    if let Some(tier) = machine_error
+        .get("community_tier_label")
+        .and_then(Value::as_str)
+    {
+        lines.push(format!("quota tier: {tier}"));
+    }
+    if let Some(next_actions) = machine_error.get("next_actions").and_then(Value::as_array) {
+        let actions = next_actions
+            .iter()
+            .filter_map(Value::as_str)
+            .collect::<Vec<_>>();
+        if !actions.is_empty() {
+            lines.push(format!("error next: {}", actions.join("; ")));
+        }
+    }
+}
+
+fn push_source_provider_statuses(lines: &mut Vec<String>, statuses: &[Value]) {
+    let entries = statuses
+        .iter()
+        .filter_map(|status| {
+            let provider = status.get("provider").and_then(Value::as_str)?;
+            let state = status.get("status").and_then(Value::as_str)?;
+            let active = status
+                .get("active")
+                .and_then(Value::as_bool)
+                .unwrap_or(false);
+            Some(format!(
+                "{provider}={state}{}",
+                if active { "(active)" } else { "" }
+            ))
+        })
+        .collect::<Vec<_>>();
+    if !entries.is_empty() {
+        lines.push(format!("source providers: {}", entries.join(", ")));
+    }
+}
+
+fn push_cli_diagnostics(lines: &mut Vec<String>, diagnostics: &[Value]) {
+    if diagnostics.is_empty() {
+        return;
+    }
+    lines.push(format!("diagnostics: {}", diagnostics.len()));
+    for diagnostic in diagnostics.iter().take(3) {
+        let severity = diagnostic
+            .get("severity")
+            .and_then(Value::as_str)
+            .unwrap_or("info");
+        let message = diagnostic
+            .get("message")
+            .and_then(Value::as_str)
+            .unwrap_or("diagnostic");
+        lines.push(format!("diagnostic {severity}: {message}"));
+    }
+}
+
+fn compact_json(value: &Value) -> String {
+    serde_json::to_string(value).unwrap_or_else(|_| "<unprintable>".to_owned())
+}
+
+pub(crate) fn apply_command_report_exit_code(value: &mut Value) -> Option<i32> {
+    for pointer in [
+        "/machine_error",
+        "/run_start/machine_error",
+        "/restart_request/machine_error",
+        "/cancel_request/machine_error",
+        "/task_restart/machine_error",
+        "/download_session/machine_error",
+        "/export_plan/machine_error",
+        "/local_export/machine_error",
+        "/local_export/download_session/machine_error",
+        "/local_export/stream/machine_error",
+    ] {
+        let Some(machine_error) = value.pointer_mut(pointer) else {
+            continue;
+        };
+        let Some(exit_code) = machine_error
+            .get("stable_exit_code")
+            .and_then(Value::as_i64)
+            .and_then(|code| i32::try_from(code).ok())
+            .filter(|code| *code != 0)
+        else {
+            continue;
+        };
+        if let Some(object) = machine_error.as_object_mut() {
+            object.insert("process_exit_code_applied".to_owned(), json!(true));
+        }
+        return Some(exit_code);
+    }
+    None
+}

crates/clusterflux-cli/src/process.rs

@@ -0,0 +1,339 @@
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
+    stored_session_for_coordinator, JsonLineSession,
+};
+use crate::config::StoredCliSession;
+use crate::process_events::{
+    process_cancel_request_summary, process_restart_request_summary, process_state_from_tasks,
+    task_summaries,
+};
+use crate::{
+    confirmation_required_report, CliScopeArgs, ProcessAbortArgs, ProcessCancelArgs,
+    ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
+};
+
+fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
+    if scope.coordinator.is_none() {
+        scope.coordinator = stored_session
+            .filter(|session| session.session_secret.is_some())
+            .map(|session| session.coordinator.clone());
+    }
+    if let Some(bound_session) = scope
+        .coordinator
+        .as_deref()
+        .and_then(|coordinator| stored_session_for_coordinator(coordinator, stored_session))
+    {
+        scope.tenant = bound_session.tenant.clone();
+        scope.project = bound_session.project.clone();
+        scope.user = bound_session.user.clone();
+    }
+}
+
+pub(crate) fn process_list_report_with_session(
+    mut args: ProcessListArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
+    let Some(coordinator) = &args.scope.coordinator else {
+        return Ok(json!({
+            "command": "process list",
+            "status": "requires_coordinator",
+            "processes": [],
+        }));
+    };
+    let mut session = JsonLineSession::connect(coordinator)?;
+    let response = session.request(authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({ "type": "list_processes" }),
+        json!({
+            "type": "list_processes",
+            "tenant": args.scope.tenant,
+            "project": args.scope.project,
+            "actor_user": args.scope.user,
+        }),
+    )?)?;
+    let processes = response
+        .get("processes")
+        .cloned()
+        .unwrap_or_else(|| json!([]));
+    Ok(json!({
+        "command": "process list",
+        "status": "ok",
+        "coordinator": coordinator,
+        "tenant": args.scope.tenant,
+        "project": args.scope.project,
+        "user": args.scope.user,
+        "processes": processes,
+        "response": response,
+        "coordinator_session_requests": session.requests(),
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn process_status_report(args: ProcessStatusArgs) -> Result<Value> {
+    process_status_report_with_session(args, None)
+}
+
+pub(crate) fn process_status_report_with_session(
+    mut args: ProcessStatusArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
+    let live = process_list_report_with_session(
+        ProcessListArgs {
+            scope: args.scope.clone(),
+        },
+        stored_session,
+    )?;
+    let live_process = live
+        .get("processes")
+        .and_then(Value::as_array)
+        .and_then(|processes| {
+            processes.iter().find(|process| {
+                process.get("process").and_then(Value::as_str) == Some(args.process.as_str())
+            })
+        })
+        .cloned();
+    let events = list_task_events_if_available_with_session(
+        args.scope.coordinator.as_deref(),
+        &args.scope,
+        Some(args.process.clone()),
+        stored_session,
+    )?;
+    let current_tasks = task_summaries(events.as_ref());
+    let current_task_count = current_tasks.as_array().map(Vec::len).unwrap_or(0);
+    let historical_state = process_state_from_tasks(events.as_ref());
+    let state = live_process
+        .as_ref()
+        .and_then(|process| process.get("state"))
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .unwrap_or_else(|| {
+            if live.get("status").and_then(Value::as_str) == Some("ok") {
+                "not_active".to_owned()
+            } else if events.is_some() {
+                historical_state.to_owned()
+            } else {
+                "unknown_without_coordinator".to_owned()
+            }
+        });
+    Ok(json!({
+        "command": "process status",
+        "process": args.process,
+        "state": state,
+        "live_process": live_process,
+        "started_entrypoint": "unknown_from_task_events",
+        "current_task_count": current_task_count,
+        "current_tasks": current_tasks,
+        "debug_state": {
+            "frozen": null,
+            "status": "unknown_from_cli_task_events"
+        },
+        "events": events,
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn process_restart_report(args: ProcessRestartArgs) -> Result<Value> {
+    process_restart_report_with_session(args, None)
+}
+
+pub(crate) fn process_restart_report_with_session(
+    mut args: ProcessRestartArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "process restart",
+            "restart_virtual_process",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "process": args.process,
+            }),
+            format!(
+                "clusterflux process restart --process {} --yes",
+                args.process
+            ),
+        ));
+    }
+    if let Some(coordinator) = &args.scope.coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "start_process",
+                "process": args.process,
+                "restart": true,
+            }),
+            json!({
+                "type": "start_process",
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "process": args.process,
+                "restart": true,
+            }),
+        )?)?;
+        let restart_request = process_restart_request_summary(&response, !args.yes);
+        return Ok(json!({
+            "command": "process restart",
+            "coordinator": coordinator,
+            "process": args.process,
+            "requires_confirmation": !args.yes,
+            "restart_request": restart_request,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "process restart",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "process": args.process,
+        "restart_request": {
+            "status": "requires_coordinator",
+            "operation": "restart_virtual_process",
+            "explicit_user_action": true,
+        },
+    }))
+}
+
+pub(crate) fn process_cancel_report(args: ProcessCancelArgs) -> Result<Value> {
+    process_cancel_report_with_session(args, None)
+}
+
+pub(crate) fn process_cancel_report_with_session(
+    mut args: ProcessCancelArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "process cancel",
+            "cancel_virtual_process",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "process": args.process,
+                "node": args.node,
+                "task": args.task,
+            }),
+            format!(
+                "clusterflux process cancel --process {} --yes",
+                args.process
+            ),
+        ));
+    }
+    if let Some(coordinator) = &args.scope.coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "cancel_process",
+                "process": args.process,
+            }),
+            json!({
+                "type": "cancel_process",
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "actor_user": args.scope.user,
+                "process": args.process,
+            }),
+        )?)?;
+        let cancel_request = process_cancel_request_summary(&response, !args.yes);
+        return Ok(json!({
+            "command": "process cancel",
+            "coordinator": coordinator,
+            "requires_confirmation": !args.yes,
+            "process": args.process,
+            "node": args.node,
+            "task": args.task,
+            "cancel_request": cancel_request,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "process cancel",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "process": args.process,
+        "node": args.node,
+        "task": args.task,
+        "cancel_request": {
+            "status": "requires_coordinator",
+            "operation": "cancel_virtual_process",
+            "whole_process_cancel_available": true,
+            "explicit_user_action": true,
+        },
+    }))
+}
+
+pub(crate) fn process_abort_report_with_session(
+    mut args: ProcessAbortArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    hydrate_process_scope(&mut args.scope, stored_session);
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "process abort",
+            "abort_virtual_process",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "process": args.process,
+            }),
+            format!("clusterflux process abort --process {} --yes", args.process),
+        ));
+    }
+    let Some(coordinator) = &args.scope.coordinator else {
+        return Ok(json!({
+            "command": "process abort",
+            "status": "requires_coordinator",
+            "process": args.process,
+            "requires_confirmation": false,
+        }));
+    };
+    let mut session = JsonLineSession::connect(coordinator)?;
+    let response = session.request(authenticated_or_local_trusted_request(
+        coordinator,
+        stored_session,
+        json!({
+            "type": "abort_process",
+            "process": args.process,
+        }),
+        json!({
+            "type": "abort_process",
+            "tenant": args.scope.tenant,
+            "project": args.scope.project,
+            "actor_user": args.scope.user,
+            "process": args.process,
+        }),
+    )?)?;
+    Ok(json!({
+        "command": "process abort",
+        "status": "aborted",
+        "coordinator": coordinator,
+        "process": args.process,
+        "requires_confirmation": false,
+        "abort_request": {
+            "accepted": response.get("type").and_then(Value::as_str) == Some("process_aborted"),
+            "operation": "abort_virtual_process",
+            "forced": true,
+            "cooperative": false,
+            "process_slot_released": true,
+        },
+        "response": response,
+        "coordinator_session_requests": session.requests(),
+    }))
+}

crates/clusterflux-cli/src/process_events.rs

@@ -0,0 +1,666 @@
+use std::path::Path;
+
+use serde_json::{json, Value};
+
+use crate::errors::{
+    cli_error_summary, cli_error_summary_for_category, cli_error_summary_with_default,
+    message_mentions_locality_failure,
+};
+
+fn task_event_values(task_events: Option<&Value>) -> Vec<&Value> {
+    task_events
+        .and_then(|task_events| task_events.pointer("/response/events"))
+        .and_then(Value::as_array)
+        .map(|events| events.iter().collect())
+        .unwrap_or_default()
+}
+
+fn event_string(event: &Value, field: &str) -> Option<String> {
+    event.get(field).and_then(Value::as_str).map(str::to_owned)
+}
+
+fn event_u64(event: &Value, field: &str) -> Option<u64> {
+    event.get(field).and_then(Value::as_u64)
+}
+
+pub(crate) fn task_summaries(task_events: Option<&Value>) -> Value {
+    Value::Array(
+        task_event_values(task_events)
+            .into_iter()
+            .map(|event| {
+                let task = event_string(event, "task").unwrap_or_else(|| "unknown".to_owned());
+                let terminal_state =
+                    event_string(event, "terminal_state").unwrap_or_else(|| "unknown".to_owned());
+                let node = event_string(event, "node");
+                let placement = event.get("placement");
+                let placement_reasons = placement
+                    .and_then(|placement| placement.get("reasons"))
+                    .cloned()
+                    .unwrap_or_else(|| json!([]));
+                let placement_score = placement
+                    .and_then(|placement| placement.get("score"))
+                    .cloned()
+                    .unwrap_or(Value::Null);
+                let failure_reason = task_failure_reason(event);
+                let locality_failure = task_locality_failure_from_reason(&failure_reason);
+                let machine_error = task_failure_machine_error_from_reason(event, &failure_reason);
+                json!({
+                    "process": event_string(event, "process"),
+                    "task": task,
+                    "state": terminal_state,
+                    "environment": event.get("environment").cloned().unwrap_or_else(|| json!("unknown_from_task_event")),
+                    "environment_digest": event.get("environment_digest").cloned().unwrap_or(Value::Null),
+                    "node_placement": {
+                        "node": node,
+                        "source": "coordinator_task_event",
+                        "score": placement_score,
+                        "reasons": placement_reasons,
+                        "explanation_available": placement.is_some(),
+                    },
+                    "failure_reason": failure_reason,
+                    "locality_failure": locality_failure,
+                    "machine_error": machine_error,
+                    "stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
+                    "stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
+                })
+            })
+            .collect(),
+    )
+}
+
+fn task_failure_reason(event: &Value) -> Value {
+    match event.get("terminal_state").and_then(Value::as_str) {
+        Some("failed") => {
+            if let Some(stderr) = event.get("stderr_tail").and_then(Value::as_str) {
+                if !stderr.is_empty() {
+                    return json!(redact_secret_like_text(stderr).0);
+                }
+            }
+            if let Some(status_code) = event.get("status_code").and_then(Value::as_i64) {
+                return json!(format!("task exited with status {status_code}"));
+            }
+            json!("task failed")
+        }
+        Some("cancelled") => json!("task cancelled"),
+        _ => Value::Null,
+    }
+}
+
+fn task_failure_machine_error_from_reason(event: &Value, reason: &Value) -> Value {
+    match event.get("terminal_state").and_then(Value::as_str) {
+        Some("failed") => {
+            let reason = reason.as_str().unwrap_or("task failed").to_owned();
+            let mut summary = cli_error_summary_with_default(&reason, "program");
+            if let Some(object) = summary.as_object_mut() {
+                if message_mentions_locality_failure(&reason.to_ascii_lowercase()) {
+                    object.insert("locality_failure".to_owned(), json!(true));
+                    object.insert(
+                        "next_actions".to_owned(),
+                        json!(locality_failure_next_actions(&reason)),
+                    );
+                }
+            }
+            summary
+        }
+        Some("cancelled") => cli_error_summary_for_category("program", "task cancelled"),
+        _ => Value::Null,
+    }
+}
+
+fn task_locality_failure_from_reason(reason: &Value) -> Value {
+    let Some(reason) = reason.as_str() else {
+        return Value::Null;
+    };
+    let lower = reason.to_ascii_lowercase();
+    if !message_mentions_locality_failure(&lower) {
+        return Value::Null;
+    }
+    let affected_data = if lower.contains("source snapshot") {
+        "source_snapshot"
+    } else if lower.contains("artifact") {
+        "artifact"
+    } else {
+        "direct_transfer"
+    };
+    json!({
+        "category": "connectivity",
+        "affected_data": affected_data,
+        "reason": reason,
+        "coordinator_bulk_relay_used": false,
+        "safe_failure": true,
+        "safe_next_actions": locality_failure_next_actions(reason),
+    })
+}
+
+fn locality_failure_next_actions(reason: &str) -> Vec<&'static str> {
+    let lower = reason.to_ascii_lowercase();
+    if lower.contains("source snapshot") {
+        return vec![
+            "attach or select a node that already has the required source snapshot",
+            "rerun source preparation on an attached node",
+            "restore direct node-to-node connectivity and retry",
+            "do not rely on coordinator bulk source relay",
+        ];
+    }
+    if lower.contains("artifact") {
+        return vec![
+            "attach or select a node that already has the required artifact",
+            "explicitly export or download the artifact before retrying",
+            "restore direct node-to-node connectivity and retry",
+            "do not rely on coordinator bulk artifact relay",
+        ];
+    }
+    vec![
+        "check node direct connectivity and NAT traversal",
+        "attach a node with the needed source/artifact locality",
+        "retry after node connectivity is restored",
+        "do not rely on coordinator bulk relay",
+    ]
+}
+
+pub(crate) fn process_state_from_tasks(task_events: Option<&Value>) -> &'static str {
+    let events = task_event_values(task_events);
+    if events.is_empty() {
+        return "no_tasks_observed";
+    }
+    if events.iter().any(|event| {
+        event
+            .get("terminal_state")
+            .and_then(Value::as_str)
+            .is_some_and(|state| state == "failed")
+    }) {
+        return "has_failed_tasks";
+    }
+    if events.iter().any(|event| {
+        event
+            .get("terminal_state")
+            .and_then(Value::as_str)
+            .is_some_and(|state| state == "cancelled")
+    }) {
+        return "has_cancelled_tasks";
+    }
+    if events.iter().all(|event| {
+        event
+            .get("terminal_state")
+            .and_then(Value::as_str)
+            .is_some_and(|state| state == "completed")
+    }) {
+        return "completed_tasks_observed";
+    }
+    "tasks_observed"
+}
+
+pub(crate) fn log_entries(task_events: Option<&Value>, task_filter: Option<&str>) -> Value {
+    Value::Array(
+        task_event_values(task_events)
+            .into_iter()
+            .filter(|event| {
+                task_filter.is_none_or( |task_filter| {
+                    event
+                        .get("task")
+                        .and_then(Value::as_str)
+                        .is_some_and(|task| task == task_filter)
+                })
+            })
+            .map(|event| {
+                let stdout_tail = event
+                    .get("stdout_tail")
+                    .and_then(Value::as_str)
+                    .unwrap_or("");
+                let stderr_tail = event
+                    .get("stderr_tail")
+                    .and_then(Value::as_str)
+                    .unwrap_or("");
+                let (stdout_tail, stdout_tail_redacted) = redact_secret_like_text(stdout_tail);
+                let (stderr_tail, stderr_tail_redacted) = redact_secret_like_text(stderr_tail);
+                json!({
+                    "process": event_string(event, "process"),
+                    "task": event_string(event, "task"),
+                    "node": event_string(event, "node"),
+                    "stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
+                    "stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
+                    "stdout_tail": stdout_tail,
+                    "stderr_tail": stderr_tail,
+                    "stdout_truncated": event.get("stdout_truncated").and_then(Value::as_bool).unwrap_or(false),
+                    "stderr_truncated": event.get("stderr_truncated").and_then(Value::as_bool).unwrap_or(false),
+                    "capped": true,
+                    "secret_like_values_redacted": stdout_tail_redacted || stderr_tail_redacted,
+                    "redacted_fields": redacted_log_fields(stdout_tail_redacted, stderr_tail_redacted),
+                })
+            })
+            .collect(),
+    )
+}
+
+fn redacted_log_fields(
+    stdout_tail_redacted: bool,
+    stderr_tail_redacted: bool,
+) -> Vec<&'static str> {
+    let mut fields = Vec::new();
+    if stdout_tail_redacted {
+        fields.push("stdout_tail");
+    }
+    if stderr_tail_redacted {
+        fields.push("stderr_tail");
+    }
+    fields
+}
+
+fn redact_secret_like_text(text: &str) -> (String, bool) {
+    let markers = [
+        "access_token=",
+        "access_token:",
+        "refresh_token=",
+        "refresh_token:",
+        "id_token=",
+        "id_token:",
+        "api_key=",
+        "api_key:",
+        "api-key=",
+        "api-key:",
+        "token=",
+        "token:",
+        "secret=",
+        "secret:",
+        "password=",
+        "password:",
+        "passwd=",
+        "passwd:",
+        "bearer ",
+    ];
+    let mut output = text.to_owned();
+    let mut redacted = false;
+    for marker in markers {
+        let (updated, changed) = redact_marker_values(output, marker);
+        output = updated;
+        redacted |= changed;
+    }
+    (output, redacted)
+}
+
+fn redact_marker_values(mut text: String, marker: &str) -> (String, bool) {
+    let mut changed = false;
+    let mut search_start = 0;
+    loop {
+        let lower = text.to_ascii_lowercase();
+        let Some(relative) = lower[search_start..].find(marker) else {
+            break;
+        };
+        let value_start = search_start + relative + marker.len();
+        let value_end = text[value_start..]
+            .char_indices()
+            .find_map(|(offset, character)| {
+                (character.is_whitespace()
+                    || matches!(
+                        character,
+                        '&' | '"' | '\'' | '`' | '<' | '>' | ',' | ';' | ')' | ']'
+                    ))
+                .then_some(value_start + offset)
+            })
+            .unwrap_or(text.len());
+        if value_start == value_end {
+            search_start = value_end;
+            if search_start >= text.len() {
+                break;
+            }
+            continue;
+        }
+        if text[value_start..value_end].starts_with("[redacted") {
+            search_start = value_end;
+            if search_start >= text.len() {
+                break;
+            }
+            continue;
+        }
+        text.replace_range(value_start..value_end, "[redacted]");
+        changed = true;
+        search_start = value_start + "[redacted]".len();
+        if search_start >= text.len() {
+            break;
+        }
+    }
+    (text, changed)
+}
+
+pub(crate) fn artifact_summaries(task_events: Option<&Value>) -> Value {
+    Value::Array(
+        task_event_values(task_events)
+            .into_iter()
+            .filter_map(|event| {
+                let path = event.get("artifact_path").and_then(Value::as_str)?;
+                let node = event_string(event, "node");
+                Some(json!({
+                    "artifact": artifact_name_from_path(path),
+                    "path": path,
+                    "producer_task": event_string(event, "task"),
+                    "producer_node": node,
+                    "process": event_string(event, "process"),
+                    "digest": event.get("artifact_digest").cloned().unwrap_or(Value::Null),
+                    "size_bytes": event.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
+                    "state": if event.get("artifact_digest").is_some() { "metadata_flushed" } else { "metadata_without_digest" },
+                    "known_locations": node.into_iter().collect::<Vec<_>>(),
+                    "durable_storage": false,
+                }))
+            })
+            .collect(),
+    )
+}
+
+fn artifact_name_from_path(path: &str) -> String {
+    path.rsplit('/').next().unwrap_or(path).to_owned()
+}
+
+fn response_error_message(response: &Value, fallback: &str) -> String {
+    response
+        .get("message")
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .unwrap_or_else(|| fallback.to_owned())
+}
+
+pub(crate) fn artifact_response_machine_error(
+    response: &Value,
+    fallback: &str,
+    default_category: &'static str,
+) -> Value {
+    let message = response_error_message(response, fallback);
+    cli_error_summary_with_default(&message, default_category)
+}
+
+pub(crate) fn process_restart_request_summary(
+    response: &Value,
+    requires_confirmation: bool,
+) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("process_started") {
+        let message = response_error_message(response, "coordinator rejected process restart");
+        return json!({
+            "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+            "operation": "restart_virtual_process",
+            "accepted": false,
+            "requires_confirmation": requires_confirmation,
+            "explicit_user_action": true,
+            "error": message,
+            "machine_error": cli_error_summary(&message),
+        });
+    }
+    json!({
+        "status": "process_started",
+        "operation": "restart_virtual_process",
+        "accepted": true,
+        "process": response.get("process").cloned().unwrap_or(Value::Null),
+        "coordinator_epoch": response.get("epoch").cloned().unwrap_or(Value::Null),
+        "requires_confirmation": requires_confirmation,
+        "explicit_user_action": true,
+        "website_required": false,
+        "single_active_process_boundary": true,
+    })
+}
+
+pub(crate) fn process_cancel_request_summary(
+    response: &Value,
+    requires_confirmation: bool,
+) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("process_cancellation_requested") {
+        let message = response_error_message(response, "coordinator rejected process cancel");
+        return json!({
+            "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+            "operation": "cancel_virtual_process",
+            "accepted": false,
+            "requires_confirmation": requires_confirmation,
+            "explicit_user_action": true,
+            "whole_process_cancel_available": true,
+            "error": message,
+            "machine_error": cli_error_summary(&message),
+        });
+    }
+    let cancelled_tasks = response
+        .get("cancelled_tasks")
+        .and_then(Value::as_array)
+        .map(Vec::len)
+        .unwrap_or(0);
+    json!({
+        "status": "process_cancellation_requested",
+        "operation": "cancel_virtual_process",
+        "accepted": true,
+        "process": response.get("process").cloned().unwrap_or(Value::Null),
+        "cancelled_task_count": cancelled_tasks,
+        "cancelled_tasks": response.get("cancelled_tasks").cloned().unwrap_or_else(|| json!([])),
+        "affected_nodes": response.get("affected_nodes").cloned().unwrap_or_else(|| json!([])),
+        "requires_confirmation": requires_confirmation,
+        "explicit_user_action": true,
+        "website_required": false,
+        "whole_process_cancel_available": true,
+        "node_must_poll_task_control": true,
+        "new_task_launches_blocked": true,
+        "surviving_state_visibility": "task and artifact state remains visible after terminal task events are reported",
+    })
+}
+
+pub(crate) fn task_restart_request_summary(response: &Value, requires_confirmation: bool) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("task_restart") {
+        let message = response_error_message(response, "coordinator rejected task restart");
+        return json!({
+            "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+            "operation": "restart_selected_task",
+            "accepted": false,
+            "requires_confirmation": requires_confirmation,
+            "explicit_user_action": true,
+            "clean_boundary_required": true,
+            "error": message,
+            "machine_error": cli_error_summary(&message),
+        });
+    }
+    json!({
+        "status": "task_restart",
+        "operation": "restart_selected_task",
+        "accepted": response.get("accepted").cloned().unwrap_or(json!(false)),
+        "process": response.get("process").cloned().unwrap_or(Value::Null),
+        "task": response.get("task").cloned().unwrap_or(Value::Null),
+        "requires_confirmation": requires_confirmation,
+        "explicit_user_action": true,
+        "clean_boundary_required": true,
+        "clean_boundary_available": response
+            .get("clean_boundary_available")
+            .cloned()
+            .unwrap_or(json!(false)),
+        "active_task": response
+            .get("active_task")
+            .cloned()
+            .unwrap_or(json!(false)),
+        "completed_event_observed": response
+            .get("completed_event_observed")
+            .cloned()
+            .unwrap_or(json!(false)),
+        "requires_whole_process_restart": response
+            .get("requires_whole_process_restart")
+            .cloned()
+            .unwrap_or(json!(true)),
+        "message": response.get("message").cloned().unwrap_or(Value::Null),
+        "audit_event": response.get("audit_event").cloned().unwrap_or(Value::Null),
+        "charged_debug_read_bytes": response
+            .get("charged_debug_read_bytes")
+            .cloned()
+            .unwrap_or_else(|| json!(0)),
+        "used_debug_read_bytes": response
+            .get("used_debug_read_bytes")
+            .cloned()
+            .unwrap_or_else(|| json!(0)),
+        "debug_reads_quota_limited": true,
+        "website_required": false,
+    })
+}
+
+pub(crate) fn artifact_download_session_summary(response: &Value) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
+        let message = response_error_message(response, "coordinator rejected artifact download");
+        return json!({
+            "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+            "link_issued": false,
+            "explicit_user_action_required": true,
+            "error": message.clone(),
+            "machine_error": cli_error_summary_with_default(&message, "connectivity"),
+        });
+    }
+
+    let link = response.get("link").unwrap_or(&Value::Null);
+    json!({
+        "status": "download_link_issued",
+        "link_issued": true,
+        "explicit_user_action_required": true,
+        "coordinator_preflight": "completed_before_link_issued",
+        "tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
+        "project": link.get("project").cloned().unwrap_or(Value::Null),
+        "process": link.get("process").cloned().unwrap_or(Value::Null),
+        "artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
+        "actor": link.get("actor").cloned().unwrap_or(Value::Null),
+        "source": link.get("source").cloned().unwrap_or(Value::Null),
+        "url_path": link.get("url_path").cloned().unwrap_or(Value::Null),
+        "expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
+        "max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
+        "token_material_returned": false,
+        "scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
+        "policy_context_digest_present": link.get("policy_context_digest").is_some(),
+        "authorization_required": true,
+        "short_lived": link.get("expires_at_epoch_seconds").is_some(),
+        "guessable_public_url": false,
+        "cross_tenant_usable": false,
+        "unauthorized_project_usable": false,
+        "default_durable_store_assumed": false,
+    })
+}
+
+pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
+        return json!([]);
+    }
+
+    let link = response.get("link").unwrap_or(&Value::Null);
+    json!([{
+        "grant": "artifact_download",
+        "description": "download scoped artifact bytes to the requesting machine",
+        "risk": "authorized artifact bytes leave the retaining node or explicit storage through an explicit download/export operation",
+        "coordinator_policy_limited": true,
+        "authorization_required": true,
+        "explicit_user_action_required": true,
+        "tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
+        "project": link.get("project").cloned().unwrap_or(Value::Null),
+        "process": link.get("process").cloned().unwrap_or(Value::Null),
+        "artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
+        "actor": link.get("actor").cloned().unwrap_or(Value::Null),
+        "source": link.get("source").cloned().unwrap_or(Value::Null),
+        "max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
+        "expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
+        "short_lived": link.get("expires_at_epoch_seconds").is_some(),
+        "scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
+        "token_material_returned": false,
+        "guessable_public_url": false,
+        "cross_tenant_reuse_allowed": false,
+        "unauthorized_project_reuse_allowed": false,
+        "default_durable_store_assumed": false,
+        "private_website_required": false,
+    }])
+}
+
+pub(crate) fn artifact_export_plan_summary(response: &Value, to: &Path) -> Value {
+    if response.get("type").and_then(Value::as_str) != Some("artifact_export_plan") {
+        let message = response_error_message(response, "coordinator rejected artifact export");
+        return json!({
+            "status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
+            "explicit_user_action": true,
+            "local_path": to,
+            "local_bytes_written_by_cli": false,
+            "default_durable_store_assumed": false,
+            "error": message.clone(),
+            "machine_error": cli_error_summary_with_default(&message, "connectivity"),
+        });
+    }
+
+    let plan = response.get("plan").unwrap_or(&Value::Null);
+    json!({
+        "status": "transfer_plan_created",
+        "explicit_user_action": true,
+        "local_path": to,
+        "local_bytes_written_by_cli": false,
+        "writes_require_data_plane_followup": true,
+        "default_durable_store_assumed": false,
+        "artifact_size_bytes": response.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
+        "source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
+        "receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
+        "transport": plan.get("transport").cloned().unwrap_or(Value::Null),
+        "artifact": plan.pointer("/scope/object/Artifact").cloned().unwrap_or(Value::Null),
+        "tenant": plan.pointer("/scope/tenant").cloned().unwrap_or(Value::Null),
+        "project": plan.pointer("/scope/project").cloned().unwrap_or(Value::Null),
+        "process": plan.pointer("/scope/process").cloned().unwrap_or(Value::Null),
+        "coordinator_assisted_rendezvous": plan.get("coordinator_assisted_rendezvous").cloned().unwrap_or(Value::Null),
+        "coordinator_bulk_relay_allowed": plan.get("coordinator_bulk_relay_allowed").cloned().unwrap_or(Value::Null),
+        "authorization_digest_present": plan.get("authorization_digest").is_some(),
+    })
+}
+
+pub(crate) fn task_event_count(task_events: Option<&Value>) -> usize {
+    task_events
+        .and_then(|task_events| task_events.pointer("/response/events"))
+        .and_then(Value::as_array)
+        .map(Vec::len)
+        .unwrap_or(0)
+}
+
+pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
+    let current_usage = quota_current_usage(attached_nodes, task_events);
+    let next_blocked_action = quota_next_blocked_action(&current_usage);
+    json!({
+        "source": "cli_project_status_summary",
+        "current_usage": current_usage,
+        "limits": quota_limits_value(),
+        "next_blocked_action": next_blocked_action,
+        "private_abuse_heuristics_exposed": false,
+    })
+}
+
+pub(crate) fn quota_current_usage(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
+    let attached_node_count = attached_nodes
+        .get("count")
+        .and_then(Value::as_u64)
+        .unwrap_or(0);
+    let online_node_count = attached_nodes
+        .get("online")
+        .and_then(Value::as_u64)
+        .unwrap_or(0);
+    json!({
+        "attached_nodes": attached_node_count,
+        "online_nodes": online_node_count,
+        "observed_task_events": task_event_count(task_events),
+        "artifact_download_bytes": 0,
+        "rendezvous_attempts": 0,
+        "hosted_wasm_processes": 0,
+    })
+}
+
+pub(crate) fn quota_limits_value() -> Value {
+    json!({
+        "source": "coordinator",
+        "configured": false,
+        "message": "connect to the selected coordinator to read its scoped quota configuration",
+    })
+}
+
+pub(crate) fn quota_next_blocked_action(current_usage: &Value) -> Value {
+    if current_usage
+        .get("online_nodes")
+        .and_then(Value::as_u64)
+        .unwrap_or(0)
+        == 0
+    {
+        return json!({
+            "action": "node_work_requires_online_attached_node",
+            "category": "capability",
+            "quota_related": false,
+            "message": "no online attached node is visible for work that requires a user node",
+            "machine_error": cli_error_summary_for_category(
+                "capability",
+                "no online attached node is visible for work that requires a user node"
+            )
+        });
+    }
+    Value::Null
+}

crates/clusterflux-cli/src/project.rs

@@ -0,0 +1,381 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, list_attached_nodes_if_available_with_session,
+    list_task_events_if_available_with_session, stored_session_for_coordinator, JsonLineSession,
+};
+use crate::config::{
+    effective_project_scope, effective_scope_value, project_config_file, read_cli_session,
+    read_project_config, write_project_config, ProjectConfig, StoredCliSession,
+};
+use crate::process::process_list_report_with_session;
+use crate::process_events::project_quota_posture;
+use crate::{
+    bundle_inspection, discovered_environment_names, BundleInspectArgs, ProcessListArgs,
+    ProjectInitArgs, ProjectListArgs, ProjectSelectArgs, ProjectStatusArgs,
+};
+
+pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result<Value> {
+    let stored_session = read_cli_session(&cwd)?;
+    let tenant = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.tenant,
+        |session| session.tenant.as_str(),
+        "tenant",
+    );
+    let project = args.new_project.clone();
+    let user = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.user,
+        |session| session.user.as_str(),
+        "user",
+    );
+    let coordinator = args.scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    let name = args.name.clone();
+    let config = ProjectConfig {
+        tenant: tenant.clone(),
+        project: project.clone(),
+        user: user.clone(),
+        coordinator: coordinator.clone(),
+    };
+    let config_file = project_config_file(&cwd);
+    if config_file.exists() && !args.yes {
+        anyhow::bail!(
+            "{} already exists; rerun with --yes to update the project link",
+            config_file.display()
+        );
+    }
+    let mut coordinator_session_requests = 0;
+    let coordinator_response = if let Some(coordinator) = &coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "create_project",
+                "project": project,
+                "name": name,
+            }),
+            json!({
+                "type": "create_project",
+                "tenant": tenant,
+                "actor_user": user,
+                "project": project,
+                "name": name,
+            }),
+        )?;
+        let response = session.request(request)?;
+        coordinator_session_requests = session.requests();
+        Some(response)
+    } else {
+        None
+    };
+    write_project_config(&cwd, &config)?;
+    let created_or_linked_project = coordinator_response
+        .as_ref()
+        .and_then(|response| response.get("project"))
+        .cloned()
+        .unwrap_or_else(|| {
+            json!({
+                "id": config.project.clone(),
+                "tenant": config.tenant.clone(),
+                "name": args.name.clone(),
+            })
+        });
+    Ok(json!({
+        "command": "project init",
+        "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
+        "private_website_required": false,
+        "project_config_written": true,
+        "project_config_write_after_coordinator_acceptance": coordinator.is_some(),
+        "coordinator_create_before_local_write": coordinator.is_some(),
+        "coordinator_session_requests": coordinator_session_requests,
+        "created_or_linked_project": created_or_linked_project,
+        "current_directory_link": {
+            "cwd": cwd,
+            "config_file": config_file,
+            "config_format": "clusterflux_project_config_v1",
+            "links_current_directory": true,
+            "writes_current_directory_only": true,
+            "private_website_required": false,
+        },
+        "safe_defaults": {
+            "tenant": config.tenant.clone(),
+            "project": config.project.clone(),
+            "user": config.user.clone(),
+            "coordinator": config.coordinator.clone(),
+            "project_name": args.name.clone(),
+            "default_project_id_used": args.new_project == "project",
+            "default_project_name_used": args.name == "Clusterflux Project",
+            "browser_interaction_required": false,
+            "private_website_required": false,
+        },
+        "project_config": config,
+        "config_file": project_config_file(&cwd),
+        "coordinator_response": coordinator_response,
+    }))
+}
+
+pub(crate) fn project_status_report(args: ProjectStatusArgs, cwd: PathBuf) -> Result<Value> {
+    let config = read_project_config(&cwd)?;
+    let stored_session = read_cli_session(&cwd)?;
+    let mut effective_scope = effective_project_scope(&args.scope, config.as_ref());
+    if effective_scope.coordinator.is_none() {
+        effective_scope.coordinator = stored_session
+            .as_ref()
+            .filter(|session| session.session_secret.is_some())
+            .map(|session| session.coordinator.clone());
+    }
+    if let (Some(config), Some(session)) = (config.as_ref(), stored_session.as_ref()) {
+        let same_coordinator = effective_scope
+            .coordinator
+            .as_deref()
+            .is_some_and(|coordinator| {
+                crate::client::control_endpoint_identity(coordinator).ok()
+                    == crate::client::control_endpoint_identity(&session.coordinator).ok()
+            });
+        if same_coordinator
+            && (session.tenant != config.tenant
+                || session.project != config.project
+                || session.user != config.user)
+        {
+            anyhow::bail!(
+                "stored CLI session is for {}/{}/{} but this workspace is configured for {}/{}/{}; run `clusterflux login --browser` from this workspace",
+                session.tenant,
+                session.project,
+                session.user,
+                config.tenant,
+                config.project,
+                config.user,
+            );
+        }
+    }
+    if let Some(bound_session) = effective_scope
+        .coordinator
+        .as_deref()
+        .and_then(|coordinator| {
+            stored_session_for_coordinator(coordinator, stored_session.as_ref())
+        })
+    {
+        effective_scope.tenant = bound_session.tenant.clone();
+        effective_scope.project = bound_session.project.clone();
+        effective_scope.user = bound_session.user.clone();
+    }
+    let inspection = bundle_inspection(
+        BundleInspectArgs {
+            project: Some(cwd.clone()),
+            source_provider: None,
+            disabled_source_providers: Vec::new(),
+            json: true,
+        },
+        cwd.clone(),
+    )
+    .ok();
+    let coordinator = effective_scope.coordinator.clone();
+    let attached_nodes = list_attached_nodes_if_available_with_session(
+        coordinator.as_deref(),
+        &effective_scope,
+        stored_session.as_ref(),
+    )?;
+    let coordinator_response = list_task_events_if_available_with_session(
+        coordinator.as_deref(),
+        &effective_scope,
+        None,
+        stored_session.as_ref(),
+    )?;
+    let process_report = process_list_report_with_session(
+        ProcessListArgs {
+            scope: effective_scope.clone(),
+        },
+        stored_session.as_ref(),
+    )?;
+    let discovered_environments = discovered_environment_names(inspection.as_ref());
+    let active_process = process_report
+        .get("processes")
+        .and_then(Value::as_array)
+        .and_then(|processes| processes.first())
+        .and_then(|process| process.get("process"))
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .unwrap_or_else(|| {
+            if process_report.get("status").and_then(Value::as_str) == Some("ok") {
+                "none".to_owned()
+            } else {
+                "unknown_without_coordinator".to_owned()
+            }
+        });
+    let quota_posture = project_quota_posture(&attached_nodes, coordinator_response.as_ref());
+    Ok(json!({
+        "command": "project status",
+        "cwd": cwd,
+        "tenant": effective_scope.tenant,
+        "project": effective_scope.project,
+        "user": effective_scope.user,
+        "coordinator": coordinator,
+        "project_identity": {
+            "tenant": effective_scope.tenant,
+            "project": effective_scope.project,
+            "user": effective_scope.user,
+            "source": if config.is_some() { "project_config_with_cli_overrides" } else { "cli_scope" }
+        },
+        "project_config": config,
+        "bundle": inspection,
+        "discovered_environments": discovered_environments,
+        "active_process": active_process,
+        "processes": process_report.get("processes").cloned().unwrap_or_else(|| json!([])),
+        "attached_nodes": attached_nodes,
+        "quota_posture": quota_posture,
+        "coordinator_response": coordinator_response,
+    }))
+}
+
+pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result<Value> {
+    let stored_session = read_cli_session(&cwd)?;
+    let coordinator = args.scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    let tenant = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.tenant,
+        |session| session.tenant.as_str(),
+        "tenant",
+    );
+    let user = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.user,
+        |session| session.user.as_str(),
+        "user",
+    );
+    if let Some(coordinator) = &coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "list_projects",
+            }),
+            json!({
+                "type": "list_projects",
+                "tenant": tenant,
+                "actor_user": user,
+            }),
+        )?;
+        let response = session.request(request)?;
+        let projects = response
+            .get("projects")
+            .cloned()
+            .unwrap_or_else(|| json!([]));
+        let project_count = projects.as_array().map(Vec::len).unwrap_or(0);
+        return Ok(json!({
+            "command": "project list",
+            "source": "public_coordinator_api",
+            "coordinator": coordinator,
+            "tenant": tenant,
+            "user": user,
+            "projects": projects,
+            "project_count": project_count,
+            "private_website_required": false,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    let projects = read_project_config(&cwd)?.into_iter().collect::<Vec<_>>();
+    let project_count = projects.len();
+    Ok(json!({
+        "command": "project list",
+        "source": "local_project_config",
+        "projects": projects,
+        "project_count": project_count,
+        "private_website_required": false,
+    }))
+}
+
+pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Result<Value> {
+    let stored_session = read_cli_session(&cwd)?;
+    let tenant = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.tenant,
+        |session| session.tenant.as_str(),
+        "tenant",
+    );
+    let user = session_or_effective_scope_value(
+        stored_session.as_ref(),
+        &args.scope.user,
+        |session| session.user.as_str(),
+        "user",
+    );
+    let coordinator = args.scope.coordinator.clone().or_else(|| {
+        stored_session
+            .as_ref()
+            .map(|session| session.coordinator.clone())
+    });
+    let config = ProjectConfig {
+        tenant: tenant.clone(),
+        project: args.selected_project.clone(),
+        user: user.clone(),
+        coordinator: coordinator.clone(),
+    };
+    let coordinator_response = if let Some(coordinator) = &coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let request = authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({
+                "type": "select_project",
+                "project": args.selected_project,
+            }),
+            json!({
+                "type": "select_project",
+                "tenant": tenant,
+                "actor_user": user,
+                "project": args.selected_project,
+            }),
+        )?;
+        Some(session.request(request)?)
+    } else {
+        None
+    };
+    write_project_config(&cwd, &config)?;
+    let selected_project = coordinator_response
+        .as_ref()
+        .and_then(|response| response.get("project"))
+        .cloned()
+        .unwrap_or_else(|| {
+            json!({
+                "id": config.project.clone(),
+                "tenant": config.tenant.clone(),
+                "name": config.project.clone(),
+            })
+        });
+    Ok(json!({
+        "command": "project select",
+        "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
+        "selected_project": selected_project,
+        "project_config_written": true,
+        "private_website_required": false,
+        "project_config": config,
+        "coordinator_response": coordinator_response,
+    }))
+}
+
+fn session_or_effective_scope_value(
+    stored_session: Option<&StoredCliSession>,
+    cli_value: &str,
+    session_value: impl FnOnce(&StoredCliSession) -> &str,
+    default_value: &str,
+) -> String {
+    if let Some(session) = stored_session.filter(|session| session.session_secret.is_some()) {
+        session_value(session).to_owned()
+    } else {
+        effective_scope_value(cli_value, stored_session.map(session_value), default_value)
+    }
+}

crates/clusterflux-cli/src/quota.rs

@@ -0,0 +1,111 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, list_attached_nodes_if_available_with_session,
+    list_task_events_if_available_with_session, stored_session_for_coordinator, JsonLineSession,
+};
+use crate::config::{effective_project_scope, read_cli_session, read_project_config};
+use crate::process_events::{quota_current_usage, quota_limits_value, quota_next_blocked_action};
+use crate::QuotaStatusArgs;
+
+pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result<Value> {
+    let config = read_project_config(&cwd)?;
+    let stored_session = read_cli_session(&cwd)?;
+    let mut effective_scope = effective_project_scope(&args.scope, config.as_ref());
+    if effective_scope.coordinator.is_none() {
+        effective_scope.coordinator = stored_session
+            .as_ref()
+            .filter(|session| session.session_secret.is_some())
+            .map(|session| session.coordinator.clone());
+    }
+    if let Some(bound_session) = effective_scope
+        .coordinator
+        .as_deref()
+        .and_then(|coordinator| {
+            stored_session_for_coordinator(coordinator, stored_session.as_ref())
+        })
+    {
+        effective_scope.tenant = bound_session.tenant.clone();
+        effective_scope.project = bound_session.project.clone();
+        effective_scope.user = bound_session.user.clone();
+    }
+    let coordinator = effective_scope.coordinator.clone();
+    let attached_nodes = list_attached_nodes_if_available_with_session(
+        coordinator.as_deref(),
+        &effective_scope,
+        stored_session.as_ref(),
+    )?;
+    let task_events = list_task_events_if_available_with_session(
+        coordinator.as_deref(),
+        &effective_scope,
+        None,
+        stored_session.as_ref(),
+    )?;
+    let quota_status = if let Some(coordinator) = coordinator.as_deref() {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        Some(session.request(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session.as_ref(),
+            json!({ "type": "quota_status" }),
+            json!({
+                "type": "quota_status",
+                "tenant": effective_scope.tenant,
+                "project": effective_scope.project,
+                "actor_user": effective_scope.user,
+            }),
+        )?)?)
+    } else {
+        None
+    };
+    let mut current_usage = quota_current_usage(&attached_nodes, task_events.as_ref());
+    if let (Some(object), Some(status)) = (current_usage.as_object_mut(), quota_status.as_ref()) {
+        object.insert(
+            "scoped_resource_usage".to_owned(),
+            status.get("usage").cloned().unwrap_or(Value::Null),
+        );
+        object.insert(
+            "window_started_epoch_seconds".to_owned(),
+            status
+                .get("window_started_epoch_seconds")
+                .cloned()
+                .unwrap_or(Value::Null),
+        );
+    }
+    let limits = quota_status
+        .as_ref()
+        .and_then(|status| status.pointer("/limits/limits"))
+        .cloned()
+        .unwrap_or_else(quota_limits_value);
+    let window_seconds = quota_status
+        .as_ref()
+        .and_then(|status| status.get("window_seconds"))
+        .cloned()
+        .unwrap_or(Value::Null);
+    let quota_tier = quota_status
+        .as_ref()
+        .and_then(|status| status.get("policy_label"))
+        .cloned()
+        .unwrap_or(Value::Null);
+    Ok(json!({
+        "command": "quota status",
+        "tenant": effective_scope.tenant,
+        "project": effective_scope.project,
+        "user": effective_scope.user,
+        "coordinator": coordinator,
+        "project_config": config,
+        "policy_surface": "generic public quota categories; hosted tuning remains private policy",
+        "limits": limits,
+        "window_seconds": window_seconds,
+        "current_usage": current_usage,
+        "attached_nodes": attached_nodes,
+        "task_events": task_events,
+        "next_blocked_action": quota_next_blocked_action(&current_usage),
+        "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
+        "quota_tier": quota_tier,
+        "private_abuse_heuristics_exposed": false,
+        "quota_response": quota_status,
+    }))
+}

crates/clusterflux-cli/src/run/local_services.rs

@@ -0,0 +1,177 @@
+use std::io::{BufRead, BufReader};
+use std::path::Path;
+use std::process::{Child, Command, Stdio};
+
+use anyhow::{Context, Result};
+use serde_json::Value;
+
+pub(super) struct LocalNodeWorker {
+    pub(super) process_id: u32,
+    child: Option<Child>,
+}
+
+impl LocalNodeWorker {
+    pub(super) fn start(coordinator: &str, project: &Path, enrollment_grant: &str) -> Result<Self> {
+        let mut command = node_command()?;
+        command.args([
+            "--coordinator",
+            coordinator,
+            "--tenant",
+            "tenant",
+            "--project-id",
+            "project",
+            "--node",
+            "node-cli-local",
+            "--enrollment-grant",
+            enrollment_grant,
+            "--worker",
+            "--project-root",
+        ]);
+        command.arg(project);
+        command.args(["--assignment-poll-ms", "20"]);
+        command.stdout(Stdio::null());
+        command.stderr(Stdio::inherit());
+        let child = command.spawn().context("failed to spawn node process")?;
+        let process_id = child.id();
+        Ok(Self {
+            process_id,
+            child: Some(child),
+        })
+    }
+
+    pub(super) fn stop(&mut self) {
+        if let Some(mut child) = self.child.take() {
+            let _ = child.kill();
+            let _ = child.wait();
+        }
+    }
+}
+
+impl Drop for LocalNodeWorker {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+pub(super) struct LocalCoordinator {
+    pub(super) address: String,
+    pub(super) process_id: Option<u32>,
+    child: Option<Child>,
+}
+
+impl LocalCoordinator {
+    pub(super) fn external(address: &str) -> Self {
+        Self {
+            address: address.to_owned(),
+            process_id: None,
+            child: None,
+        }
+    }
+
+    pub(super) fn start_ephemeral() -> Result<Self> {
+        let mut command = coordinator_command()?;
+        command.args(["--listen", "127.0.0.1:0", "--allow-local-trusted-loopback"]);
+        command.stdout(Stdio::piped());
+        command.stderr(Stdio::inherit());
+
+        let mut child = command
+            .spawn()
+            .context("failed to spawn local coordinator process")?;
+        let process_id = child.id();
+        let address = match read_coordinator_ready_address(&mut child) {
+            Ok(address) => address,
+            Err(error) => {
+                let _ = child.kill();
+                let _ = child.wait();
+                return Err(error);
+            }
+        };
+
+        Ok(Self {
+            address,
+            process_id: Some(process_id),
+            child: Some(child),
+        })
+    }
+}
+
+impl Drop for LocalCoordinator {
+    fn drop(&mut self) {
+        if let Some(mut child) = self.child.take() {
+            let _ = child.kill();
+            let _ = child.wait();
+        }
+    }
+}
+
+fn read_coordinator_ready_address(child: &mut Child) -> Result<String> {
+    let stdout = child
+        .stdout
+        .take()
+        .context("local coordinator stdout was not captured")?;
+    let mut ready_line = String::new();
+    BufReader::new(stdout)
+        .read_line(&mut ready_line)
+        .context("failed to read local coordinator ready line")?;
+    if ready_line.trim().is_empty() {
+        anyhow::bail!("local coordinator exited before reporting its listen address");
+    }
+    let ready: Value =
+        serde_json::from_str(&ready_line).context("local coordinator ready line was not JSON")?;
+    ready
+        .get("listen")
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .context("local coordinator did not report a listen address")
+}
+
+fn node_command() -> Result<Command> {
+    if let Some(path) = std::env::var_os("CLUSTERFLUX_NODE_BIN") {
+        return Ok(Command::new(path));
+    }
+
+    let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
+    sibling.set_file_name(format!("clusterflux-node{}", std::env::consts::EXE_SUFFIX));
+    if sibling.is_file() {
+        return Ok(Command::new(sibling));
+    }
+
+    let mut command = Command::new("cargo");
+    command.args([
+        "run",
+        "-q",
+        "-p",
+        "clusterflux-node",
+        "--bin",
+        "clusterflux-node",
+        "--",
+    ]);
+    Ok(command)
+}
+
+fn coordinator_command() -> Result<Command> {
+    if let Some(path) = std::env::var_os("CLUSTERFLUX_COORDINATOR_BIN") {
+        return Ok(Command::new(path));
+    }
+
+    let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
+    sibling.set_file_name(format!(
+        "clusterflux-coordinator{}",
+        std::env::consts::EXE_SUFFIX
+    ));
+    if sibling.is_file() {
+        return Ok(Command::new(sibling));
+    }
+
+    let mut command = Command::new("cargo");
+    command.args([
+        "run",
+        "-q",
+        "-p",
+        "clusterflux-coordinator",
+        "--bin",
+        "clusterflux-coordinator",
+        "--",
+    ]);
+    Ok(command)
+}

crates/clusterflux-cli/src/task.rs

@@ -0,0 +1,106 @@
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::client::{
+    authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
+    JsonLineSession,
+};
+use crate::config::StoredCliSession;
+use crate::process_events::{task_restart_request_summary, task_summaries};
+use crate::{confirmation_required_report, TaskListArgs, TaskRestartArgs};
+
+#[cfg(test)]
+pub(crate) fn task_list_report(args: TaskListArgs) -> Result<Value> {
+    task_list_report_with_session(args, None)
+}
+
+pub(crate) fn task_list_report_with_session(
+    args: TaskListArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    let events = list_task_events_if_available_with_session(
+        args.scope.coordinator.as_deref(),
+        &args.scope,
+        args.process.clone(),
+        stored_session,
+    )?;
+    let tasks = task_summaries(events.as_ref());
+    Ok(json!({
+        "command": "task list",
+        "process": args.process,
+        "tasks": tasks,
+        "events": events,
+    }))
+}
+
+#[cfg(test)]
+pub(crate) fn task_restart_report(args: TaskRestartArgs) -> Result<Value> {
+    task_restart_report_with_session(args, None)
+}
+
+pub(crate) fn task_restart_report_with_session(
+    args: TaskRestartArgs,
+    stored_session: Option<&StoredCliSession>,
+) -> Result<Value> {
+    if !args.yes {
+        return Ok(confirmation_required_report(
+            "task restart",
+            "restart_selected_task",
+            json!({
+                "coordinator": args.scope.coordinator,
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "process": args.process,
+                "task": args.task,
+            }),
+            format!(
+                "clusterflux task restart {} --process {} --yes",
+                args.task, args.process
+            ),
+        ));
+    }
+    if let Some(coordinator) = &args.scope.coordinator {
+        let mut session = JsonLineSession::connect(coordinator)?;
+        let response = session.request_allow_error(authenticated_or_local_trusted_request(
+            coordinator,
+            stored_session,
+            json!({
+                "type": "restart_task",
+                "process": args.process,
+                "task": args.task,
+            }),
+            json!({
+                "type": "restart_task",
+                "tenant": args.scope.tenant,
+                "project": args.scope.project,
+                "actor_user": args.scope.user,
+                "process": args.process,
+                "task": args.task,
+            }),
+        )?)?;
+        let restart_request = task_restart_request_summary(&response, !args.yes);
+        return Ok(json!({
+            "command": "task restart",
+            "coordinator": coordinator,
+            "process": args.process,
+            "task": args.task,
+            "requires_confirmation": !args.yes,
+            "restart_request": restart_request,
+            "response": response,
+            "coordinator_session_requests": session.requests(),
+        }));
+    }
+    Ok(json!({
+        "command": "task restart",
+        "status": "requires_coordinator",
+        "requires_confirmation": !args.yes,
+        "process": args.process,
+        "task": args.task,
+        "restart_request": {
+            "status": "requires_coordinator",
+            "operation": "restart_selected_task",
+            "explicit_user_action": true,
+            "clean_boundary_required": true,
+        },
+    }))
+}

crates/clusterflux-cli/src/tools.rs

@@ -0,0 +1,63 @@
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use anyhow::Result;
+
+pub(crate) fn command_available(command: &str) -> bool {
+    Command::new(command)
+        .arg("--version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .is_ok()
+}
+
+pub(crate) fn command_nonce(prefix: &str) -> String {
+    let now = unix_timestamp_nanos();
+    format!("{prefix}-{now}-{}", std::process::id())
+}
+
+pub(crate) fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or_default()
+}
+
+fn unix_timestamp_nanos() -> u128 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_nanos())
+        .unwrap_or_default()
+}
+
+pub(crate) fn sibling_binary(name: &str) -> Option<PathBuf> {
+    let mut sibling = std::env::current_exe().ok()?;
+    sibling.set_file_name(format!("{name}{}", std::env::consts::EXE_SUFFIX));
+    sibling.is_file().then_some(sibling)
+}
+
+pub(crate) fn dap_binary_path() -> Result<PathBuf> {
+    if let Some(path) = std::env::var_os("CLUSTERFLUX_DAP_BIN") {
+        return Ok(PathBuf::from(path));
+    }
+    if let Some(path) = sibling_binary("clusterflux-debug-dap") {
+        return Ok(path);
+    }
+    let release = PathBuf::from("target/release").join(format!(
+        "clusterflux-debug-dap{}",
+        std::env::consts::EXE_SUFFIX
+    ));
+    if release.is_file() {
+        return Ok(release);
+    }
+    let debug = PathBuf::from("target/debug").join(format!(
+        "clusterflux-debug-dap{}",
+        std::env::consts::EXE_SUFFIX
+    ));
+    if debug.is_file() {
+        return Ok(debug);
+    }
+    anyhow::bail!("could not locate clusterflux-debug-dap; set CLUSTERFLUX_DAP_BIN")
+}

crates/clusterflux-control/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "clusterflux-control"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde_json.workspace = true
+thiserror.workspace = true
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+ureq.workspace = true

crates/clusterflux-control/src/lib.rs

@@ -0,0 +1,317 @@
+#[cfg(not(target_arch = "wasm32"))]
+use std::io::{BufRead, BufReader, Read, Write};
+#[cfg(not(target_arch = "wasm32"))]
+use std::net::TcpStream;
+use std::net::{IpAddr, ToSocketAddrs};
+#[cfg(not(target_arch = "wasm32"))]
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+
+use serde_json::Value;
+use thiserror::Error;
+
+pub const CONTROL_API_PATH: &str = "/api/v1/control";
+pub const LOGIN_API_PATH: &str = "/api/v1/login";
+pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024;
+
+#[derive(Debug, Error)]
+pub enum ControlTransportError {
+    #[error("invalid coordinator endpoint: {0}")]
+    InvalidEndpoint(String),
+    #[error("insecure remote coordinator endpoint is forbidden: {0}")]
+    InsecureRemote(String),
+    #[error("coordinator transport I/O failed: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("coordinator transport JSON failed: {0}")]
+    Json(#[from] serde_json::Error),
+    #[error("coordinator HTTP request failed: {0}")]
+    Http(String),
+    #[error("coordinator control frame exceeds {MAX_CONTROL_FRAME_BYTES} bytes")]
+    FrameTooLarge,
+    #[error("coordinator closed the local control session without a response")]
+    Closed,
+    #[error("coordinator network transport is unavailable inside a Wasm guest")]
+    UnavailableInWasm,
+}
+
+enum ControlTransport {
+    #[cfg(not(target_arch = "wasm32"))]
+    Https { agent: ureq::Agent, url: String },
+    #[cfg(not(target_arch = "wasm32"))]
+    LoopbackJsonLine {
+        writer: TcpStream,
+        reader: BufReader<TcpStream>,
+    },
+    #[cfg(target_arch = "wasm32")]
+    #[allow(dead_code)]
+    Unavailable,
+}
+
+pub struct ControlSession {
+    transport: ControlTransport,
+    requests: u64,
+}
+
+impl ControlSession {
+    pub fn connect(endpoint: &str) -> Result<Self, ControlTransportError> {
+        Self::connect_with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
+    }
+
+    pub fn connect_to_api_path(
+        endpoint: &str,
+        api_path: &str,
+    ) -> Result<Self, ControlTransportError> {
+        let mut session = Self::connect(endpoint)?;
+        #[cfg(not(target_arch = "wasm32"))]
+        if let ControlTransport::Https { url, .. } = &mut session.transport {
+            *url = endpoint_api_url(endpoint, api_path)?;
+        }
+        Ok(session)
+    }
+
+    pub fn connect_with_timeouts(
+        endpoint: &str,
+        connect_timeout: Duration,
+        io_timeout: Duration,
+    ) -> Result<Self, ControlTransportError> {
+        #[cfg(target_arch = "wasm32")]
+        {
+            let _ = (endpoint, connect_timeout, io_timeout);
+            return Err(ControlTransportError::UnavailableInWasm);
+        }
+
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let endpoint = endpoint.trim();
+            if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
+                let url = control_api_url(endpoint)?;
+                if endpoint.starts_with("http://") && !endpoint_is_loopback(endpoint) {
+                    return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
+                }
+                let agent = ureq::AgentBuilder::new()
+                    .timeout_connect(connect_timeout)
+                    .timeout_read(io_timeout)
+                    .timeout_write(io_timeout)
+                    .build();
+                return Ok(Self {
+                    transport: ControlTransport::Https { agent, url },
+                    requests: 0,
+                });
+            }
+
+            let loopback_address = endpoint
+                .strip_prefix("clusterflux+tcp://")
+                .unwrap_or(endpoint);
+            if !endpoint_is_loopback(loopback_address) {
+                return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
+            }
+            let writer = TcpStream::connect(loopback_address)?;
+            writer.set_read_timeout(Some(io_timeout))?;
+            writer.set_write_timeout(Some(io_timeout))?;
+            let reader = BufReader::new(writer.try_clone()?);
+            Ok(Self {
+                transport: ControlTransport::LoopbackJsonLine { writer, reader },
+                requests: 0,
+            })
+        }
+    }
+
+    pub fn request(&mut self, value: &Value) -> Result<Value, ControlTransportError> {
+        #[cfg(target_arch = "wasm32")]
+        {
+            let _ = (&self.transport, self.requests, value);
+            return Err(ControlTransportError::UnavailableInWasm);
+        }
+
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let encoded = serde_json::to_vec(value)?;
+            if encoded.len() > MAX_CONTROL_FRAME_BYTES {
+                return Err(ControlTransportError::FrameTooLarge);
+            }
+            let inject_response_loss = should_inject_response_loss(value);
+            let response = match &mut self.transport {
+                #[cfg(not(target_arch = "wasm32"))]
+                ControlTransport::Https { agent, url } => {
+                    let response = agent
+                        .post(url)
+                        .set("Content-Type", "application/json")
+                        .set("Accept", "application/json")
+                        .send_bytes(&encoded)
+                        .map_err(|error| ControlTransportError::Http(error.to_string()))?;
+                    if inject_response_loss {
+                        return Err(ControlTransportError::Closed);
+                    }
+                    if response.status() != 200 {
+                        return Err(ControlTransportError::Http(format!(
+                            "coordinator returned HTTP {} {}",
+                            response.status(),
+                            response.status_text()
+                        )));
+                    }
+                    let mut bytes = Vec::new();
+                    response
+                        .into_reader()
+                        .take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
+                        .read_to_end(&mut bytes)?;
+                    if bytes.len() > MAX_CONTROL_FRAME_BYTES {
+                        return Err(ControlTransportError::FrameTooLarge);
+                    }
+                    serde_json::from_slice(&bytes)?
+                }
+                ControlTransport::LoopbackJsonLine { writer, reader } => {
+                    writer.write_all(&encoded)?;
+                    writer.write_all(b"\n")?;
+                    writer.flush()?;
+                    if inject_response_loss {
+                        return Err(ControlTransportError::Closed);
+                    }
+                    let mut bytes = Vec::new();
+                    reader
+                        .take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
+                        .read_until(b'\n', &mut bytes)?;
+                    if bytes.is_empty() {
+                        return Err(ControlTransportError::Closed);
+                    }
+                    if bytes.len() > MAX_CONTROL_FRAME_BYTES {
+                        return Err(ControlTransportError::FrameTooLarge);
+                    }
+                    serde_json::from_slice(&bytes)?
+                }
+            };
+            self.requests += 1;
+            Ok(response)
+        }
+    }
+
+    pub fn requests(&self) -> u64 {
+        self.requests
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn should_inject_response_loss(value: &Value) -> bool {
+    static INJECTED: AtomicBool = AtomicBool::new(false);
+    let Ok(expected_operation) = std::env::var("CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION")
+    else {
+        return false;
+    };
+    let operation = value
+        .pointer("/payload/request/type")
+        .or_else(|| value.pointer("/payload/type"))
+        .or_else(|| value.get("type"))
+        .and_then(Value::as_str);
+    operation == Some(expected_operation.as_str())
+        && INJECTED
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+}
+
+pub fn control_api_url(endpoint: &str) -> Result<String, ControlTransportError> {
+    endpoint_api_url(endpoint, CONTROL_API_PATH)
+}
+
+pub fn endpoint_api_url(endpoint: &str, api_path: &str) -> Result<String, ControlTransportError> {
+    let endpoint = endpoint.trim().trim_end_matches('/');
+    if !(endpoint.starts_with("https://") || endpoint.starts_with("http://")) {
+        return Err(ControlTransportError::InvalidEndpoint(endpoint.to_owned()));
+    }
+    if !api_path.starts_with('/') {
+        return Err(ControlTransportError::InvalidEndpoint(api_path.to_owned()));
+    }
+    if endpoint.ends_with(api_path) {
+        Ok(endpoint.to_owned())
+    } else {
+        let base = endpoint
+            .strip_suffix(CONTROL_API_PATH)
+            .or_else(|| endpoint.strip_suffix(LOGIN_API_PATH))
+            .unwrap_or(endpoint);
+        Ok(format!("{base}{api_path}"))
+    }
+}
+
+pub fn endpoint_identity(endpoint: &str) -> Result<String, ControlTransportError> {
+    let endpoint = endpoint.trim();
+    if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
+        if endpoint.starts_with("http://") && !endpoint_is_loopback(endpoint) {
+            return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
+        }
+        return control_api_url(endpoint);
+    }
+    let loopback_address = endpoint
+        .strip_prefix("clusterflux+tcp://")
+        .unwrap_or(endpoint);
+    if endpoint_is_loopback(loopback_address) {
+        return Ok(format!("clusterflux+tcp://{loopback_address}"));
+    }
+    Err(ControlTransportError::InsecureRemote(endpoint.to_owned()))
+}
+
+pub fn endpoint_is_loopback(endpoint: &str) -> bool {
+    let authority = endpoint
+        .trim()
+        .strip_prefix("https://")
+        .or_else(|| endpoint.trim().strip_prefix("http://"))
+        .or_else(|| endpoint.trim().strip_prefix("clusterflux+tcp://"))
+        .unwrap_or(endpoint.trim())
+        .split('/')
+        .next()
+        .unwrap_or_default();
+    let host = if authority.starts_with('[') {
+        authority
+            .strip_prefix('[')
+            .and_then(|value| value.split_once(']'))
+            .map(|(host, _)| host)
+            .unwrap_or(authority)
+    } else {
+        authority
+            .rsplit_once(':')
+            .map(|(host, _)| host)
+            .unwrap_or(authority)
+    };
+    if host.eq_ignore_ascii_case("localhost") {
+        return true;
+    }
+    if host
+        .parse::<IpAddr>()
+        .is_ok_and(|address| address.is_loopback())
+    {
+        return true;
+    }
+    authority
+        .to_socket_addrs()
+        .ok()
+        .is_some_and(|mut addresses| addresses.all(|address| address.ip().is_loopback()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn hosted_endpoints_are_real_https_api_urls() {
+        assert_eq!(
+            control_api_url("https://clusterflux.example").unwrap(),
+            "https://clusterflux.example/api/v1/control"
+        );
+        assert_eq!(
+            endpoint_identity("https://clusterflux.example/api/v1/control").unwrap(),
+            "https://clusterflux.example/api/v1/control"
+        );
+    }
+
+    #[test]
+    fn plaintext_transport_is_restricted_to_loopback() {
+        assert!(endpoint_is_loopback("127.0.0.1:7999"));
+        assert!(endpoint_is_loopback("clusterflux+tcp://127.0.0.1:7999"));
+        assert!(endpoint_is_loopback("http://[::1]:7999"));
+        assert!(matches!(
+            ControlSession::connect("http://example.com:7999"),
+            Err(ControlTransportError::InsecureRemote(_))
+        ));
+        assert!(matches!(
+            ControlSession::connect("example.com:7999"),
+            Err(ControlTransportError::InsecureRemote(_))
+        ));
+    }
+}

crates/clusterflux-coordinator/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "clusterflux-coordinator"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+clusterflux-core = { path = "../clusterflux-core" }
+clusterflux-wasm-runtime = { path = "../clusterflux-wasm-runtime" }
+postgres.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+wasmparser.workspace = true

crates/clusterflux-coordinator/src/agents.rs

@@ -0,0 +1,174 @@
+use clusterflux_core::{
+    verify_agent_workflow_signature, Actor, AgentId, AgentSignedRequest, AgentWorkflowScope,
+    AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId,
+};
+
+use crate::{
+    AgentPublicKeyRecord, Coordinator, CoordinatorError, CredentialRecord, ProjectPermissionRecord,
+};
+
+impl Coordinator {
+    pub fn grant_project_debug(&mut self, tenant: TenantId, project: ProjectId, user: UserId) {
+        self.durable.project_permissions.insert(
+            (tenant.clone(), project.clone(), user.clone()),
+            ProjectPermissionRecord {
+                tenant,
+                project,
+                user,
+                can_debug: true,
+            },
+        );
+    }
+
+    pub fn register_agent_public_key(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        user: UserId,
+        agent: AgentId,
+        public_key: impl Into<String>,
+    ) -> AgentPublicKeyRecord {
+        let key = (tenant.clone(), project.clone(), agent.clone());
+        let version = self
+            .durable
+            .agent_public_keys
+            .get(&key)
+            .map_or(1, |record| record.version.saturating_add(1));
+        let public_key = public_key.into();
+        let public_key_fingerprint = Digest::sha256(&public_key);
+        let record = AgentPublicKeyRecord {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            user: user.clone(),
+            agent: agent.clone(),
+            public_key,
+            public_key_fingerprint: public_key_fingerprint.clone(),
+            version,
+            revoked: false,
+            scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
+            human_account_creation_privilege: false,
+            browser_interaction_required_each_run: false,
+        };
+        self.durable.agent_public_keys.insert(key, record.clone());
+        let subject = format!("agent:{tenant}:{project}:{agent}");
+        self.durable.credentials.insert(
+            subject.clone(),
+            CredentialRecord {
+                subject,
+                tenant,
+                project: Some(project),
+                kind: CredentialKind::PublicKey,
+                public_key_fingerprint: Some(public_key_fingerprint),
+            },
+        );
+        record
+    }
+
+    pub fn list_agent_public_keys(&self, context: &AuthContext) -> Vec<AgentPublicKeyRecord> {
+        self.durable
+            .agent_public_keys
+            .values()
+            .filter(|record| record.tenant == context.tenant && record.project == context.project)
+            .filter(|record| match &context.actor {
+                Actor::User(user) => &record.user == user,
+                Actor::Agent(agent) => &record.agent == agent,
+                Actor::Node(_) | Actor::Task(_) => false,
+            })
+            .cloned()
+            .collect()
+    }
+
+    pub fn revoke_agent_public_key(
+        &mut self,
+        context: &AuthContext,
+        agent: &AgentId,
+    ) -> Result<AgentPublicKeyRecord, CoordinatorError> {
+        let key = (
+            context.tenant.clone(),
+            context.project.clone(),
+            agent.clone(),
+        );
+        let record = self
+            .durable
+            .agent_public_keys
+            .get_mut(&key)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent public key is not registered for this project".to_owned(),
+                )
+            })?;
+        match &context.actor {
+            Actor::User(user) if &record.user == user => {}
+            Actor::User(_) => {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent public key is outside the signed-in user scope".to_owned(),
+                ));
+            }
+            _ => {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent public-key revocation requires a user identity".to_owned(),
+                ));
+            }
+        }
+        record.revoked = true;
+        let subject = format!(
+            "agent:{}:{}:{}",
+            context.tenant, context.project, record.agent
+        );
+        self.durable.credentials.remove(&subject);
+        Ok(record.clone())
+    }
+
+    pub fn authorize_agent_project_run(
+        &self,
+        scope: AgentWorkflowScope<'_>,
+        public_key_fingerprint: Option<&Digest>,
+        payload_digest: &Digest,
+        signature: &AgentSignedRequest,
+        now_epoch_seconds: u64,
+    ) -> Result<AgentPublicKeyRecord, CoordinatorError> {
+        let record = self
+            .durable
+            .agent_public_keys
+            .get(&(
+                scope.tenant.clone(),
+                scope.project.clone(),
+                scope.agent.clone(),
+            ))
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent public key is not registered for this tenant/project".to_owned(),
+                )
+            })?;
+        if record.revoked {
+            return Err(CoordinatorError::Unauthorized(
+                "agent public key has been revoked".to_owned(),
+            ));
+        }
+        if let Some(public_key_fingerprint) = public_key_fingerprint {
+            if &record.public_key_fingerprint != public_key_fingerprint {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent public key fingerprint does not match the registered key".to_owned(),
+                ));
+            }
+        }
+        let max_signature_skew_seconds = 300;
+        if signature
+            .issued_at_epoch_seconds
+            .abs_diff(now_epoch_seconds)
+            > max_signature_skew_seconds
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "agent signed request is expired or outside the allowed clock skew".to_owned(),
+            ));
+        }
+        if !record.scopes.iter().any(|scope| scope == "project:run") {
+            return Err(CoordinatorError::Unauthorized(
+                "agent public key is not scoped for project runs".to_owned(),
+            ));
+        }
+        verify_agent_workflow_signature(&record.public_key, scope, payload_digest, signature)
+            .map_err(CoordinatorError::Unauthorized)?;
+        Ok(record.clone())
+    }
+}

crates/clusterflux-coordinator/src/durable.rs

@@ -0,0 +1,147 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
+};
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TenantRecord {
+    pub id: TenantId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct UserRecord {
+    pub id: UserId,
+    pub tenant: TenantId,
+    pub credential_kind: CredentialKind,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProjectRecord {
+    pub id: ProjectId,
+    pub tenant: TenantId,
+    pub name: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeIdentityRecord {
+    pub id: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub public_key: String,
+    pub enrollment_scope: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CredentialRecord {
+    pub subject: String,
+    pub tenant: TenantId,
+    pub project: Option<ProjectId>,
+    pub kind: CredentialKind,
+    pub public_key_fingerprint: Option<Digest>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CliSessionRecord {
+    pub session_digest: Digest,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub credential_kind: CredentialKind,
+    pub expires_at_epoch_seconds: Option<u64>,
+    pub revoked: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourceProviderConfigRecord {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub provider: SourceProviderKind,
+    pub manifest_digest: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ServicePolicyRecord {
+    pub tenant: TenantId,
+    pub name: String,
+    pub digest: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AccountPolicyState {
+    pub account_status: String,
+    pub suspended: bool,
+    pub disabled: bool,
+    pub deleted: bool,
+    pub manual_review: bool,
+    pub sanitized_reason: Option<String>,
+    pub next_actions: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProjectPermissionRecord {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub can_debug: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AgentPublicKeyRecord {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub user: UserId,
+    pub agent: AgentId,
+    pub public_key: String,
+    pub public_key_fingerprint: Digest,
+    pub version: u64,
+    pub revoked: bool,
+    pub scopes: Vec<String>,
+    pub human_account_creation_privilege: bool,
+    pub browser_interaction_required_each_run: bool,
+}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct DurableState {
+    pub tenants: BTreeMap<TenantId, TenantRecord>,
+    pub users: BTreeMap<UserId, UserRecord>,
+    pub projects: BTreeMap<ProjectId, ProjectRecord>,
+    pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
+    pub credentials: BTreeMap<String, CredentialRecord>,
+    pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
+    pub source_provider_configs:
+        BTreeMap<(TenantId, ProjectId, String), SourceProviderConfigRecord>,
+    pub service_policy_records: BTreeMap<(TenantId, String), ServicePolicyRecord>,
+    pub project_permissions: BTreeMap<(TenantId, ProjectId, UserId), ProjectPermissionRecord>,
+    pub agent_public_keys: BTreeMap<(TenantId, ProjectId, AgentId), AgentPublicKeyRecord>,
+    #[serde(default)]
+    pub artifact_relay: crate::service::ArtifactRelayDurableState,
+}
+
+pub trait DurableStore {
+    fn load(&self) -> DurableState;
+    fn save(&mut self, state: DurableState);
+}
+
+pub trait FallibleDurableStore {
+    type Error;
+
+    fn load_state(&mut self) -> Result<DurableState, Self::Error>;
+    fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error>;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct InMemoryDurableStore {
+    state: DurableState,
+}
+
+impl DurableStore for InMemoryDurableStore {
+    fn load(&self) -> DurableState {
+        self.state.clone()
+    }
+
+    fn save(&mut self, state: DurableState) {
+        self.state = state;
+    }
+}

crates/clusterflux-coordinator/src/main.rs

@@ -0,0 +1,62 @@
+use std::io::Write;
+
+use clusterflux_coordinator::{service::bind_listener, CoordinatorService};
+use clusterflux_core::{ProjectId, TenantId, UserId};
+use serde_json::json;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let mut listen = "127.0.0.1:0".to_owned();
+    let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
+        .ok()
+        .as_deref()
+        == Some("1");
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        if arg == "--listen" {
+            listen = args.next().ok_or("--listen requires an address")?;
+        } else if arg == "--allow-local-trusted-loopback" {
+            allow_local_trusted = true;
+        }
+    }
+
+    let (listener, addr) = bind_listener(&listen)?;
+    let database_url = std::env::var("DATABASE_URL").ok();
+    let mut service = CoordinatorService::new_with_database_url(1, database_url.as_deref())?;
+    let self_hosted_session_secret = std::env::var("CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET")
+        .ok()
+        .filter(|secret| !secret.trim().is_empty());
+    if let Some(session_secret) = self_hosted_session_secret.as_deref() {
+        service.issue_cli_session(
+            TenantId::new(
+                std::env::var("CLUSTERFLUX_SELF_HOSTED_TENANT")
+                    .unwrap_or_else(|_| "tenant".to_owned()),
+            ),
+            ProjectId::new(
+                std::env::var("CLUSTERFLUX_SELF_HOSTED_PROJECT")
+                    .unwrap_or_else(|_| "project".to_owned()),
+            ),
+            UserId::new(
+                std::env::var("CLUSTERFLUX_SELF_HOSTED_USER").unwrap_or_else(|_| "user".to_owned()),
+            ),
+            session_secret,
+            None,
+        )?;
+    }
+    println!(
+        "{}",
+        json!({
+            "listen": addr.to_string(),
+            "client_authority": if allow_local_trusted { "local_trusted_loopback" } else { "strict" },
+            "self_hosted_session_bootstrapped": self_hosted_session_secret.is_some(),
+            "durable_store": service.durable_store_kind(),
+        })
+    );
+    std::io::stdout().flush()?;
+
+    if allow_local_trusted {
+        service.serve_tcp_local_trusted(listener)?;
+    } else {
+        service.serve_tcp(listener)?;
+    }
+    Ok(())
+}

crates/clusterflux-coordinator/src/postgres_store.rs

@@ -0,0 +1,604 @@
+use postgres::{Client, NoTls};
+use serde::{de::DeserializeOwned, Serialize};
+use thiserror::Error;
+
+use crate::{
+    AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
+    DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
+    ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct PostgresTable {
+    pub name: &'static str,
+    pub durable_record: &'static str,
+    pub restart_surviving: bool,
+}
+
+pub const POSTGRES_DURABLE_TABLES: &[PostgresTable] = &[
+    PostgresTable {
+        name: "clusterflux_artifact_relay_state",
+        durable_record: "artifact relay reservations and usage",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_tenants",
+        durable_record: "tenants",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_users",
+        durable_record: "users",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_projects",
+        durable_record: "projects",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_node_identities",
+        durable_record: "node identities",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_credentials",
+        durable_record: "credentials",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_cli_sessions",
+        durable_record: "CLI sessions",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_agent_public_keys",
+        durable_record: "agent public keys",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_source_provider_configs",
+        durable_record: "source-provider configuration",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_service_policy_records",
+        durable_record: "durable service policy records",
+        restart_surviving: true,
+    },
+    PostgresTable {
+        name: "clusterflux_project_permissions",
+        durable_record: "explicit project permissions",
+        restart_surviving: true,
+    },
+];
+
+#[derive(Debug, Error)]
+pub enum PostgresStoreError {
+    #[error("postgres durable store error: {0}")]
+    Postgres(#[from] postgres::Error),
+    #[error("durable state serialization error: {0}")]
+    Serialization(#[from] serde_json::Error),
+}
+
+pub struct PostgresDurableStore {
+    client: Client,
+}
+
+impl PostgresDurableStore {
+    pub fn connect(connection_string: &str) -> Result<Self, PostgresStoreError> {
+        let mut store = Self {
+            client: Client::connect(connection_string, NoTls)?,
+        };
+        store.migrate()?;
+        Ok(store)
+    }
+
+    pub fn from_client(client: Client) -> Result<Self, PostgresStoreError> {
+        let mut store = Self { client };
+        store.migrate()?;
+        Ok(store)
+    }
+
+    pub fn schema_sql() -> &'static str {
+        POSTGRES_SCHEMA_SQL
+    }
+
+    pub fn durable_tables() -> &'static [PostgresTable] {
+        POSTGRES_DURABLE_TABLES
+    }
+
+    pub fn migrate(&mut self) -> Result<(), PostgresStoreError> {
+        self.client.batch_execute(Self::schema_sql())?;
+        Ok(())
+    }
+
+    fn query_records<T: DeserializeOwned>(
+        &mut self,
+        sql: &str,
+    ) -> Result<Vec<T>, PostgresStoreError> {
+        self.client
+            .query(sql, &[])?
+            .into_iter()
+            .map(|row| {
+                let value: serde_json::Value = row.get("record");
+                Ok(serde_json::from_value(value)?)
+            })
+            .collect()
+    }
+
+    fn record_value(record: &impl Serialize) -> Result<serde_json::Value, PostgresStoreError> {
+        Ok(serde_json::to_value(record)?)
+    }
+}
+
+impl FallibleDurableStore for PostgresDurableStore {
+    type Error = PostgresStoreError;
+
+    fn load_state(&mut self) -> Result<DurableState, Self::Error> {
+        let mut state = DurableState::default();
+
+        for record in self.query_records::<TenantRecord>(
+            "SELECT record FROM clusterflux_tenants ORDER BY tenant_id",
+        )? {
+            state.tenants.insert(record.id.clone(), record);
+        }
+        for record in self
+            .query_records::<UserRecord>("SELECT record FROM clusterflux_users ORDER BY user_id")?
+        {
+            state.users.insert(record.id.clone(), record);
+        }
+        for record in self.query_records::<ProjectRecord>(
+            "SELECT record FROM clusterflux_projects ORDER BY project_id",
+        )? {
+            state.projects.insert(record.id.clone(), record);
+        }
+        for record in self.query_records::<NodeIdentityRecord>(
+            "SELECT record FROM clusterflux_node_identities ORDER BY node_id",
+        )? {
+            state.node_identities.insert(record.id.clone(), record);
+        }
+        for record in self.query_records::<CredentialRecord>(
+            "SELECT record FROM clusterflux_credentials ORDER BY subject",
+        )? {
+            state.credentials.insert(record.subject.clone(), record);
+        }
+        for record in self.query_records::<CliSessionRecord>(
+            "SELECT record FROM clusterflux_cli_sessions ORDER BY session_digest",
+        )? {
+            state
+                .cli_sessions
+                .insert(record.session_digest.clone(), record);
+        }
+        for record in self.query_records::<AgentPublicKeyRecord>(
+            "SELECT record FROM clusterflux_agent_public_keys ORDER BY tenant_id, project_id, agent_id",
+        )? {
+            state.agent_public_keys.insert(
+                (
+                    record.tenant.clone(),
+                    record.project.clone(),
+                    record.agent.clone(),
+                ),
+                record,
+            );
+        }
+        for record in self.query_records::<SourceProviderConfigRecord>(
+            "SELECT record FROM clusterflux_source_provider_configs ORDER BY tenant_id, project_id, provider_key",
+        )? {
+            let provider_key = format!("{:?}", record.provider);
+            state.source_provider_configs.insert(
+                (record.tenant.clone(), record.project.clone(), provider_key),
+                record,
+            );
+        }
+        for record in self.query_records::<ServicePolicyRecord>(
+            "SELECT record FROM clusterflux_service_policy_records ORDER BY tenant_id, name",
+        )? {
+            state
+                .service_policy_records
+                .insert((record.tenant.clone(), record.name.clone()), record);
+        }
+        for record in self.query_records::<ProjectPermissionRecord>(
+            "SELECT record FROM clusterflux_project_permissions ORDER BY tenant_id, project_id, user_id",
+        )? {
+            state.project_permissions.insert(
+                (
+                    record.tenant.clone(),
+                    record.project.clone(),
+                    record.user.clone(),
+                ),
+                record,
+            );
+        }
+
+        if let Some(relay) = self
+            .query_records::<ArtifactRelayDurableState>(
+                "SELECT record FROM clusterflux_artifact_relay_state WHERE singleton = TRUE",
+            )?
+            .into_iter()
+            .next()
+        {
+            state.artifact_relay = relay;
+        }
+
+        Ok(state)
+    }
+
+    fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
+        let mut tx = self.client.transaction()?;
+        tx.batch_execute(
+            "
+            DELETE FROM clusterflux_artifact_relay_state;
+            DELETE FROM clusterflux_project_permissions;
+            DELETE FROM clusterflux_service_policy_records;
+            DELETE FROM clusterflux_source_provider_configs;
+            DELETE FROM clusterflux_agent_public_keys;
+            DELETE FROM clusterflux_cli_sessions;
+            DELETE FROM clusterflux_credentials;
+            DELETE FROM clusterflux_node_identities;
+            DELETE FROM clusterflux_projects;
+            DELETE FROM clusterflux_users;
+            DELETE FROM clusterflux_tenants;
+            ",
+        )?;
+
+        let relay = Self::record_value(&state.artifact_relay)?;
+        tx.execute(
+            "INSERT INTO clusterflux_artifact_relay_state (singleton, record) VALUES (TRUE, $1)",
+            &[&relay],
+        )?;
+
+        for record in state.tenants.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_tenants (tenant_id, record) VALUES ($1, $2)",
+                &[&record.id.as_str(), &value],
+            )?;
+        }
+        for record in state.users.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_users (user_id, tenant_id, record) VALUES ($1, $2, $3)",
+                &[&record.id.as_str(), &record.tenant.as_str(), &value],
+            )?;
+        }
+        for record in state.projects.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_projects (project_id, tenant_id, record) VALUES ($1, $2, $3)",
+                &[&record.id.as_str(), &record.tenant.as_str(), &value],
+            )?;
+        }
+        for record in state.node_identities.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_node_identities (node_id, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
+                &[
+                    &record.id.as_str(),
+                    &record.tenant.as_str(),
+                    &record.project.as_str(),
+                    &value,
+                ],
+            )?;
+        }
+        for record in state.credentials.values() {
+            let value = Self::record_value(record)?;
+            let project_id = record.project.as_ref().map(|project| project.as_str());
+            tx.execute(
+                "INSERT INTO clusterflux_credentials (subject, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
+                &[&record.subject.as_str(), &record.tenant.as_str(), &project_id, &value],
+            )?;
+        }
+        for record in state.cli_sessions.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_cli_sessions (session_digest, tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4, $5)",
+                &[
+                    &record.session_digest.as_str(),
+                    &record.tenant.as_str(),
+                    &record.project.as_str(),
+                    &record.user.as_str(),
+                    &value,
+                ],
+            )?;
+        }
+        for record in state.agent_public_keys.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_agent_public_keys (tenant_id, project_id, user_id, agent_id, record) VALUES ($1, $2, $3, $4, $5)",
+                &[
+                    &record.tenant.as_str(),
+                    &record.project.as_str(),
+                    &record.user.as_str(),
+                    &record.agent.as_str(),
+                    &value,
+                ],
+            )?;
+        }
+        for ((_, _, provider_key), record) in &state.source_provider_configs {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_source_provider_configs (tenant_id, project_id, provider_key, record) VALUES ($1, $2, $3, $4)",
+                &[
+                    &record.tenant.as_str(),
+                    &record.project.as_str(),
+                    &provider_key.as_str(),
+                    &value,
+                ],
+            )?;
+        }
+        for record in state.service_policy_records.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_service_policy_records (tenant_id, name, record) VALUES ($1, $2, $3)",
+                &[&record.tenant.as_str(), &record.name.as_str(), &value],
+            )?;
+        }
+        for record in state.project_permissions.values() {
+            let value = Self::record_value(record)?;
+            tx.execute(
+                "INSERT INTO clusterflux_project_permissions (tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4)",
+                &[
+                    &record.tenant.as_str(),
+                    &record.project.as_str(),
+                    &record.user.as_str(),
+                    &value,
+                ],
+            )?;
+        }
+
+        tx.commit()?;
+        Ok(())
+    }
+}
+
+const POSTGRES_SCHEMA_SQL: &str = r#"
+CREATE TABLE IF NOT EXISTS clusterflux_artifact_relay_state (
+    singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton),
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_tenants (
+    tenant_id TEXT PRIMARY KEY,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_users (
+    user_id TEXT PRIMARY KEY,
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_projects (
+    project_id TEXT PRIMARY KEY,
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
+    node_id TEXT PRIMARY KEY,
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_credentials (
+    subject TEXT PRIMARY KEY,
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
+    session_digest TEXT PRIMARY KEY,
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
+    record JSONB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_agent_public_keys (
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
+    agent_id TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, agent_id)
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_source_provider_configs (
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    provider_key TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, provider_key)
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_service_policy_records (
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    name TEXT NOT NULL,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, name)
+);
+
+CREATE TABLE IF NOT EXISTS clusterflux_project_permissions (
+    tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
+    project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
+    user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
+    record JSONB NOT NULL,
+    PRIMARY KEY (tenant_id, project_id, user_id)
+);
+"#;
+
+#[cfg(test)]
+mod tests {
+    use clusterflux_core::{
+        CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
+    };
+
+    use super::*;
+    use crate::{Coordinator, DurableStore, FallibleDurableStore, InMemoryDurableStore};
+
+    #[test]
+    fn postgres_schema_contains_only_restart_surviving_durable_tables() {
+        let names = PostgresDurableStore::durable_tables()
+            .iter()
+            .map(|table| table.name)
+            .collect::<Vec<_>>();
+
+        assert_eq!(names.len(), 11);
+        assert!(names.contains(&"clusterflux_artifact_relay_state"));
+        assert!(names.contains(&"clusterflux_tenants"));
+        assert!(names.contains(&"clusterflux_users"));
+        assert!(names.contains(&"clusterflux_projects"));
+        assert!(names.contains(&"clusterflux_node_identities"));
+        assert!(names.contains(&"clusterflux_credentials"));
+        assert!(names.contains(&"clusterflux_cli_sessions"));
+        assert!(names.contains(&"clusterflux_agent_public_keys"));
+        assert!(names.contains(&"clusterflux_source_provider_configs"));
+        assert!(names.contains(&"clusterflux_service_policy_records"));
+        assert!(names.contains(&"clusterflux_project_permissions"));
+        assert!(PostgresDurableStore::durable_tables()
+            .iter()
+            .all(|table| table.restart_surviving));
+
+        for runtime_only in [
+            "active_process",
+            "virtual_thread",
+            "scheduler_state",
+            "debug_epoch",
+            "vfs_manifest",
+            "transient_artifact_location",
+        ] {
+            assert!(
+                !PostgresDurableStore::schema_sql().contains(runtime_only),
+                "{runtime_only} must remain outside Postgres durable state"
+            );
+        }
+    }
+
+    #[test]
+    fn fallible_store_boot_uses_durable_state_and_still_drops_live_processes() {
+        #[derive(Default)]
+        struct FallibleMemoryStore {
+            inner: InMemoryDurableStore,
+        }
+
+        impl FallibleDurableStore for FallibleMemoryStore {
+            type Error = std::convert::Infallible;
+
+            fn load_state(&mut self) -> Result<DurableState, Self::Error> {
+                Ok(self.inner.load())
+            }
+
+            fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
+                self.inner.save(state.clone());
+                Ok(())
+            }
+        }
+
+        let mut store = FallibleMemoryStore::default();
+        let mut first = Coordinator::try_boot(&mut store, 1).unwrap();
+        first.upsert_tenant(TenantId::from("tenant"));
+        first.upsert_user(
+            TenantId::from("tenant"),
+            UserId::from("user"),
+            CredentialKind::CliDeviceSession,
+        );
+        first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
+        first.enroll_node(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            NodeId::from("node"),
+            "public-key",
+            "node:attach",
+        );
+        first.upsert_source_provider_config(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            SourceProviderKind::Git,
+            Digest::sha256("git-manifest"),
+        );
+        first.start_process(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            clusterflux_core::ProcessId::from("process"),
+        );
+        first.try_persist(&mut store).unwrap();
+
+        let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
+
+        assert!(restarted.project(&ProjectId::from("project")).is_some());
+        assert!(restarted.node_identity(&NodeId::from("node")).is_some());
+        assert_eq!(restarted.active_process_count(), 0);
+    }
+
+    #[test]
+    fn postgres_round_trip_runs_when_dsn_is_configured() {
+        let Ok(dsn) = std::env::var("CLUSTERFLUX_TEST_POSTGRES") else {
+            return;
+        };
+
+        let mut store = PostgresDurableStore::connect(&dsn).unwrap();
+        let mut state = DurableState::default();
+        state.tenants.insert(
+            TenantId::from("tenant"),
+            TenantRecord {
+                id: TenantId::from("tenant"),
+            },
+        );
+        state.projects.insert(
+            ProjectId::from("project"),
+            ProjectRecord {
+                id: ProjectId::from("project"),
+                tenant: TenantId::from("tenant"),
+                name: "demo".to_owned(),
+            },
+        );
+        state.users.insert(
+            UserId::from("user"),
+            UserRecord {
+                id: UserId::from("user"),
+                tenant: TenantId::from("tenant"),
+                credential_kind: CredentialKind::CliDeviceSession,
+            },
+        );
+        let session_digests = [
+            Digest::sha256("postgres-round-trip-session-one"),
+            Digest::sha256("postgres-round-trip-session-two"),
+        ];
+        for session_digest in &session_digests {
+            state.cli_sessions.insert(
+                session_digest.clone(),
+                CliSessionRecord {
+                    session_digest: session_digest.clone(),
+                    tenant: TenantId::from("tenant"),
+                    project: ProjectId::from("project"),
+                    user: UserId::from("user"),
+                    credential_kind: CredentialKind::CliDeviceSession,
+                    expires_at_epoch_seconds: None,
+                    revoked: false,
+                },
+            );
+            let subject = format!("cli-session:{}", session_digest.as_str());
+            state.credentials.insert(
+                subject.clone(),
+                CredentialRecord {
+                    subject,
+                    tenant: TenantId::from("tenant"),
+                    project: Some(ProjectId::from("project")),
+                    kind: CredentialKind::CliDeviceSession,
+                    public_key_fingerprint: None,
+                },
+            );
+        }
+        store.save_state(&state).unwrap();
+        let loaded = store.load_state().unwrap();
+
+        assert!(loaded.projects.contains_key(&ProjectId::from("project")));
+        assert!(session_digests
+            .iter()
+            .all(|session_digest| loaded.cli_sessions.contains_key(session_digest)));
+        assert_eq!(loaded.credentials.len(), 2);
+    }
+}

crates/clusterflux-coordinator/src/service.rs

@@ -0,0 +1,571 @@
+// Request handlers intentionally spell out their deserialized protocol fields.
+// Keeping those authority and payload values explicit at this boundary is safer
+// than passing an unvalidated wire request deeper into the service.
+#![allow(clippy::too_many_arguments)]
+
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
+    NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
+    RateLimit, TenantId, TransportError, UserId,
+};
+use thiserror::Error;
+
+use crate::{Coordinator, CoordinatorError};
+
+mod admin;
+mod artifacts;
+mod authenticated;
+mod authorization;
+mod debug;
+mod debug_requests;
+mod durable_runtime;
+mod keys;
+mod logs;
+mod main_runtime;
+mod nodes;
+mod panels;
+mod process_launch;
+mod processes;
+mod protocol;
+mod quota;
+mod relay;
+mod routing;
+mod signed_nodes;
+mod tcp;
+mod wire_protocol;
+use authorization::authorize_authenticated_user_operation;
+use durable_runtime::RuntimeDurableStore;
+use keys::{
+    artifact_id_from_path, enrollment_grant_key, EnrollmentGrantKey, PanelStopKey,
+    ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
+};
+pub use protocol::{
+    ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
+    CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
+    DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
+    TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
+    TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
+    TaskTerminalState, VirtualProcessStatus, WorkflowActor,
+};
+pub use quota::CoordinatorQuotaConfiguration;
+pub use relay::{
+    ArtifactRelayConfiguration, ArtifactRelayDurableState, ArtifactRelayError, ArtifactRelayUsage,
+};
+pub use tcp::{bind_listener, ClientAuthorityMode};
+pub use wire_protocol::CoordinatorWireRequest;
+
+const MAX_TASK_LOG_TAIL_BYTES: usize = 256 * 1024;
+const DEBUG_CONTROL_READ_BYTES: u64 = 1024;
+const MAX_REPLAY_NONCES_PER_AUTHORITY: usize = 1_024;
+const NODE_SIGNATURE_WINDOW_SECONDS: u64 = 30;
+const MAX_NODE_REPLAY_NONCES_PER_AUTHORITY: usize = 4_096;
+const MAX_ENROLLMENT_GRANTS_PER_PROJECT: usize = 64;
+const MAX_TASK_EVENTS_PER_PROCESS: usize = 128;
+const MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS: usize = 256;
+const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
+const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
+const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
+const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
+const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
+const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
+const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
+const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
+fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
+    requested.clamp(1, maximum)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CoordinatorMainRuntimeConfiguration {
+    pub fuel_units_per_second: u64,
+    pub fuel_burst_seconds: u64,
+    pub memory_bytes: usize,
+    pub nested_join_timeout_ms: u64,
+    pub max_active_mains: usize,
+    pub max_wakeups_per_minute: u64,
+    pub max_output_bytes: usize,
+    pub max_state_bytes: usize,
+}
+
+impl Default for CoordinatorMainRuntimeConfiguration {
+    fn default() -> Self {
+        Self {
+            fuel_units_per_second: 10_000_000,
+            fuel_burst_seconds: 60,
+            memory_bytes: 256 * 1024 * 1024,
+            nested_join_timeout_ms: 24 * 60 * 60 * 1_000,
+            max_active_mains: usize::MAX,
+            max_wakeups_per_minute: 6_000,
+            max_output_bytes: MAX_TASK_LOG_TAIL_BYTES,
+            max_state_bytes: clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CoordinatorAdmission {
+    pub workflow_placement_allowed: bool,
+    pub max_node_enrollment_ttl_seconds: u64,
+    pub max_artifact_download_ttl_seconds: u64,
+}
+
+impl Default for CoordinatorAdmission {
+    fn default() -> Self {
+        Self {
+            workflow_placement_allowed: true,
+            max_node_enrollment_ttl_seconds: 15 * 60,
+            max_artifact_download_ttl_seconds: 15 * 60,
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum CoordinatorServiceError {
+    #[error("coordinator protocol I/O error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("coordinator protocol JSON error: {0}")]
+    Json(#[from] serde_json::Error),
+    #[error("coordinator protocol error: {0}")]
+    Protocol(String),
+    #[error("coordinator request failed: {0}")]
+    Coordinator(#[from] CoordinatorError),
+    #[error("artifact download request failed: {0}")]
+    Download(#[from] clusterflux_core::DownloadError),
+    #[error("scheduler placement failed: {0}")]
+    Scheduler(#[from] clusterflux_core::PlacementError),
+    #[error("transport request failed: {0}")]
+    Transport(#[from] TransportError),
+    #[error("resource limit failed: {0}")]
+    Resource(#[from] LimitError),
+    #[error("operator panel request failed: {0}")]
+    Panel(#[from] clusterflux_core::PanelError),
+    #[error("invalid node capability report: {0}")]
+    CapabilityReport(#[from] CapabilityReportError),
+    #[error("invalid VFS artifact path reported by node: {0}")]
+    InvalidArtifactPath(String),
+    #[error("invalid task log tail reported by node: {0}")]
+    InvalidTaskLogTail(String),
+    #[error("durable coordinator state failed: {0}")]
+    Durable(String),
+}
+
+pub struct CoordinatorService {
+    coordinator: Coordinator,
+    store: RuntimeDurableStore,
+    node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
+    node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
+    node_stale_after_seconds: u64,
+    debug_freeze_timeout: std::time::Duration,
+    enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
+    task_events: VecDeque<TaskCompletionEvent>,
+    process_scope_history: VecDeque<ProcessControlKey>,
+    debug_audit_events: VecDeque<DebugAuditEvent>,
+    debug_epochs: BTreeMap<ProcessControlKey, u64>,
+    debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
+    debug_breakpoints: BTreeMap<ProcessControlKey, debug::DebugBreakpointPlan>,
+    debug_commands: BTreeMap<TaskControlKey, debug::DebugPendingCommand>,
+    task_assignments: BTreeMap<TaskAssignmentKey, VecDeque<TaskAssignment>>,
+    task_restart_checkpoints: BTreeMap<TaskRestartKey, processes::TaskRestartCheckpoint>,
+    task_restart_checkpoint_order: VecDeque<TaskRestartKey>,
+    task_attempts: BTreeMap<TaskRestartKey, Vec<protocol::TaskAttemptSnapshot>>,
+    restart_launches: BTreeSet<TaskRestartKey>,
+    main_runtime: main_runtime::CoordinatorMainRuntime,
+    pending_task_launches: VecDeque<processes::PendingTaskLaunch>,
+    task_placements: BTreeMap<TaskControlKey, Placement>,
+    active_tasks: BTreeSet<TaskControlKey>,
+    task_cancellations: BTreeSet<TaskControlKey>,
+    task_aborts: BTreeSet<TaskControlKey>,
+    process_cancellations: BTreeSet<ProcessControlKey>,
+    process_aborts: BTreeSet<ProcessControlKey>,
+    agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
+    node_replay_nonces: BTreeMap<(NodeId, String), u64>,
+    panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
+    stopped_panels: BTreeSet<PanelStopKey>,
+    panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
+    artifact_registry: ArtifactRegistry,
+    artifact_reverse_transfers: BTreeMap<String, artifacts::ArtifactReverseTransfer>,
+    artifact_transfer_by_token: BTreeMap<Digest, String>,
+    artifact_relay: relay::ArtifactRelayLedger,
+    transport: NativeQuicTransport,
+    quota: quota::CoordinatorQuota,
+    admission: CoordinatorAdmission,
+    #[cfg(test)]
+    server_time_override: Option<u64>,
+    admin_token_digest: Option<Digest>,
+    admin_replay_nonces: BTreeMap<String, u64>,
+}
+
+impl CoordinatorService {
+    pub fn configure_artifact_relay(
+        &mut self,
+        configuration: ArtifactRelayConfiguration,
+    ) -> Result<(), CoordinatorServiceError> {
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let mut candidate = self.artifact_relay.clone();
+        candidate
+            .configure(configuration)
+            .map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
+        candidate.reconcile_after_restart(now_epoch_seconds);
+        self.commit_artifact_relay(candidate)
+    }
+
+    pub fn artifact_relay_usage(&self) -> ArtifactRelayUsage {
+        self.artifact_relay.usage()
+    }
+
+    fn commit_artifact_relay(
+        &mut self,
+        candidate: relay::ArtifactRelayLedger,
+    ) -> Result<(), CoordinatorServiceError> {
+        let previous = self.artifact_relay.clone();
+        let previous_state = previous.durable_state();
+        self.artifact_relay = candidate;
+        self.coordinator
+            .set_artifact_relay_state(self.artifact_relay.durable_state());
+        if let Err(error) = self.persist_durable_state() {
+            self.artifact_relay = previous;
+            self.coordinator.set_artifact_relay_state(previous_state);
+            return Err(error);
+        }
+        Ok(())
+    }
+
+    fn mutate_artifact_relay<T>(
+        &mut self,
+        mutation: impl FnOnce(&mut relay::ArtifactRelayLedger) -> Result<T, ArtifactRelayError>,
+    ) -> Result<T, CoordinatorServiceError> {
+        let mut candidate = self.artifact_relay.clone();
+        let result = mutation(&mut candidate)
+            .map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
+        self.commit_artifact_relay(candidate)?;
+        Ok(result)
+    }
+
+    pub fn set_debug_freeze_timeout(&mut self, timeout: std::time::Duration) {
+        self.debug_freeze_timeout = timeout.max(std::time::Duration::from_millis(1));
+    }
+
+    pub fn configure_coordinator_main_runtime(
+        &mut self,
+        configuration: CoordinatorMainRuntimeConfiguration,
+    ) -> Result<(), CoordinatorServiceError> {
+        self.main_runtime.configure(configuration)
+    }
+
+    pub fn record_service_policy(
+        &mut self,
+        tenant: TenantId,
+        name: impl Into<String>,
+        digest: Digest,
+    ) -> Result<crate::ServicePolicyRecord, CoordinatorServiceError> {
+        let name = name.into();
+        self.coordinator
+            .upsert_service_policy_record(tenant.clone(), name.clone(), digest);
+        self.persist_durable_state()?;
+        Ok(self
+            .coordinator
+            .service_policy_record(&tenant, &name)
+            .expect("service policy record was persisted immediately after insertion")
+            .clone())
+    }
+
+    pub(super) fn authorize_node_for_process_or_termination(
+        &self,
+        node: &NodeId,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let process_key = keys::process_control_key(tenant, project, process);
+        if self.process_cancellations.contains(&process_key)
+            || self.process_aborts.contains(&process_key)
+        {
+            let identity = self
+                .coordinator
+                .node_identity(node)
+                .ok_or(CoordinatorError::UnknownNode)?;
+            if &identity.tenant != tenant || &identity.project != project {
+                return Err(CoordinatorError::Unauthorized(
+                    "node process-control request is outside its enrolled tenant/project scope"
+                        .to_owned(),
+                )
+                .into());
+            }
+            return Ok(());
+        }
+        self.coordinator
+            .authorize_node_for_process(node, tenant, project, process)?;
+        Ok(())
+    }
+
+    pub fn new(coordinator_epoch: u64) -> Self {
+        Self::new_with_optional_admin_token_and_admission(
+            coordinator_epoch,
+            std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
+            CoordinatorAdmission::default(),
+        )
+    }
+
+    pub fn new_with_admin_token(coordinator_epoch: u64, admin_token: impl Into<String>) -> Self {
+        Self::new_with_optional_admin_token_and_admission(
+            coordinator_epoch,
+            Some(admin_token.into()),
+            CoordinatorAdmission::default(),
+        )
+    }
+
+    pub fn new_with_admission(coordinator_epoch: u64, admission: CoordinatorAdmission) -> Self {
+        Self::new_with_optional_admin_token_and_admission(
+            coordinator_epoch,
+            std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
+            admission,
+        )
+    }
+
+    fn new_with_optional_admin_token_and_admission(
+        coordinator_epoch: u64,
+        admin_token: Option<String>,
+        admission: CoordinatorAdmission,
+    ) -> Self {
+        Self::try_new_with_optional_admin_token_admission_and_database_url(
+            coordinator_epoch,
+            admin_token,
+            admission,
+            None,
+            CoordinatorQuotaConfiguration::default(),
+        )
+        .expect("in-memory durable coordinator store initialization cannot fail")
+    }
+
+    pub fn new_with_database_url(
+        coordinator_epoch: u64,
+        database_url: Option<&str>,
+    ) -> Result<Self, CoordinatorServiceError> {
+        Self::try_new_with_optional_admin_token_admission_and_database_url(
+            coordinator_epoch,
+            std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
+            CoordinatorAdmission::default(),
+            database_url,
+            CoordinatorQuotaConfiguration::default(),
+        )
+    }
+
+    pub fn new_with_admin_token_and_database_url(
+        coordinator_epoch: u64,
+        admin_token: impl Into<String>,
+        database_url: Option<&str>,
+    ) -> Result<Self, CoordinatorServiceError> {
+        Self::new_with_admin_token_database_url_and_quota(
+            coordinator_epoch,
+            admin_token,
+            database_url,
+            CoordinatorQuotaConfiguration::default(),
+        )
+    }
+
+    pub fn new_with_admin_token_database_url_and_quota(
+        coordinator_epoch: u64,
+        admin_token: impl Into<String>,
+        database_url: Option<&str>,
+        quota_configuration: CoordinatorQuotaConfiguration,
+    ) -> Result<Self, CoordinatorServiceError> {
+        Self::try_new_with_optional_admin_token_admission_and_database_url(
+            coordinator_epoch,
+            Some(admin_token.into()),
+            CoordinatorAdmission::default(),
+            database_url,
+            quota_configuration,
+        )
+    }
+
+    fn try_new_with_optional_admin_token_admission_and_database_url(
+        coordinator_epoch: u64,
+        admin_token: Option<String>,
+        admission: CoordinatorAdmission,
+        database_url: Option<&str>,
+        quota_configuration: CoordinatorQuotaConfiguration,
+    ) -> Result<Self, CoordinatorServiceError> {
+        let mut store = RuntimeDurableStore::from_database_url(database_url)
+            .map_err(CoordinatorServiceError::Durable)?;
+        let coordinator = Coordinator::try_boot(&mut store, coordinator_epoch)
+            .map_err(CoordinatorServiceError::Durable)?;
+        let artifact_relay_state = coordinator.artifact_relay_state().clone();
+        let admin_token_digest = admin_token
+            .filter(|token| !token.trim().is_empty())
+            .map(Digest::sha256);
+        Ok(Self {
+            coordinator,
+            store,
+            node_descriptors: BTreeMap::new(),
+            node_last_seen_epoch_seconds: BTreeMap::new(),
+            node_stale_after_seconds: std::env::var("CLUSTERFLUX_NODE_STALE_AFTER_SECONDS")
+                .ok()
+                .and_then(|value| value.parse::<u64>().ok())
+                .filter(|seconds| *seconds > 0)
+                .unwrap_or(DEFAULT_NODE_STALE_AFTER_SECONDS),
+            debug_freeze_timeout: std::time::Duration::from_secs(5),
+            enrollment_grants: BTreeMap::new(),
+            task_events: VecDeque::new(),
+            process_scope_history: VecDeque::new(),
+            debug_audit_events: VecDeque::new(),
+            debug_epochs: BTreeMap::new(),
+            debug_epoch_runtime: BTreeMap::new(),
+            debug_breakpoints: BTreeMap::new(),
+            debug_commands: BTreeMap::new(),
+            task_assignments: BTreeMap::new(),
+            task_restart_checkpoints: BTreeMap::new(),
+            task_restart_checkpoint_order: VecDeque::new(),
+            task_attempts: BTreeMap::new(),
+            restart_launches: BTreeSet::new(),
+            main_runtime: main_runtime::CoordinatorMainRuntime::default(),
+            pending_task_launches: VecDeque::new(),
+            task_placements: BTreeMap::new(),
+            active_tasks: BTreeSet::new(),
+            task_cancellations: BTreeSet::new(),
+            task_aborts: BTreeSet::new(),
+            process_cancellations: BTreeSet::new(),
+            process_aborts: BTreeSet::new(),
+            agent_replay_nonces: BTreeMap::new(),
+            node_replay_nonces: BTreeMap::new(),
+            panel_snapshots: BTreeMap::new(),
+            stopped_panels: BTreeSet::new(),
+            panel_event_limits: BTreeMap::new(),
+            artifact_registry: ArtifactRegistry::default(),
+            artifact_reverse_transfers: BTreeMap::new(),
+            artifact_transfer_by_token: BTreeMap::new(),
+            artifact_relay: relay::ArtifactRelayLedger::from_durable(
+                ArtifactRelayConfiguration::default(),
+                artifact_relay_state,
+            ),
+            transport: NativeQuicTransport,
+            quota: quota::CoordinatorQuota::new(quota_configuration),
+            admission,
+            #[cfg(test)]
+            server_time_override: None,
+            admin_token_digest,
+            admin_replay_nonces: BTreeMap::new(),
+        })
+    }
+
+    pub fn durable_store_kind(&self) -> &'static str {
+        self.store.kind()
+    }
+
+    fn persist_durable_state(&mut self) -> Result<(), CoordinatorServiceError> {
+        self.coordinator
+            .try_persist(&mut self.store)
+            .map_err(CoordinatorServiceError::Durable)
+    }
+
+    fn current_epoch_seconds(&self) -> Result<u64, CoordinatorServiceError> {
+        #[cfg(test)]
+        if let Some(now) = self.server_time_override {
+            return Ok(now);
+        }
+        SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|duration| duration.as_secs())
+            .map_err(|err| CoordinatorServiceError::Protocol(format!("system clock error: {err}")))
+    }
+
+    fn handle_quota_status(
+        &self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let durable_project = self.coordinator.project(&project).ok_or_else(|| {
+            CoordinatorError::Unauthorized("quota status requires an existing project".to_owned())
+        })?;
+        if durable_project.tenant != tenant {
+            return Err(CoordinatorError::Unauthorized(
+                "quota status project is outside the tenant scope".to_owned(),
+            )
+            .into());
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let status = self
+            .quota
+            .project_status(&tenant, &project, now_epoch_seconds);
+        Ok(CoordinatorResponse::QuotaStatus {
+            tenant,
+            project,
+            actor,
+            policy_label: status.policy_label,
+            limits: status.limits,
+            window_seconds: status.window_seconds,
+            usage: status.usage,
+            window_started_epoch_seconds: status.window_started_epoch_seconds,
+        })
+    }
+
+    #[cfg(test)]
+    fn set_server_time(&mut self, now_epoch_seconds: u64) {
+        self.server_time_override = Some(now_epoch_seconds);
+    }
+
+    pub fn issue_cli_session(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        user: UserId,
+        session_secret: &str,
+        expires_at_epoch_seconds: Option<u64>,
+    ) -> Result<crate::CliSessionRecord, CoordinatorServiceError> {
+        if let Some(existing) = self.coordinator.project(&project) {
+            if existing.tenant != tenant {
+                return Err(CoordinatorError::Unauthorized(
+                    "CLI session project belongs to a different tenant".to_owned(),
+                )
+                .into());
+            }
+        } else {
+            self.coordinator
+                .upsert_project(tenant.clone(), project.clone(), "Session project");
+        }
+        self.coordinator
+            .grant_project_debug(tenant.clone(), project.clone(), user.clone());
+        let record = self.coordinator.issue_cli_session(
+            tenant,
+            project,
+            user,
+            session_secret,
+            expires_at_epoch_seconds,
+        );
+        self.persist_durable_state()?;
+        Ok(record)
+    }
+
+    pub fn authenticate_cli_session_context(
+        &self,
+        session_secret: &str,
+    ) -> Result<clusterflux_core::AuthContext, CoordinatorServiceError> {
+        Ok(self.coordinator.authenticate_cli_session(session_secret)?)
+    }
+
+    pub fn authenticate_cli_session_status_context(
+        &self,
+        session_secret: &str,
+    ) -> Result<clusterflux_core::AuthContext, CoordinatorServiceError> {
+        Ok(self
+            .coordinator
+            .authenticate_cli_session_for_status(session_secret)?)
+    }
+
+    pub fn revoke_cli_session(
+        &mut self,
+        session_secret: &str,
+    ) -> Result<crate::CliSessionRecord, CoordinatorServiceError> {
+        let record = self.coordinator.revoke_cli_session(session_secret)?;
+        self.persist_durable_state()?;
+        Ok(record)
+    }
+}
+
+#[cfg(test)]
+mod tests;

crates/clusterflux-coordinator/src/service/admin.rs

@@ -0,0 +1,147 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    admin_request_proof_from_token_digest, CredentialKind, Digest, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+const ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS: u64 = 300;
+
+impl CoordinatorService {
+    pub(super) fn handle_admin_status(
+        &mut self,
+        tenant: String,
+        actor_user: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.verify_admin_request(
+            "admin_status",
+            &tenant,
+            &actor_user,
+            &tenant,
+            &admin_proof,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        )?;
+        let tenant = TenantId::new(tenant);
+        let actor = UserId::new(actor_user);
+        Ok(CoordinatorResponse::AdminStatus {
+            suspended: self.coordinator.tenant_suspended(&tenant),
+            tenant,
+            actor,
+            safe_default: "read_only".to_owned(),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_suspend_tenant(
+        &mut self,
+        tenant: String,
+        actor_user: String,
+        target_tenant: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.verify_admin_request(
+            "suspend_tenant",
+            &tenant,
+            &actor_user,
+            &target_tenant,
+            &admin_proof,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        )?;
+        let actor_tenant = TenantId::new(tenant);
+        let actor = UserId::new(actor_user);
+        let target_tenant = TenantId::new(target_tenant);
+        self.coordinator.upsert_tenant(actor_tenant.clone());
+        self.coordinator.upsert_user(
+            actor_tenant,
+            actor.clone(),
+            CredentialKind::CliDeviceSession,
+        );
+        let policy = self
+            .coordinator
+            .suspend_tenant(target_tenant.clone(), actor.clone());
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::TenantSuspended {
+            tenant: target_tenant,
+            actor,
+            policy,
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn verify_admin_request(
+        &mut self,
+        operation: &str,
+        tenant: &str,
+        actor_user: &str,
+        target_tenant: &str,
+        admin_proof: &Digest,
+        admin_nonce: &str,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<(), CoordinatorServiceError> {
+        let expected = self.admin_token_digest.as_ref().ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "self-hosted admin credential is not configured".to_owned(),
+            )
+        })?;
+        if admin_nonce.trim().is_empty() || admin_nonce.len() > 256 {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request nonce is missing or invalid".to_owned(),
+            )
+            .into());
+        }
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|duration| duration.as_secs())
+            .unwrap_or_default();
+        if now.abs_diff(issued_at_epoch_seconds) > ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request timestamp is outside the allowed 300-second window".to_owned(),
+            )
+            .into());
+        }
+        let expected_proof = admin_request_proof_from_token_digest(
+            expected,
+            operation,
+            tenant,
+            actor_user,
+            target_tenant,
+            admin_nonce,
+            issued_at_epoch_seconds,
+        );
+        if admin_proof != &expected_proof {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request proof is invalid".to_owned(),
+            )
+            .into());
+        }
+        self.admin_replay_nonces.retain(|_, issued_at| {
+            now <= issued_at.saturating_add(ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS)
+        });
+        if self.admin_replay_nonces.contains_key(admin_nonce) {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request nonce was already used".to_owned(),
+            )
+            .into());
+        }
+        if self.admin_replay_nonces.len() >= super::MAX_REPLAY_NONCES_PER_AUTHORITY {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request replay window is full; retry after the bounded signature window advances"
+                    .to_owned(),
+            )
+            .into());
+        }
+        self.admin_replay_nonces
+            .insert(admin_nonce.to_owned(), issued_at_epoch_seconds);
+        Ok(())
+    }
+}

crates/clusterflux-coordinator/src/service/artifacts.rs

@@ -0,0 +1,799 @@
+use std::io::{Read, Seek, SeekFrom, Write};
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    generate_opaque_token, Actor, ArtifactId, AuthContext, DataPlaneObject, DataPlaneScope, Digest,
+    DownloadPolicy, NodeEndpoint, NodeId, ProjectId, RendezvousRequest, ResourceLimits,
+    ResourceMeter, StorageLocation, TenantId, UserId,
+};
+use sha2::{Digest as _, Sha256};
+
+use crate::CoordinatorError;
+
+use super::relay::RelayFinishReason;
+use super::{
+    bounded_ttl, ArtifactTransferAssignment, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+pub(super) const MAX_ARTIFACT_REVERSE_CHUNK_BYTES: u64 = 256 * 1024;
+
+#[derive(Debug)]
+pub(super) struct ArtifactReverseTransfer {
+    transfer_id: String,
+    token_digest: Digest,
+    tenant: TenantId,
+    project: ProjectId,
+    source_node: NodeId,
+    artifact: ArtifactId,
+    expected_digest: Digest,
+    expected_size_bytes: u64,
+    expires_at_epoch_seconds: u64,
+    spool: tempfile::NamedTempFile,
+    received_bytes: u64,
+    content_hasher: Sha256,
+    delivered_offset: u64,
+    error: Option<String>,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_create_artifact_download_link(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        ttl_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let policy = DownloadPolicy { max_bytes };
+        let action = self
+            .artifact_registry
+            .download_action(&context, &artifact, &policy)?;
+        self.ensure_download_source_connectivity(&action.source)?;
+        let downloadable_size = self
+            .artifact_registry
+            .downloadable_size(&context, &artifact, &policy)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.mutate_artifact_relay(|ledger| {
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })?;
+        self.quota.can_charge_download(
+            &context.tenant,
+            &context.project,
+            downloadable_size,
+            now_epoch_seconds,
+        )?;
+        let token_nonce = generate_opaque_token("artifact_download")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let ttl_seconds = bounded_ttl(
+            ttl_seconds,
+            self.admission.max_artifact_download_ttl_seconds,
+        );
+        let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
+        let account = match &context.actor {
+            Actor::User(user) => user.clone(),
+            Actor::Agent(_) | Actor::Node(_) | Actor::Task(_) => {
+                return Err(CoordinatorServiceError::Protocol(
+                    "artifact relay download requires a user account".to_owned(),
+                ))
+            }
+        };
+        let mut relay_candidate = self.artifact_relay.clone();
+        relay_candidate
+            .reserve(
+                token_nonce.clone(),
+                context.tenant.clone(),
+                context.project.clone(),
+                account,
+                downloadable_size,
+                MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
+                expires_at_epoch_seconds,
+                now_epoch_seconds,
+            )
+            .map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
+        let link = match self.artifact_registry.create_download_link(
+            &context,
+            &artifact,
+            &policy,
+            &token_nonce,
+            now_epoch_seconds,
+            ttl_seconds,
+        ) {
+            Ok(link) => link,
+            Err(error) => return Err(error.into()),
+        };
+        if let Err(error) =
+            relay_candidate.rekey(&token_nonce, link.scoped_token_digest.as_str().to_owned())
+        {
+            let _ = self.artifact_registry.revoke_download_link(
+                &context,
+                &artifact,
+                &link.scoped_token_digest,
+            );
+            return Err(CoordinatorServiceError::Protocol(error.to_string()));
+        }
+        if let Err(error) = self.commit_artifact_relay(relay_candidate) {
+            let _ = self.artifact_registry.revoke_download_link(
+                &context,
+                &artifact,
+                &link.scoped_token_digest,
+            );
+            return Err(error);
+        }
+        Ok(CoordinatorResponse::ArtifactDownloadLink { link })
+    }
+
+    pub(super) fn handle_open_artifact_download_stream(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let policy = DownloadPolicy { max_bytes };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.mutate_artifact_relay(|ledger| {
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let downloadable_size = self
+            .artifact_registry
+            .downloadable_size(&context, &artifact, &policy)?;
+        let validation_limits = ResourceLimits::unlimited();
+        let mut validation_meter = ResourceMeter::default();
+        let mut stream = self.artifact_registry.open_download_stream(
+            clusterflux_core::DownloadStreamRequest {
+                context: &context,
+                artifact: &artifact,
+                policy: &policy,
+                presented_token_digest: &token_digest,
+                now_epoch_seconds,
+                limits: &validation_limits,
+            },
+            &mut validation_meter,
+        )?;
+        self.ensure_download_source_connectivity(&stream.link.source)?;
+        self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
+
+        if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
+            if let Some(message) = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .and_then(|transfer| transfer.error.clone())
+            {
+                self.artifact_reverse_transfers.remove(&transfer_id);
+                self.artifact_transfer_by_token.remove(&token_digest);
+                self.mutate_artifact_relay(|ledger| {
+                    ledger.finish(token_digest.as_str(), RelayFinishReason::Failed);
+                    Ok(())
+                })?;
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "retaining node could not stream artifact: {message}"
+                )));
+            }
+            let (content_offset, content, end, complete) = {
+                let transfer = self
+                    .artifact_reverse_transfers
+                    .get_mut(&transfer_id)
+                    .ok_or_else(|| {
+                        CoordinatorServiceError::Protocol(
+                            "artifact reverse transfer index is inconsistent".to_owned(),
+                        )
+                    })?;
+                if transfer.tenant != context.tenant
+                    || transfer.project != context.project
+                    || transfer.artifact != artifact
+                    || transfer.token_digest != token_digest
+                {
+                    return Err(clusterflux_core::DownloadError::InvalidToken.into());
+                }
+                if transfer.received_bytes != transfer.expected_size_bytes {
+                    let overhead = self.artifact_relay.framing_overhead_bytes();
+                    self.mutate_artifact_relay(|ledger| {
+                        ledger.charge_egress(token_digest.as_str(), overhead, now_epoch_seconds)
+                    })?;
+                    return Ok(CoordinatorResponse::ArtifactDownloadStream {
+                        link: stream.link,
+                        streamed_bytes: 0,
+                        charged_download_bytes: self.quota.used_download_bytes(
+                            &context.tenant,
+                            &context.project,
+                            now_epoch_seconds,
+                        ),
+                        content_bytes_available: false,
+                        content_offset: None,
+                        content_eof: false,
+                        content_base64: None,
+                        content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
+                    });
+                }
+                if chunk_bytes == 0 && downloadable_size != 0 {
+                    return Err(CoordinatorServiceError::Protocol(
+                        "artifact download chunk_bytes must be greater than zero".to_owned(),
+                    ));
+                }
+                let requested = chunk_bytes
+                    .min(downloadable_size)
+                    .min(MAX_ARTIFACT_REVERSE_CHUNK_BYTES);
+                let start = transfer.delivered_offset;
+                let end = start.saturating_add(requested).min(transfer.received_bytes);
+                let length = usize::try_from(end.saturating_sub(start)).map_err(|_| {
+                    CoordinatorServiceError::Protocol(
+                        "artifact download chunk length does not fit memory bounds".to_owned(),
+                    )
+                })?;
+                let mut content = vec![0_u8; length];
+                let mut spool = transfer.spool.reopen().map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "open bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                spool.seek(SeekFrom::Start(start)).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "seek bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                spool.read_exact(&mut content).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "read bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                (start, content, end, end == transfer.received_bytes)
+            };
+            let streamed_bytes = content.len() as u64;
+            self.artifact_registry.stream_download_chunk(
+                &mut stream,
+                &validation_limits,
+                &mut validation_meter,
+                streamed_bytes,
+            )?;
+            let charged_download_bytes = self.quota.charge_download(
+                &context.tenant,
+                &context.project,
+                streamed_bytes,
+                now_epoch_seconds,
+            )?;
+            let content_base64 = BASE64_STANDARD.encode(content);
+            let egress_wire_bytes = (content_base64.len() as u64)
+                .saturating_add(self.artifact_relay.framing_overhead_bytes());
+            self.mutate_artifact_relay(|ledger| {
+                ledger.charge_egress(
+                    token_digest.as_str(),
+                    egress_wire_bytes,
+                    now_epoch_seconds,
+                )?;
+                if complete {
+                    ledger.finish(token_digest.as_str(), RelayFinishReason::Completed);
+                }
+                Ok(())
+            })?;
+            if let Some(transfer) = self.artifact_reverse_transfers.get_mut(&transfer_id) {
+                transfer.delivered_offset = end;
+            }
+            if complete {
+                self.artifact_reverse_transfers.remove(&transfer_id);
+                self.artifact_transfer_by_token.remove(&token_digest);
+            }
+            return Ok(CoordinatorResponse::ArtifactDownloadStream {
+                link: stream.link,
+                streamed_bytes,
+                charged_download_bytes,
+                content_bytes_available: true,
+                content_offset: Some(content_offset),
+                content_eof: complete,
+                content_base64: Some(content_base64),
+                content_source: Some("retaining_node_reverse_stream".to_owned()),
+            });
+        }
+
+        let StorageLocation::RetainedNode(source_node) = &stream.link.source else {
+            return Err(clusterflux_core::DownloadError::Unavailable.into());
+        };
+        let metadata = self
+            .artifact_registry
+            .metadata(&artifact)
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        let transfer_id = generate_opaque_token("artifact_transfer")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let spool = tempfile::Builder::new()
+            .prefix("clusterflux-artifact-transfer-")
+            .tempfile()
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "create bounded artifact transfer spool: {error}"
+                ))
+            })?;
+        let transfer = ArtifactReverseTransfer {
+            transfer_id: transfer_id.clone(),
+            token_digest: token_digest.clone(),
+            tenant: context.tenant.clone(),
+            project: context.project.clone(),
+            source_node: source_node.clone(),
+            artifact,
+            expected_digest: metadata.digest.clone(),
+            expected_size_bytes: metadata.size,
+            expires_at_epoch_seconds: stream.link.expires_at_epoch_seconds,
+            spool,
+            received_bytes: 0,
+            content_hasher: Sha256::new(),
+            delivered_offset: 0,
+            error: None,
+        };
+        let overhead = self.artifact_relay.framing_overhead_bytes();
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_egress(
+                stream.link.scoped_token_digest.as_str(),
+                overhead,
+                now_epoch_seconds,
+            )
+        })?;
+        self.artifact_reverse_transfers
+            .insert(transfer_id.clone(), transfer);
+        self.artifact_transfer_by_token
+            .insert(token_digest, transfer_id);
+        Ok(CoordinatorResponse::ArtifactDownloadStream {
+            link: stream.link,
+            streamed_bytes: 0,
+            charged_download_bytes: self.quota.used_download_bytes(
+                &context.tenant,
+                &context.project,
+                now_epoch_seconds,
+            ),
+            content_bytes_available: false,
+            content_offset: None,
+            content_eof: false,
+            content_base64: None,
+            content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
+        })
+    }
+
+    pub(super) fn handle_revoke_artifact_download_link(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        token_digest: Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let link = self.artifact_registry.revoke_download_link(
+            &context,
+            &ArtifactId::new(artifact),
+            &token_digest,
+        )?;
+        if let Some(transfer_id) = self.artifact_transfer_by_token.remove(&token_digest) {
+            self.artifact_reverse_transfers.remove(&transfer_id);
+        }
+        self.mutate_artifact_relay(|ledger| {
+            ledger.finish(token_digest.as_str(), RelayFinishReason::Cancelled);
+            Ok(())
+        })?;
+        Ok(CoordinatorResponse::ArtifactDownloadLinkRevoked { link })
+    }
+
+    pub(super) fn handle_export_artifact_to_node(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let receiver_node = NodeId::new(receiver_node);
+        let action = self.artifact_registry.download_action(
+            &context,
+            &artifact,
+            &DownloadPolicy {
+                max_bytes: u64::MAX,
+            },
+        )?;
+        let StorageLocation::RetainedNode(source_node) = action.source else {
+            return Err(clusterflux_core::DownloadError::Unavailable.into());
+        };
+        let metadata = self
+            .artifact_registry
+            .metadata(&artifact)
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
+        let destination =
+            self.export_endpoint(&receiver_node, &context.tenant, &context.project)?;
+        let plan = self.transport.plan_authenticated_direct_bulk_transfer(
+            RendezvousRequest {
+                scope: DataPlaneScope {
+                    tenant: context.tenant.clone(),
+                    project: context.project.clone(),
+                    process: metadata.process.clone(),
+                    object: DataPlaneObject::Artifact(artifact.clone()),
+                    authorization_subject: format!(
+                        "artifact-export:{}-to-{}",
+                        source_node, receiver_node
+                    ),
+                },
+                source,
+                destination,
+            },
+            direct_connectivity,
+            failure_reason,
+        )?;
+        Ok(CoordinatorResponse::ArtifactExportPlan {
+            plan,
+            source_node,
+            receiver_node,
+            artifact_size_bytes: metadata.size,
+        })
+    }
+
+    pub(super) fn handle_poll_artifact_transfer(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let transfer = self.artifact_reverse_transfers.values().find(|transfer| {
+            transfer.tenant == tenant
+                && transfer.project == project
+                && transfer.source_node == node
+                && transfer.error.is_none()
+                && transfer.received_bytes < transfer.expected_size_bytes
+        });
+        Ok(CoordinatorResponse::ArtifactTransferAssignment {
+            transfer: transfer.map(|transfer| ArtifactTransferAssignment {
+                transfer_id: transfer.transfer_id.clone(),
+                artifact: transfer.artifact.clone(),
+                expected_digest: transfer.expected_digest.clone(),
+                expected_size_bytes: transfer.expected_size_bytes,
+                offset: transfer.received_bytes,
+                max_chunk_bytes: MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
+            }),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_upload_artifact_transfer_chunk(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        offset: u64,
+        content_base64: String,
+        chunk_digest: Digest,
+        eof: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let token_digest = {
+            let transfer = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(
+                        "unknown artifact reverse transfer".to_owned(),
+                    )
+                })?;
+            if transfer.tenant != tenant
+                || transfer.project != project
+                || transfer.source_node != node
+                || transfer.artifact.as_str() != artifact
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "artifact reverse transfer is outside the signed node scope".to_owned(),
+                )
+                .into());
+            }
+            transfer.token_digest.clone()
+        };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let ingress_wire_bytes = (content_base64.len() as u64)
+            .saturating_add(self.artifact_relay.framing_overhead_bytes());
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_ingress(token_digest.as_str(), ingress_wire_bytes, now_epoch_seconds)
+        })?;
+        let transfer = self
+            .artifact_reverse_transfers
+            .get_mut(&transfer_id)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
+            })?;
+        if transfer.tenant != tenant
+            || transfer.project != project
+            || transfer.source_node != node
+            || transfer.artifact.as_str() != artifact
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer is outside the signed node scope".to_owned(),
+            )
+            .into());
+        }
+        if transfer.received_bytes != offset {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "artifact reverse transfer expected offset {}, received {offset}",
+                transfer.received_bytes
+            )));
+        }
+        let content = BASE64_STANDARD.decode(content_base64).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "artifact reverse transfer chunk is not valid base64: {error}"
+            ))
+        })?;
+        if content.len() as u64 > MAX_ARTIFACT_REVERSE_CHUNK_BYTES {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer chunk exceeds the control-plane limit".to_owned(),
+            ));
+        }
+        if Digest::sha256(&content) != chunk_digest {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer chunk digest mismatch".to_owned(),
+            ));
+        }
+        let next_offset = offset.saturating_add(content.len() as u64);
+        if next_offset > transfer.expected_size_bytes {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer exceeds retained metadata size".to_owned(),
+            ));
+        }
+        let complete = next_offset == transfer.expected_size_bytes;
+        if eof != complete {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer EOF does not match retained metadata size".to_owned(),
+            ));
+        }
+        transfer
+            .spool
+            .as_file_mut()
+            .seek(SeekFrom::Start(offset))
+            .and_then(|_| transfer.spool.as_file_mut().write_all(&content))
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "write bounded artifact transfer spool: {error}"
+                ))
+            })?;
+        transfer.content_hasher.update(&content);
+        transfer.received_bytes = next_offset;
+        if complete {
+            transfer.spool.as_file().sync_all().map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "sync bounded artifact transfer spool: {error}"
+                ))
+            })?;
+            let digest_hex = format!("{:x}", transfer.content_hasher.clone().finalize());
+            let digest =
+                Digest::from_sha256_hex(&digest_hex).map_err(CoordinatorServiceError::Protocol)?;
+            if digest != transfer.expected_digest {
+                transfer.spool.as_file_mut().set_len(0).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "reset invalid artifact transfer spool: {error}"
+                    ))
+                })?;
+                transfer.received_bytes = 0;
+                transfer.content_hasher = Sha256::new();
+                return Err(CoordinatorServiceError::Protocol(
+                    "artifact reverse transfer content digest mismatch".to_owned(),
+                ));
+            }
+        }
+        Ok(CoordinatorResponse::ArtifactTransferChunkAccepted {
+            transfer_id,
+            next_offset,
+            complete,
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_fail_artifact_transfer(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        message: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let token_digest = {
+            let transfer = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(
+                        "unknown artifact reverse transfer".to_owned(),
+                    )
+                })?;
+            if transfer.tenant != tenant
+                || transfer.project != project
+                || transfer.source_node != node
+                || transfer.artifact.as_str() != artifact
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "artifact reverse transfer failure is outside the signed node scope".to_owned(),
+                )
+                .into());
+            }
+            transfer.token_digest.clone()
+        };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let failure_wire_bytes =
+            (message.len() as u64).saturating_add(self.artifact_relay.framing_overhead_bytes());
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_ingress(token_digest.as_str(), failure_wire_bytes, now_epoch_seconds)
+        })?;
+        let transfer = self
+            .artifact_reverse_transfers
+            .get_mut(&transfer_id)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
+            })?;
+        if transfer.tenant != tenant
+            || transfer.project != project
+            || transfer.source_node != node
+            || transfer.artifact.as_str() != artifact
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer failure is outside the signed node scope".to_owned(),
+            )
+            .into());
+        }
+        let message = message.trim();
+        transfer.error = Some(if message.is_empty() {
+            "retained artifact bytes are unavailable".to_owned()
+        } else {
+            message.chars().take(1024).collect()
+        });
+        self.mutate_artifact_relay(|ledger| {
+            ledger.finish(token_digest.as_str(), RelayFinishReason::Failed);
+            Ok(())
+        })?;
+        Ok(CoordinatorResponse::ArtifactTransferFailed { transfer_id })
+    }
+
+    fn authorize_artifact_transfer_node(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        node: &NodeId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer node is outside its enrolled tenant/project scope"
+                    .to_owned(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    fn expire_artifact_reverse_transfers(
+        &mut self,
+        now_epoch_seconds: u64,
+    ) -> Result<(), CoordinatorServiceError> {
+        let expired = self
+            .artifact_reverse_transfers
+            .iter()
+            .filter(|(_, transfer)| transfer.expires_at_epoch_seconds < now_epoch_seconds)
+            .map(|(id, transfer)| (id.clone(), transfer.token_digest.clone()))
+            .collect::<Vec<_>>();
+        for (id, token) in &expired {
+            self.artifact_reverse_transfers.remove(id);
+            self.artifact_transfer_by_token.remove(token);
+        }
+        self.mutate_artifact_relay(|ledger| {
+            for (_, token) in &expired {
+                ledger.finish(token.as_str(), RelayFinishReason::Expired);
+            }
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })
+    }
+
+    fn export_endpoint(
+        &self,
+        node: &NodeId,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> Result<NodeEndpoint, CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact export node is outside the tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+            clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                "node {node} has not reported export connectivity"
+            ))
+        })?;
+        if descriptor.tenant != *tenant || descriptor.project != *project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact export node descriptor is outside the tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        if !self.node_is_live(node) {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "node {node} is offline for artifact export"
+                ))
+                .into(),
+            );
+        }
+        if !descriptor.direct_connectivity {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "direct connectivity unavailable to node {node} for artifact export"
+                ))
+                .into(),
+            );
+        }
+        Ok(NodeEndpoint {
+            node: node.clone(),
+            advertised_addr: format!("{node}.mesh.invalid:4433"),
+            public_key_fingerprint: Digest::sha256(&identity.public_key),
+        })
+    }
+
+    fn ensure_download_source_connectivity(
+        &self,
+        source: &StorageLocation,
+    ) -> Result<(), clusterflux_core::DownloadError> {
+        let StorageLocation::RetainedNode(node) = source else {
+            return Ok(());
+        };
+        let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+            clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                "retaining node {node} has not reported online status for artifact download"
+            ))
+        })?;
+        if !self.node_is_live(node) {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "retaining node {node} is offline for artifact download"
+                )),
+            );
+        }
+        Ok(())
+    }
+}
+
+fn user_context(tenant: String, project: String, actor_user: String) -> AuthContext {
+    AuthContext {
+        tenant: TenantId::new(tenant),
+        project: ProjectId::new(project),
+        actor: Actor::User(UserId::new(actor_user)),
+    }
+}

crates/clusterflux-coordinator/src/service/authenticated.rs

@@ -0,0 +1,139 @@
+use clusterflux_core::{AuthContext, UserId};
+
+use super::{
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_authenticated_agent_key_request(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let request = match request {
+            AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { agent, public_key } => {
+                CoordinatorRequest::RegisterAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                    public_key,
+                }
+            }
+            AuthenticatedCoordinatorRequest::ListAgentPublicKeys => {
+                CoordinatorRequest::ListAgentPublicKeys {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                }
+            }
+            AuthenticatedCoordinatorRequest::RotateAgentPublicKey { agent, public_key } => {
+                CoordinatorRequest::RotateAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                    public_key,
+                }
+            }
+            AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => {
+                CoordinatorRequest::RevokeAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                }
+            }
+            _ => unreachable!("caller filters authenticated agent key operations"),
+        };
+        self.handle_request(request)
+    }
+
+    pub(super) fn handle_authenticated_launch_task(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::LaunchTask {
+            task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        } = request
+        else {
+            unreachable!("caller filters authenticated task launches");
+        };
+        self.handle_launch_task(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            Some(actor.as_str().to_owned()),
+            None,
+            None,
+            None,
+            None,
+            *task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        )
+    }
+
+    pub(super) fn handle_authenticated_schedule_task(
+        &mut self,
+        context: &AuthContext,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::ScheduleTask {
+            environment,
+            environment_digest,
+            required_capabilities,
+            dependency_cache,
+            source_snapshot,
+            required_artifacts,
+            prefer_node,
+        } = request
+        else {
+            unreachable!("caller filters authenticated task scheduling");
+        };
+        self.handle_schedule_task(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            environment,
+            environment_digest,
+            required_capabilities,
+            dependency_cache,
+            source_snapshot,
+            required_artifacts,
+            prefer_node,
+        )
+    }
+
+    pub(super) fn handle_authenticated_artifact_export(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::ExportArtifactToNode {
+            artifact,
+            receiver_node,
+            direct_connectivity,
+            failure_reason,
+        } = request
+        else {
+            unreachable!("caller filters authenticated artifact exports");
+        };
+        self.handle_export_artifact_to_node(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            actor.as_str().to_owned(),
+            artifact,
+            receiver_node,
+            direct_connectivity,
+            failure_reason,
+        )
+    }
+}

crates/clusterflux-coordinator/src/service/authorization.rs

@@ -0,0 +1,204 @@
+use clusterflux_core::{Actor, AuthContext, UserId};
+
+use crate::CoordinatorError;
+
+use super::{AuthenticatedCoordinatorRequest, CoordinatorServiceError};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum PublicUserOperation {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject,
+    SelectProject,
+    ListProjects,
+    RegisterAgentPublicKey,
+    ListAgentPublicKeys,
+    RotateAgentPublicKey,
+    RevokeAgentPublicKey,
+    CreateNodeEnrollmentGrant,
+    ListNodeDescriptors,
+    RevokeNodeCredential,
+    StartProcess,
+    ScheduleTask,
+    LaunchTask,
+    CancelProcess,
+    AbortProcess,
+    ListProcesses,
+    QuotaStatus,
+    RestartTask,
+    ResolveTaskFailure,
+    DebugAttach,
+    SetDebugBreakpoints,
+    InspectDebugBreakpoints,
+    CreateDebugEpoch,
+    ResumeDebugEpoch,
+    InspectDebugEpoch,
+    ListTaskEvents,
+    ListTaskSnapshots,
+    JoinTask,
+    CreateArtifactDownloadLink,
+    OpenArtifactDownloadStream,
+    RevokeArtifactDownloadLink,
+    ExportArtifactToNode,
+}
+
+impl PublicUserOperation {
+    pub(super) fn as_str(self) -> &'static str {
+        match self {
+            Self::AuthStatus => "auth_status",
+            Self::RevokeCliSession => "revoke_cli_session",
+            Self::CreateProject => "create_project",
+            Self::SelectProject => "select_project",
+            Self::ListProjects => "list_projects",
+            Self::RegisterAgentPublicKey => "register_agent_public_key",
+            Self::ListAgentPublicKeys => "list_agent_public_keys",
+            Self::RotateAgentPublicKey => "rotate_agent_public_key",
+            Self::RevokeAgentPublicKey => "revoke_agent_public_key",
+            Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
+            Self::ListNodeDescriptors => "list_node_descriptors",
+            Self::RevokeNodeCredential => "revoke_node_credential",
+            Self::StartProcess => "start_process",
+            Self::ScheduleTask => "schedule_task",
+            Self::LaunchTask => "launch_task",
+            Self::CancelProcess => "cancel_process",
+            Self::AbortProcess => "abort_process",
+            Self::ListProcesses => "list_processes",
+            Self::QuotaStatus => "quota_status",
+            Self::RestartTask => "restart_task",
+            Self::ResolveTaskFailure => "resolve_task_failure",
+            Self::DebugAttach => "debug_attach",
+            Self::SetDebugBreakpoints => "set_debug_breakpoints",
+            Self::InspectDebugBreakpoints => "inspect_debug_breakpoints",
+            Self::CreateDebugEpoch => "create_debug_epoch",
+            Self::ResumeDebugEpoch => "resume_debug_epoch",
+            Self::InspectDebugEpoch => "inspect_debug_epoch",
+            Self::ListTaskEvents => "list_task_events",
+            Self::ListTaskSnapshots => "list_task_snapshots",
+            Self::JoinTask => "join_task",
+            Self::CreateArtifactDownloadLink => "create_artifact_download_link",
+            Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
+            Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
+            Self::ExportArtifactToNode => "export_artifact_to_node",
+        }
+    }
+}
+
+impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
+    fn from(request: &AuthenticatedCoordinatorRequest) -> Self {
+        match request {
+            AuthenticatedCoordinatorRequest::AuthStatus => Self::AuthStatus,
+            AuthenticatedCoordinatorRequest::RevokeCliSession => Self::RevokeCliSession,
+            AuthenticatedCoordinatorRequest::CreateProject { .. } => Self::CreateProject,
+            AuthenticatedCoordinatorRequest::SelectProject { .. } => Self::SelectProject,
+            AuthenticatedCoordinatorRequest::ListProjects => Self::ListProjects,
+            AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. } => {
+                Self::RegisterAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Self::ListAgentPublicKeys,
+            AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. } => {
+                Self::RotateAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. } => {
+                Self::RevokeAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } => {
+                Self::CreateNodeEnrollmentGrant
+            }
+            AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
+            AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
+                Self::RevokeNodeCredential
+            }
+            AuthenticatedCoordinatorRequest::StartProcess { .. } => Self::StartProcess,
+            AuthenticatedCoordinatorRequest::ScheduleTask { .. } => Self::ScheduleTask,
+            AuthenticatedCoordinatorRequest::LaunchTask { .. } => Self::LaunchTask,
+            AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
+            AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
+            AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
+            AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
+            AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
+            AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
+            AuthenticatedCoordinatorRequest::DebugAttach { .. } => Self::DebugAttach,
+            AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } => {
+                Self::SetDebugBreakpoints
+            }
+            AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. } => {
+                Self::InspectDebugBreakpoints
+            }
+            AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. } => Self::CreateDebugEpoch,
+            AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. } => Self::ResumeDebugEpoch,
+            AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
+            AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
+            AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
+            AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
+            AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
+                Self::CreateArtifactDownloadLink
+            }
+            AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { .. } => {
+                Self::OpenArtifactDownloadStream
+            }
+            AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { .. } => {
+                Self::RevokeArtifactDownloadLink
+            }
+            AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
+                Self::ExportArtifactToNode
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct AuthorizedPublicUser {
+    pub(super) actor: UserId,
+    pub(super) operation: PublicUserOperation,
+}
+
+pub(super) fn authorize_authenticated_user_operation(
+    context: &AuthContext,
+    request: &AuthenticatedCoordinatorRequest,
+) -> Result<AuthorizedPublicUser, CoordinatorServiceError> {
+    let operation = PublicUserOperation::from(request);
+    let actor = match &context.actor {
+        Actor::User(user) => user.clone(),
+        _ => {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "authenticated {} request requires a user CLI session",
+                operation.as_str()
+            ))
+            .into());
+        }
+    };
+    Ok(AuthorizedPublicUser { actor, operation })
+}
+
+#[cfg(test)]
+mod tests {
+    use clusterflux_core::{AgentId, ProjectId, TenantId};
+
+    use super::*;
+
+    #[test]
+    fn authenticated_public_authorization_requires_user_context_and_names_operation() {
+        let request = AuthenticatedCoordinatorRequest::DebugAttach {
+            process: "vp".to_owned(),
+        };
+        let agent_context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::Agent(AgentId::from("agent-ci")),
+        };
+
+        let denied = authorize_authenticated_user_operation(&agent_context, &request).unwrap_err();
+        assert!(denied
+            .to_string()
+            .contains("authenticated debug_attach request requires a user CLI session"));
+
+        let user_context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let authorized = authorize_authenticated_user_operation(&user_context, &request).unwrap();
+        assert_eq!(authorized.actor, UserId::from("user"));
+        assert_eq!(authorized.operation, PublicUserOperation::DebugAttach);
+    }
+}

crates/clusterflux-coordinator/src/service/debug/validation.rs

@@ -0,0 +1,92 @@
+use std::collections::BTreeSet;
+
+use crate::CoordinatorServiceError;
+
+use super::{DebugAcknowledgementState, DebugEpochRuntime};
+
+pub(super) fn runtime_all_in_state(
+    runtime: &DebugEpochRuntime,
+    state: DebugAcknowledgementState,
+) -> bool {
+    !runtime.expected.is_empty()
+        && runtime.expected.iter().all(|key| {
+            runtime
+                .acknowledgements
+                .get(key)
+                .is_some_and(|ack| ack.state == state)
+        })
+}
+
+pub(super) fn validate_probe_symbols(
+    probe_symbols: Vec<String>,
+) -> Result<Vec<String>, CoordinatorServiceError> {
+    if probe_symbols.len() > 128 {
+        return Err(CoordinatorServiceError::Protocol(
+            "debug breakpoint request exceeds 128 probe symbols".to_owned(),
+        ));
+    }
+    let mut unique = BTreeSet::new();
+    for symbol in probe_symbols {
+        if symbol.trim().is_empty()
+            || symbol.len() > 256
+            || !symbol
+                .chars()
+                .all(|character| character.is_ascii_alphanumeric() || "._:-/".contains(character))
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "debug breakpoint probe symbol is invalid".to_owned(),
+            ));
+        }
+        unique.insert(symbol);
+    }
+    Ok(unique.into_iter().collect())
+}
+
+pub(super) fn validate_debug_snapshot(
+    stack_frames: &[String],
+    local_values: &[(String, String)],
+    task_args: &[(String, String)],
+    handles: &[(String, String)],
+    command_status: Option<&str>,
+    recent_output: &[String],
+    message: Option<&str>,
+) -> Result<(), CoordinatorServiceError> {
+    const MAX_ITEMS: usize = 128;
+    const MAX_TEXT_BYTES: usize = 16 * 1024;
+    if [
+        stack_frames.len(),
+        local_values.len(),
+        task_args.len(),
+        handles.len(),
+        recent_output.len(),
+    ]
+    .into_iter()
+    .any(|count| count > MAX_ITEMS)
+    {
+        return Err(CoordinatorServiceError::Protocol(
+            "debug participant snapshot exceeds the 128-item field limit".to_owned(),
+        ));
+    }
+    let text_bytes = stack_frames.iter().map(String::len).sum::<usize>()
+        + local_values
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + task_args
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + handles
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + recent_output.iter().map(String::len).sum::<usize>()
+        + command_status.map(str::len).unwrap_or(0)
+        + message.map(str::len).unwrap_or(0);
+    if text_bytes > MAX_TEXT_BYTES {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "debug participant snapshot is {text_bytes} bytes; maximum is {MAX_TEXT_BYTES}"
+        )));
+    }
+    Ok(())
+}

crates/clusterflux-coordinator/src/service/debug_requests.rs

@@ -0,0 +1,594 @@
+use std::collections::BTreeSet;
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    Actor, Capability, CredentialKind, Digest, EnvironmentResource, ProcessId, ProjectId,
+    RestartDecision, RestartPolicy, RestartRequest, TaskDispatch, TaskInstanceId, TenantId, UserId,
+    WasmExportAbi,
+};
+
+use crate::{CoordinatorError, CoordinatorServiceError};
+
+use super::keys::task_restart_key;
+use super::protocol::{TaskAttemptState, TaskFailureResolution};
+use super::{
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
+    TaskReplacementBundle, TaskTerminalState, WorkflowActor,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_debug_request(
+        &mut self,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        match request {
+            CoordinatorRequest::DebugAttach {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_debug_attach(tenant, project, actor_user, process),
+            CoordinatorRequest::SetDebugBreakpoints {
+                tenant,
+                project,
+                actor_user,
+                process,
+                probe_symbols,
+            } => self.handle_set_debug_breakpoints(
+                tenant,
+                project,
+                actor_user,
+                process,
+                probe_symbols,
+            ),
+            CoordinatorRequest::InspectDebugBreakpoints {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_inspect_debug_breakpoints(tenant, project, actor_user, process),
+            CoordinatorRequest::CreateDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                stopped_task,
+                reason,
+            } => self.handle_create_debug_epoch(
+                tenant,
+                project,
+                actor_user,
+                process,
+                stopped_task,
+                reason,
+            ),
+            CoordinatorRequest::ResumeDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                epoch,
+            } => self.handle_resume_debug_epoch(tenant, project, actor_user, process, epoch),
+            CoordinatorRequest::InspectDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                epoch,
+            } => self.handle_inspect_debug_epoch(tenant, project, actor_user, process, epoch),
+            _ => unreachable!("handle_debug_request only accepts debug coordinator requests"),
+        }
+    }
+
+    pub(super) fn handle_authenticated_debug_request(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        match request {
+            AuthenticatedCoordinatorRequest::DebugAttach { process } => self.handle_debug_attach(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+            ),
+            AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
+                process,
+                probe_symbols,
+            } => self.handle_set_debug_breakpoints(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                probe_symbols,
+            ),
+            AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
+                .handle_inspect_debug_breakpoints(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::CreateDebugEpoch {
+                process,
+                stopped_task,
+                reason,
+            } => self.handle_create_debug_epoch(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                stopped_task,
+                reason,
+            ),
+            AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, epoch } => self
+                .handle_resume_debug_epoch(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                    epoch,
+                ),
+            AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, epoch } => self
+                .handle_inspect_debug_epoch(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                    epoch,
+                ),
+            _ => unreachable!(
+                "handle_authenticated_debug_request only accepts debug coordinator requests"
+            ),
+        }
+    }
+
+    pub(super) fn handle_restart_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor.clone()),
+        };
+        let authorization = self.coordinator.authorize_debug_attach(&context, &process);
+        if !authorization.allowed {
+            let _ = self.record_debug_audit_event(
+                tenant,
+                project,
+                process,
+                Some(task),
+                actor,
+                "restart_task",
+                false,
+                authorization.reason.clone(),
+            )?;
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task restart denied: {}",
+                authorization.reason
+            ))
+            .into());
+        }
+        let active_key = self
+            .active_tasks
+            .iter()
+            .find(|(task_tenant, task_project, task_process, _, task_id)| {
+                task_tenant == &tenant
+                    && task_project == &project
+                    && task_process == &process
+                    && task_id == &task
+            })
+            .cloned();
+        let process_key = super::keys::process_control_key(&tenant, &project, &process);
+        let active_main = self
+            .main_runtime
+            .controls
+            .get(&process_key)
+            .is_some_and(|control| {
+                control.task_instance == task
+                    && matches!(control.state.as_str(), "running" | "stopping")
+            });
+        let active_task = active_key.is_some() || active_main;
+        let completed_event_observed = self.task_events.iter().any(|event| {
+            event.tenant == tenant
+                && event.project == project
+                && event.process == process
+                && event.task == task
+        });
+        let checkpoint_key = task_restart_key(&tenant, &project, &process, &task);
+        let checkpoint = self.task_restart_checkpoints.get(&checkpoint_key).cloned();
+        let mut accepted = false;
+        let mut restarted_task_instance = None;
+        let mut restarted_attempt_id = None;
+        let mut clean_boundary_available = false;
+        let mut requires_whole_process_restart = true;
+        let message = if active_main {
+            "selected coordinator main is still active; restart the whole virtual process to rerun its capless entry boundary".to_owned()
+        } else if active_task {
+            "selected task is still active; wait for its terminal event or abort the whole virtual process before restarting from its clean entry boundary".to_owned()
+        } else if !completed_event_observed {
+            "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
+        } else if let Some(checkpoint) = checkpoint {
+            let vfs_available =
+                checkpoint
+                    .checkpoint
+                    .vfs_manifest
+                    .objects
+                    .iter()
+                    .all(|(path, object)| {
+                        let artifact = super::keys::artifact_id_from_path(path);
+                        self.artifact_registry
+                            .metadata(&artifact)
+                            .is_some_and(|metadata| {
+                                metadata.tenant == tenant
+                                    && metadata.project == project
+                                    && metadata.digest == object.digest
+                                    && metadata.size == object.size
+                                    && !metadata.retaining_nodes.is_empty()
+                            })
+                    });
+            if !vfs_available {
+                "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
+            } else {
+                let replacement = replacement_bundle
+                    .as_ref()
+                    .map(|bundle| {
+                        validate_task_replacement(
+                            bundle,
+                            &checkpoint.assignment.task_spec.task_definition,
+                            checkpoint.assignment.task_spec.environment_id.as_deref(),
+                        )
+                    })
+                    .transpose()?;
+                let request = RestartRequest {
+                    task: task.clone(),
+                    entrypoint: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.task_entrypoint.clone(),
+                        |replacement| replacement.export.clone(),
+                    ),
+                    serialized_args: checkpoint.checkpoint.boundary.serialized_args.clone(),
+                    environment_digest: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.environment_digest.clone(),
+                        |replacement| replacement.environment_digest.clone(),
+                    ),
+                    task_abi: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.task_abi.clone(),
+                        |replacement| replacement.restart_compatibility.clone(),
+                    ),
+                    source_edited: replacement.is_some(),
+                };
+                match RestartPolicy.decide(&checkpoint.checkpoint, &request) {
+                    RestartDecision::RestartTask { from_vfs_epoch, .. } => {
+                        clean_boundary_available = true;
+                        let mut assignment = checkpoint.assignment;
+                        assignment.task = task.clone();
+                        assignment.task_spec.task_instance = task.clone();
+                        let next_attempt = self
+                            .task_attempts
+                            .get(&checkpoint_key)
+                            .map_or(1, |attempts| attempts.len() + 1);
+                        assignment.artifact_path = format!(
+                            "/vfs/artifacts/{}-attempt-{next_attempt}-result.json",
+                            task.as_str()
+                        );
+                        if let Some(replacement) = replacement {
+                            if assignment.task_spec.source_snapshot.is_some() {
+                                let replacement_source = replacement_bundle
+                                    .as_ref()
+                                    .and_then(|bundle| bundle.source_snapshot.clone())
+                                    .ok_or_else(|| {
+                                        CoordinatorServiceError::Protocol(
+                                            "replacement task omitted the current SourceSnapshot for source-bound arguments"
+                                                .to_owned(),
+                                        )
+                                    })?;
+                                assignment
+                                    .task_spec
+                                    .rebind_source_snapshot(replacement_source)
+                                    .map_err(|error| {
+                                        CoordinatorServiceError::Protocol(format!(
+                                            "replacement task SourceSnapshot is invalid: {error}"
+                                        ))
+                                    })?;
+                            }
+                            assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
+                                export: Some(replacement.export),
+                                abi: WasmExportAbi::TaskV1,
+                            };
+                            assignment.task_spec.environment = replacement
+                                .environment
+                                .map(|environment| environment.requirements);
+                            assignment.task_spec.environment_digest =
+                                Some(replacement.environment_digest);
+                            assignment.task_spec.required_capabilities =
+                                replacement.required_capabilities;
+                            assignment.task_spec.bundle_digest =
+                                Some(replacement.bundle_digest.clone());
+                            assignment.wasm_module_base64 = replacement.wasm_module_base64;
+                        }
+                        let restart_task_spec = assignment.task_spec.clone();
+                        let restart_artifact_path = assignment.artifact_path.clone();
+                        self.restart_launches.insert(checkpoint_key.clone());
+                        let launch = self.handle_launch_task_with_actor(
+                            tenant.clone(),
+                            project.clone(),
+                            WorkflowActor {
+                                kind: "user".to_owned(),
+                                user: Some(actor.clone()),
+                                agent: None,
+                                credential_kind: CredentialKind::CliDeviceSession,
+                                public_key_fingerprint: None,
+                                authenticated_without_browser: false,
+                                scopes: vec!["process:restart-task".to_owned()],
+                            },
+                            assignment.task_spec,
+                            true,
+                            assignment.artifact_path,
+                            assignment.wasm_module_base64,
+                        );
+                        self.restart_launches.remove(&checkpoint_key);
+                        let launch = launch?;
+                        let queued = matches!(launch, CoordinatorResponse::TaskQueued { .. });
+                        accepted = matches!(
+                            &launch,
+                            CoordinatorResponse::TaskLaunched { .. }
+                                | CoordinatorResponse::TaskQueued { .. }
+                        );
+                        if accepted {
+                            restarted_task_instance = Some(task.clone());
+                            restarted_attempt_id = self
+                                .task_attempts
+                                .get(&checkpoint_key)
+                                .and_then(|attempts| attempts.last())
+                                .map(|attempt| attempt.attempt_id.clone());
+                            if restarted_attempt_id.is_none() {
+                                restarted_attempt_id = Some(self.begin_task_attempt(
+                                    &restart_task_spec,
+                                    None,
+                                    Some(&restart_artifact_path),
+                                    queued,
+                                )?);
+                            }
+                        }
+                        requires_whole_process_restart = !accepted;
+                        format!(
+                            "selected logical task {task} restarted as a new attempt from clean VFS entry boundary epoch {from_vfs_epoch}; unflushed task changes were discarded"
+                        )
+                    }
+                    RestartDecision::RestartWholeVirtualProcess { message } => message,
+                }
+            }
+        } else {
+            "selected task has terminal metadata but no captured clean VFS entry boundary; restart the whole virtual process".to_owned()
+        };
+        let audit_event = self.record_debug_audit_event(
+            tenant,
+            project,
+            process.clone(),
+            Some(task.clone()),
+            actor.clone(),
+            "restart_task",
+            true,
+            &message,
+        )?;
+        Ok(CoordinatorResponse::TaskRestart {
+            process,
+            task,
+            restarted_task_instance,
+            restarted_attempt_id,
+            actor,
+            accepted,
+            clean_boundary_available,
+            active_task,
+            completed_event_observed,
+            requires_whole_process_restart,
+            message,
+            charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
+            used_debug_read_bytes: audit_event.used_debug_read_bytes,
+            audit_event,
+        })
+    }
+
+    pub(super) fn handle_resolve_task_failure(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor),
+        };
+        let authorization = self.coordinator.authorize_debug_attach(&context, &process);
+        if !authorization.allowed {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task failure resolution denied: {}",
+                authorization.reason
+            ))
+            .into());
+        }
+        let key = task_restart_key(&tenant, &project, &process, &task);
+        let attempt = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+            .filter(|attempt| attempt.state == TaskAttemptState::FailedAwaitingAction)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(
+                    "task is not failed awaiting operator action".to_owned(),
+                )
+            })?;
+        attempt.state = match resolution {
+            TaskFailureResolution::AcceptFailure => TaskAttemptState::Failed,
+            TaskFailureResolution::Cancel => TaskAttemptState::Cancelled,
+        };
+        attempt.command_state = Some(
+            match resolution {
+                TaskFailureResolution::AcceptFailure => "failure_accepted",
+                TaskFailureResolution::Cancel => "cancelled",
+            }
+            .to_owned(),
+        );
+        let attempt_id = attempt.attempt_id.clone();
+        let mut event = self
+            .task_events
+            .iter()
+            .rev()
+            .find(|event| {
+                event.tenant == tenant
+                    && event.project == project
+                    && event.process == process
+                    && event.task == task
+                    && event.attempt_id.as_deref() == Some(attempt_id.as_str())
+            })
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(
+                    "failed attempt terminal event is unavailable".to_owned(),
+                )
+            })?;
+        if resolution == TaskFailureResolution::Cancel {
+            event.terminal_state = TaskTerminalState::Cancelled;
+            event.stderr_tail = "operator cancelled task after failure".to_owned();
+        }
+        self.record_task_completion_event(event.clone());
+        self.notify_coordinator_main_waiters(&event);
+        Ok(CoordinatorResponse::TaskFailureResolved {
+            process,
+            task,
+            attempt_id,
+            resolution,
+        })
+    }
+}
+
+struct ValidatedTaskReplacement {
+    bundle_digest: Digest,
+    wasm_module_base64: String,
+    export: String,
+    restart_compatibility: Digest,
+    environment: Option<EnvironmentResource>,
+    environment_digest: Digest,
+    required_capabilities: BTreeSet<Capability>,
+}
+
+fn validate_task_replacement(
+    replacement: &TaskReplacementBundle,
+    task_definition: &clusterflux_core::TaskDefinitionId,
+    environment_id: Option<&str>,
+) -> Result<ValidatedTaskReplacement, CoordinatorServiceError> {
+    let module = BASE64_STANDARD
+        .decode(&replacement.wasm_module_base64)
+        .map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "replacement task bundle is not valid base64: {error}"
+            ))
+        })?;
+    let actual_digest = Digest::sha256(&module);
+    if actual_digest != replacement.bundle_digest {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "replacement task bundle digest mismatch: expected {}, actual {actual_digest}",
+            replacement.bundle_digest
+        )));
+    }
+    let descriptors = super::main_runtime::task_descriptors(&module)?;
+    let descriptor = descriptors.get(task_definition.as_str()).ok_or_else(|| {
+        CoordinatorServiceError::Protocol(format!(
+            "replacement bundle has no task definition `{task_definition}`"
+        ))
+    })?;
+    if descriptor
+        .get("abi_version")
+        .and_then(serde_json::Value::as_u64)
+        != Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
+    {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "replacement task `{task_definition}` uses an unsupported task ABI"
+        )));
+    }
+    let export = descriptor
+        .get("export")
+        .and_then(serde_json::Value::as_str)
+        .filter(|export| !export.is_empty())
+        .ok_or_else(|| {
+            CoordinatorServiceError::Protocol(format!(
+                "replacement task `{task_definition}` omitted its export"
+            ))
+        })?
+        .to_owned();
+    let restart_compatibility = serde_json::from_value(
+        descriptor
+            .get("restart_compatibility_hash")
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(format!(
+                    "replacement task `{task_definition}` omitted restart compatibility metadata"
+                ))
+            })?,
+    )?;
+    let mut required_capabilities = descriptor
+        .get("required_capabilities")
+        .and_then(serde_json::Value::as_array)
+        .into_iter()
+        .flatten()
+        .map(|capability| {
+            super::main_runtime::capability_from_descriptor(capability.as_str().ok_or_else(
+                || {
+                    CoordinatorServiceError::Protocol(
+                        "replacement task capability is not a string".to_owned(),
+                    )
+                },
+            )?)
+            .map_err(CoordinatorServiceError::Protocol)
+        })
+        .collect::<Result<BTreeSet<_>, _>>()?;
+    let environment = environment_id
+        .map(|environment_id| {
+            super::main_runtime::bundle_environments(&module)?
+                .remove(environment_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "replacement bundle has no environment `{environment_id}`"
+                    ))
+                })
+        })
+        .transpose()?;
+    if let Some(environment) = &environment {
+        required_capabilities.extend(environment.requirements.capabilities.iter().cloned());
+    }
+    let environment_digest = environment.as_ref().map_or_else(
+        || Digest::sha256("clusterflux.environment.unconstrained.v1"),
+        |environment| environment.digest.clone(),
+    );
+    Ok(ValidatedTaskReplacement {
+        bundle_digest: replacement.bundle_digest.clone(),
+        wasm_module_base64: replacement.wasm_module_base64.clone(),
+        export,
+        restart_compatibility,
+        environment,
+        environment_digest,
+        required_capabilities,
+    })
+}

crates/clusterflux-coordinator/src/service/durable_runtime.rs

@@ -0,0 +1,47 @@
+use crate::{
+    DurableState, DurableStore, FallibleDurableStore, InMemoryDurableStore, PostgresDurableStore,
+};
+
+pub(super) enum RuntimeDurableStore {
+    InMemory(InMemoryDurableStore),
+    Postgres(PostgresDurableStore),
+}
+
+impl RuntimeDurableStore {
+    pub(super) fn from_database_url(database_url: Option<&str>) -> Result<Self, String> {
+        match database_url.map(str::trim).filter(|url| !url.is_empty()) {
+            Some(url) => PostgresDurableStore::connect(url)
+                .map(Self::Postgres)
+                .map_err(|error| error.to_string()),
+            None => Ok(Self::InMemory(InMemoryDurableStore::default())),
+        }
+    }
+
+    pub(super) fn kind(&self) -> &'static str {
+        match self {
+            Self::InMemory(_) => "in_memory",
+            Self::Postgres(_) => "postgres",
+        }
+    }
+}
+
+impl FallibleDurableStore for RuntimeDurableStore {
+    type Error = String;
+
+    fn load_state(&mut self) -> Result<DurableState, Self::Error> {
+        match self {
+            Self::InMemory(store) => Ok(store.load()),
+            Self::Postgres(store) => store.load_state().map_err(|error| error.to_string()),
+        }
+    }
+
+    fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
+        match self {
+            Self::InMemory(store) => {
+                store.save(state.clone());
+                Ok(())
+            }
+            Self::Postgres(store) => store.save_state(state).map_err(|error| error.to_string()),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/keys.rs

@@ -0,0 +1,73 @@
+use clusterflux_core::{
+    ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
+};
+
+pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
+pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
+pub(super) type TaskAssignmentKey = (TenantId, ProjectId, NodeId);
+pub(super) type PanelStopKey = (TenantId, ProjectId, ProcessId);
+pub(super) type EnrollmentGrantKey = (TenantId, ProjectId, String);
+pub(super) type ProcessControlKey = (TenantId, ProjectId, ProcessId);
+
+pub(super) fn task_control_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+    node: &NodeId,
+    task: &TaskInstanceId,
+) -> TaskControlKey {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        node.clone(),
+        task.clone(),
+    )
+}
+
+pub(super) fn task_restart_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+    task: &TaskInstanceId,
+) -> TaskRestartKey {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        task.clone(),
+    )
+}
+
+pub(super) fn process_control_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+) -> ProcessControlKey {
+    (tenant.clone(), project.clone(), process.clone())
+}
+
+pub(super) fn panel_stop_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+) -> PanelStopKey {
+    (tenant.clone(), project.clone(), process.clone())
+}
+
+pub(super) fn enrollment_grant_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    grant: &str,
+) -> EnrollmentGrantKey {
+    (tenant.clone(), project.clone(), grant.to_owned())
+}
+
+pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
+    let value = path
+        .as_str()
+        .strip_prefix("/vfs/artifacts/")
+        .unwrap_or(path.as_str())
+        .replace('/', ":");
+    ArtifactId::new(value)
+}

crates/clusterflux-coordinator/src/service/logs.rs

@@ -0,0 +1,661 @@
+use clusterflux_core::{
+    ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
+    TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key, task_restart_key};
+use super::protocol::TaskAttemptState;
+use super::{
+    artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
+    TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_report_task_log(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        backpressured: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        validate_task_log_tail("stdout_tail", &stdout_tail)?;
+        validate_task_log_tail("stderr_tail", &stderr_tail)?;
+        let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+        self.quota
+            .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+        Ok(CoordinatorResponse::TaskLogRecorded {
+            process,
+            task,
+            stdout_bytes,
+            stderr_bytes,
+            stdout_tail: if stdout_truncated {
+                format!("{stdout_tail}\n... truncated")
+            } else {
+                stdout_tail
+            },
+            stderr_tail: if stderr_truncated {
+                format!("{stderr_tail}\n... truncated")
+            } else {
+                stderr_tail
+            },
+            backpressured,
+        })
+    }
+
+    pub(super) fn handle_report_vfs_metadata(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        large_bytes_uploaded: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let artifact_path = artifact_path
+            .map(VfsPath::new)
+            .transpose()
+            .map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
+            self.flush_artifact_metadata(ArtifactFlush {
+                id: artifact_id_from_path(path),
+                tenant,
+                project,
+                process: process.clone(),
+                producer_task: task.clone(),
+                retaining_node: node,
+                digest,
+                size: artifact_size_bytes.unwrap_or_default(),
+            })?;
+        }
+        Ok(CoordinatorResponse::VfsMetadataRecorded {
+            process,
+            task,
+            artifact_path,
+            large_bytes_uploaded,
+        })
+    }
+
+    pub(super) fn handle_task_completed(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        terminal_state: Option<TaskTerminalState>,
+        status_code: Option<i32>,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        result: Option<TaskBoundaryValue>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        validate_task_log_tail("stdout_tail", &stdout_tail)?;
+        validate_task_log_tail("stderr_tail", &stderr_tail)?;
+        let artifact_path = artifact_path
+            .map(VfsPath::new)
+            .transpose()
+            .map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let checkpoint = self
+            .task_restart_checkpoints
+            .get(&super::keys::task_restart_key(
+                &tenant, &project, &process, &task,
+            ))
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "signed node task completion does not name a coordinator-issued task instance"
+                        .to_owned(),
+                )
+            })?;
+        if checkpoint.assignment.node != node {
+            return Err(CoordinatorError::Unauthorized(
+                "signed node task completion came from a node other than the assigned node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let mut event = TaskCompletionEvent {
+            tenant,
+            project,
+            process,
+            node,
+            executor: super::TaskExecutor::Node,
+            task_definition: checkpoint.assignment.task_spec.task_definition.clone(),
+            task,
+            attempt_id: None,
+            placement: None,
+            terminal_state: terminal_state
+                .unwrap_or_else(|| TaskTerminalState::from_status_code(status_code)),
+            status_code,
+            stdout_bytes,
+            stderr_bytes,
+            stdout_tail,
+            stderr_tail,
+            stdout_truncated,
+            stderr_truncated,
+            artifact_path,
+            artifact_digest: artifact_digest.clone(),
+            artifact_size_bytes,
+            result,
+        };
+        let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota.can_charge_log_bytes(
+            &event.tenant,
+            &event.project,
+            reported_bytes,
+            now_epoch_seconds,
+        )?;
+        self.quota.charge_log_bytes(
+            &event.tenant,
+            &event.project,
+            reported_bytes,
+            now_epoch_seconds,
+        )?;
+        let task_key = task_control_key(
+            &event.tenant,
+            &event.project,
+            &event.process,
+            &event.node,
+            &event.task,
+        );
+        let process_key = process_control_key(&event.tenant, &event.project, &event.process);
+        let process_was_aborted = self.process_aborts.contains(&process_key);
+        event.placement = self.task_placements.remove(&task_key);
+        if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
+            self.flush_artifact_metadata(ArtifactFlush {
+                id: artifact_id_from_path(path),
+                tenant: event.tenant.clone(),
+                project: event.project.clone(),
+                process: event.process.clone(),
+                producer_task: event.task.clone(),
+                retaining_node: event.node.clone(),
+                digest,
+                size: artifact_size_bytes.unwrap_or(stdout_bytes),
+            })?;
+        }
+        self.task_cancellations.remove(&task_key);
+        self.task_aborts.remove(&task_key);
+        self.debug_commands.remove(&task_key);
+        self.active_tasks.remove(&task_key);
+        let no_active_tasks =
+            !self
+                .active_tasks
+                .iter()
+                .any(|(task_tenant, task_project, task_process, _, _)| {
+                    task_tenant == &event.tenant
+                        && task_project == &event.project
+                        && task_process == &event.process
+                });
+        if no_active_tasks {
+            self.process_aborts.remove(&process_key);
+            if self.process_cancellations.remove(&process_key)
+                && !self.main_runtime.controls.contains_key(&process_key)
+            {
+                self.coordinator
+                    .abort_process(&event.tenant, &event.project, &event.process)?;
+                self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
+                self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
+            }
+        }
+        if process_was_aborted {
+            let checkpoint_key = super::keys::task_restart_key(
+                &event.tenant,
+                &event.project,
+                &event.process,
+                &event.task,
+            );
+            self.task_restart_checkpoints.remove(&checkpoint_key);
+            self.task_restart_checkpoint_order
+                .retain(|retained| retained != &checkpoint_key);
+        }
+        let awaiting_operator = self.finish_task_attempt(&mut event);
+        self.record_task_completion_event(event.clone());
+        if !awaiting_operator {
+            self.notify_coordinator_main_waiters(&event);
+        }
+        Ok(CoordinatorResponse::TaskRecorded {
+            process: event.process,
+            task: event.task,
+            events_recorded: self.task_events.len(),
+        })
+    }
+
+    pub(super) fn handle_list_task_events(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: Option<String>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = process.map(ProcessId::new);
+        if let Some(process) = &process {
+            self.authorize_task_event_process_scope(&tenant, &project, process)?;
+        }
+        let events = self
+            .task_events
+            .iter()
+            .filter(|event| {
+                event.tenant == tenant
+                    && event.project == project
+                    && process
+                        .as_ref()
+                        .is_none_or(|process| event.process == *process)
+            })
+            .cloned()
+            .collect();
+        Ok(CoordinatorResponse::TaskEvents { events })
+    }
+
+    pub(super) fn handle_list_task_snapshots(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        self.authorize_task_event_process_scope(&tenant, &project, &process)?;
+        let snapshots = self
+            .task_attempts
+            .iter()
+            .filter(
+                |((attempt_tenant, attempt_project, attempt_process, _), _)| {
+                    attempt_tenant == &tenant
+                        && attempt_project == &project
+                        && attempt_process == &process
+                },
+            )
+            .flat_map(|(_, attempts)| attempts.iter().cloned())
+            .collect();
+        Ok(CoordinatorResponse::TaskSnapshots { snapshots })
+    }
+
+    fn authorize_task_event_process_scope(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let active_in_scope = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .is_some();
+        let historical_in_scope = self.task_events.iter().any(|event| {
+            event.tenant == *tenant && event.project == *project && event.process == *process
+        }) || self.process_scope_history.iter().any(
+            |(historical_tenant, historical_project, historical_process)| {
+                historical_tenant == tenant
+                    && historical_project == project
+                    && historical_process == process
+            },
+        );
+        let process_exists_outside_scope = self
+            .coordinator
+            .active_process_exists_outside_scope(tenant, project, process)
+            || self.task_events.iter().any(|event| {
+                event.process == *process && (event.tenant != *tenant || event.project != *project)
+            })
+            || self.process_scope_history.iter().any(
+                |(historical_tenant, historical_project, historical_process)| {
+                    historical_process == process
+                        && (historical_tenant != tenant || historical_project != project)
+                },
+            );
+        if !active_in_scope && !historical_in_scope && process_exists_outside_scope {
+            return Err(CoordinatorError::Unauthorized(
+                "task event access is outside the virtual process tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    pub(super) fn handle_join_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        Ok(CoordinatorResponse::TaskJoined {
+            join: self.task_join_result(tenant, project, process, task),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_join_child_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let parent_task = TaskInstanceId::new(parent_task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        if !self.active_tasks.contains(&super::keys::task_control_key(
+            &tenant,
+            &project,
+            &process,
+            &node,
+            &parent_task,
+        )) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task join requires a currently active parent task on the signed node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        Ok(CoordinatorResponse::TaskJoined {
+            join: self.task_join_result(tenant, project, process, TaskInstanceId::new(task)),
+        })
+    }
+
+    pub(super) fn task_join_result(
+        &self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        task: TaskInstanceId,
+    ) -> TaskJoinResult {
+        let attempt_key = task_restart_key(&tenant, &project, &process, &task);
+        if self
+            .task_attempts
+            .get(&attempt_key)
+            .is_some_and(|attempts| {
+                attempts
+                    .iter()
+                    .rev()
+                    .find(|attempt| attempt.current)
+                    .is_some_and(|attempt| {
+                        matches!(
+                            attempt.state,
+                            TaskAttemptState::Queued
+                                | TaskAttemptState::Running
+                                | TaskAttemptState::FailedAwaitingAction
+                        )
+                    })
+            })
+        {
+            return TaskJoinResult::pending(
+                process,
+                task,
+                "logical task is still running or awaiting operator action",
+            );
+        }
+        let event = self.task_events.iter().rev().find(|event| {
+            event.tenant == tenant
+                && event.project == project
+                && event.process == process
+                && event.task == task
+        });
+
+        if let Some(event) = event {
+            TaskJoinResult::from_remote_completion(
+                event.process.clone(),
+                event.task.clone(),
+                event.node.clone(),
+                join_state_for_terminal(&event.terminal_state),
+                event.result.clone(),
+                event.status_code,
+                join_message_for_event(event),
+            )
+        } else {
+            let known = self.task_is_known_or_active(&tenant, &project, &process, &task);
+            TaskJoinResult::pending(
+                process,
+                task,
+                if known {
+                    "waiting for signed node task_completed event before join returns"
+                } else {
+                    "no signed node completion event has been observed for this task"
+                },
+            )
+        }
+    }
+
+    pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
+        event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
+        event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
+        let process_scope = (
+            event.tenant.clone(),
+            event.project.clone(),
+            event.process.clone(),
+        );
+        self.process_scope_history
+            .retain(|retained| retained != &process_scope);
+        while self.process_scope_history.len() >= super::MAX_TASK_EVENTS_TOTAL {
+            self.process_scope_history.pop_front();
+        }
+        self.process_scope_history.push_back(process_scope);
+        while self
+            .task_events
+            .iter()
+            .filter(|retained| {
+                retained.tenant == event.tenant
+                    && retained.project == event.project
+                    && retained.process == event.process
+            })
+            .count()
+            >= super::MAX_TASK_EVENTS_PER_PROCESS
+        {
+            let Some(index) = self.task_events.iter().position(|retained| {
+                retained.tenant == event.tenant
+                    && retained.project == event.project
+                    && retained.process == event.process
+            }) else {
+                break;
+            };
+            self.task_events.remove(index);
+        }
+        while self.task_events.len() >= super::MAX_TASK_EVENTS_TOTAL {
+            self.task_events.pop_front();
+        }
+        self.task_events.push_back(event);
+    }
+
+    fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
+        let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
+        let Some(attempt) = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+        else {
+            return false;
+        };
+        event.attempt_id = Some(attempt.attempt_id.clone());
+        attempt.status_code = event.status_code;
+        attempt.artifact_path = event.artifact_path.clone();
+        attempt.artifact_digest = event.artifact_digest.clone();
+        attempt.artifact_size_bytes = event.artifact_size_bytes;
+        attempt.error = (!event.stderr_tail.trim().is_empty()).then(|| event.stderr_tail.clone());
+        let awaiting_operator = event.terminal_state == TaskTerminalState::Failed
+            && attempt.failure_policy == clusterflux_core::TaskFailurePolicy::AwaitOperator;
+        attempt.state = if awaiting_operator {
+            TaskAttemptState::FailedAwaitingAction
+        } else {
+            match event.terminal_state {
+                TaskTerminalState::Completed => TaskAttemptState::Completed,
+                TaskTerminalState::Failed => TaskAttemptState::Failed,
+                TaskTerminalState::Cancelled => TaskAttemptState::Cancelled,
+            }
+        };
+        attempt.command_state = Some(if awaiting_operator {
+            "failed_awaiting_action".to_owned()
+        } else {
+            format!("{:?}", event.terminal_state).to_ascii_lowercase()
+        });
+        awaiting_operator
+    }
+
+    fn flush_artifact_metadata(
+        &mut self,
+        flush: ArtifactFlush,
+    ) -> Result<(), CoordinatorServiceError> {
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let pinned = self
+            .task_restart_checkpoints
+            .values()
+            .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
+            .chain(
+                self.pending_task_launches
+                    .iter()
+                    .flat_map(|pending| pending.task_spec.required_artifacts.iter()),
+            )
+            .cloned()
+            .collect::<std::collections::BTreeSet<ArtifactId>>();
+        self.artifact_registry
+            .flush_metadata_bounded(flush, &pinned)
+            .map(|_| ())
+            .map_err(CoordinatorServiceError::Protocol)
+    }
+
+    fn task_is_known_or_active(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+    ) -> bool {
+        self.active_tasks
+            .iter()
+            .any(|(task_tenant, task_project, task_process, _, task_id)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && task_id == task
+            })
+            || self.pending_task_launches.iter().any(|pending| {
+                &pending.tenant == tenant
+                    && &pending.project == project
+                    && &pending.process == process
+                    && &pending.task == task
+            })
+            || self.task_assignments.values().any(|assignments| {
+                assignments.iter().any(|assignment| {
+                    &assignment.tenant == tenant
+                        && &assignment.project == project
+                        && &assignment.process == process
+                        && &assignment.task == task
+                })
+            })
+    }
+}
+
+fn checked_reported_log_bytes(
+    stdout_bytes: u64,
+    stderr_bytes: u64,
+) -> Result<u64, CoordinatorServiceError> {
+    stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
+        CoordinatorServiceError::Protocol(
+            "reported task log byte counts exceed the supported range".to_owned(),
+        )
+    })
+}
+
+fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
+    if value.len() > MAX_TASK_LOG_TAIL_BYTES {
+        return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
+            "{kind} is {} bytes; max is {MAX_TASK_LOG_TAIL_BYTES}",
+            value.len()
+        )));
+    }
+    Ok(())
+}
+
+fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
+    if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
+        return value;
+    }
+    let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
+    while !value.is_char_boundary(boundary) {
+        boundary -= 1;
+    }
+    value.truncate(boundary);
+    *truncated = true;
+    value
+}
+
+fn join_state_for_terminal(terminal: &TaskTerminalState) -> TaskJoinState {
+    match terminal {
+        TaskTerminalState::Completed => TaskJoinState::Completed,
+        TaskTerminalState::Failed => TaskJoinState::Failed,
+        TaskTerminalState::Cancelled => TaskJoinState::Cancelled,
+    }
+}
+
+fn join_message_for_event(event: &TaskCompletionEvent) -> String {
+    match event.terminal_state {
+        TaskTerminalState::Completed => {
+            "joined result from signed node task_completed event".to_owned()
+        }
+        TaskTerminalState::Failed => {
+            let stderr = event.stderr_tail.trim();
+            if stderr.is_empty() {
+                "remote task failed".to_owned()
+            } else {
+                format!("remote task failed: {stderr}")
+            }
+        }
+        TaskTerminalState::Cancelled => "remote task was cancelled".to_owned(),
+    }
+}

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -0,0 +1,452 @@
+use std::collections::BTreeSet;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    generate_opaque_token, verify_node_request_signature, Actor, ArtifactId, CredentialKind,
+    Digest, NodeCapabilities, NodeDescriptor, NodeId, NodeSignedRequest, ProjectId,
+    SourceProviderKind, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::{
+    bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+impl CoordinatorService {
+    pub fn set_node_stale_after_seconds(&mut self, seconds: u64) {
+        self.node_stale_after_seconds = seconds.max(1);
+    }
+
+    pub(super) fn liveness_now_epoch_seconds(&self) -> u64 {
+        #[cfg(test)]
+        if let Some(now) = self.server_time_override {
+            return now;
+        }
+        unix_timestamp_seconds()
+    }
+
+    pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
+        self.node_last_seen_epoch_seconds
+            .get(node)
+            .is_some_and(|last_seen| {
+                self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
+                    <= self.node_stale_after_seconds
+            })
+    }
+
+    pub(super) fn live_node_descriptors(&self) -> Vec<NodeDescriptor> {
+        self.node_descriptors
+            .values()
+            .cloned()
+            .map(|mut descriptor| {
+                descriptor.online = self.node_is_live(&descriptor.id);
+                descriptor
+            })
+            .collect()
+    }
+
+    pub(super) fn handle_attach_node(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        self.coordinator.upsert_tenant(tenant.clone());
+        self.coordinator.upsert_user(
+            tenant.clone(),
+            UserId::from("local-user"),
+            CredentialKind::CliDeviceSession,
+        );
+        self.coordinator
+            .upsert_project(tenant.clone(), project.clone(), "local");
+        self.coordinator.upsert_source_provider_config(
+            tenant.clone(),
+            project.clone(),
+            SourceProviderKind::Filesystem,
+            Digest::sha256("local-filesystem"),
+        );
+        self.coordinator.enroll_node(
+            tenant.clone(),
+            project.clone(),
+            node.clone(),
+            public_key,
+            "node:attach",
+        );
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeAttached {
+            node,
+            tenant,
+            project,
+        })
+    }
+
+    pub(super) fn handle_create_node_enrollment_grant(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        ttl_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        self.coordinator.upsert_tenant(tenant.clone());
+        self.coordinator
+            .upsert_user(tenant.clone(), actor, CredentialKind::CliDeviceSession);
+        self.coordinator
+            .upsert_project(tenant.clone(), project.clone(), "local");
+        self.coordinator.upsert_source_provider_config(
+            tenant.clone(),
+            project.clone(),
+            SourceProviderKind::Filesystem,
+            Digest::sha256("local-filesystem"),
+        );
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.enrollment_grants.retain(|_, grant| {
+            !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
+        });
+        if self
+            .enrollment_grants
+            .values()
+            .filter(|grant| grant.tenant == tenant && grant.project == project)
+            .count()
+            >= super::MAX_ENROLLMENT_GRANTS_PER_PROJECT
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "node enrollment grant limit reached for this project; consume a grant or wait for one to expire"
+                    .to_owned(),
+            ));
+        }
+        let grant =
+            generate_opaque_token("node_grant").map_err(CoordinatorServiceError::Protocol)?;
+        let ttl_seconds = bounded_ttl(ttl_seconds, self.admission.max_node_enrollment_ttl_seconds);
+        let scope = "node:attach".to_owned();
+        let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
+        let enrollment = self.coordinator.create_node_enrollment_grant(
+            tenant.clone(),
+            project.clone(),
+            grant.clone(),
+            scope.clone(),
+            expires_at_epoch_seconds,
+        );
+        self.enrollment_grants
+            .insert(enrollment_grant_key(&tenant, &project, &grant), enrollment);
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeEnrollmentGrantCreated {
+            tenant,
+            project,
+            grant,
+            scope,
+            expires_at_epoch_seconds,
+        })
+    }
+
+    pub(super) fn handle_exchange_node_enrollment_grant(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+        enrollment_grant: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.enrollment_grants.retain(|_, grant| {
+            !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
+        });
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        if self.coordinator.node_identity(&node).is_none() {
+            self.quota.ensure_node_admission(
+                &tenant,
+                self.coordinator.node_identity_count_for_tenant(&tenant),
+            )?;
+        }
+        let grant_key = enrollment_grant_key(&tenant, &project, &enrollment_grant);
+        let grant =
+            self.enrollment_grants
+                .get_mut(&grant_key)
+                .ok_or(CoordinatorError::Enrollment(
+                    clusterflux_core::EnrollmentError::Expired,
+                ))?;
+        let credential = self.coordinator.exchange_node_enrollment_grant(
+            grant,
+            node.clone(),
+            &public_key,
+            "node:attach",
+            now_epoch_seconds,
+        )?;
+        self.enrollment_grants.remove(&grant_key);
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeEnrollmentExchanged {
+            node,
+            tenant,
+            project,
+            credential,
+        })
+    }
+
+    pub(super) fn handle_node_heartbeat(
+        &mut self,
+        node: String,
+        node_signature: Option<NodeSignedRequest>,
+        payload_digest: &Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let node = NodeId::new(node);
+        self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
+        Ok(CoordinatorResponse::NodeHeartbeat {
+            node,
+            epoch: self.coordinator.coordinator_epoch(),
+        })
+    }
+
+    pub(super) fn handle_report_node_capabilities(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        capabilities: NodeCapabilities,
+        cached_environment_digests: Vec<Digest>,
+        dependency_cache_digests: Vec<Digest>,
+        source_snapshots: Vec<Digest>,
+        artifact_locations: Vec<String>,
+        direct_connectivity: bool,
+        _online: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "node capability report is outside the enrolled tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        capabilities.validate_public_report()?;
+        for (kind, count) in [
+            ("cached environments", cached_environment_digests.len()),
+            ("dependency caches", dependency_cache_digests.len()),
+            ("source snapshots", source_snapshots.len()),
+            ("artifact locations", artifact_locations.len()),
+        ] {
+            if count > super::MAX_NODE_REPORTED_OBJECTS_PER_KIND {
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "node capability report contains {count} {kind}; limit is {}",
+                    super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+                )));
+            }
+        }
+        if cached_environment_digests
+            .iter()
+            .chain(&dependency_cache_digests)
+            .chain(&source_snapshots)
+            .any(|digest| !digest.is_valid_sha256())
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "node capability report contains an invalid digest".to_owned(),
+            ));
+        }
+        if artifact_locations.iter().any(|artifact| {
+            artifact.trim().is_empty()
+                || artifact.len() > 256
+                || artifact
+                    .chars()
+                    .any(|character| matches!(character, '/' | '\\' | '\0'))
+        }) {
+            return Err(CoordinatorServiceError::Protocol(
+                "node capability report contains an invalid artifact id".to_owned(),
+            ));
+        }
+        let artifact_locations = artifact_locations
+            .into_iter()
+            .map(ArtifactId::new)
+            .collect::<BTreeSet<_>>();
+        self.artifact_registry
+            .reconcile_node_retention(&node, &artifact_locations);
+
+        let online = self.node_is_live(&node);
+        self.node_descriptors.insert(
+            node.clone(),
+            NodeDescriptor {
+                id: node.clone(),
+                tenant,
+                project,
+                capabilities,
+                cached_environments: cached_environment_digests.into_iter().collect(),
+                dependency_caches: dependency_cache_digests.into_iter().collect(),
+                source_snapshots: source_snapshots.into_iter().collect(),
+                artifact_locations,
+                direct_connectivity,
+                online,
+            },
+        );
+        Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
+            node,
+            node_descriptors: self.node_descriptors.len(),
+        })
+    }
+
+    pub(super) fn handle_list_node_descriptors(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let descriptors = self
+            .live_node_descriptors()
+            .into_iter()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .collect();
+        Ok(CoordinatorResponse::NodeDescriptors { descriptors, actor })
+    }
+
+    pub(super) fn handle_revoke_node_credential(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let node = NodeId::new(node);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor.clone()),
+        };
+        self.coordinator.revoke_node_credential(&context, &node)?;
+        let descriptor_removed = self.node_descriptors.remove(&node).is_some();
+        self.node_last_seen_epoch_seconds.remove(&node);
+        self.artifact_registry.garbage_collect_node(&node);
+        let queued_assignments_removed = self
+            .task_assignments
+            .remove(&(tenant.clone(), project.clone(), node.clone()))
+            .map_or(0, |assignments| assignments.len());
+        self.active_tasks
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_aborts
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_placements
+            .retain(|(task_tenant, task_project, _, task_node, _), _| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeCredentialRevoked {
+            node,
+            tenant,
+            project,
+            actor,
+            descriptor_removed,
+            queued_assignments_removed,
+        })
+    }
+
+    pub(super) fn authenticate_node_request(
+        &mut self,
+        node: &NodeId,
+        node_signature: Option<NodeSignedRequest>,
+        request_kind: &str,
+        payload_digest: &Digest,
+    ) -> Result<(), CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        let signature = node_signature.ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "node request requires a signed proof of enrolled private-key possession"
+                    .to_owned(),
+            )
+        })?;
+        if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request nonce is missing or invalid".to_owned(),
+            )
+            .into());
+        }
+        let now_epoch_seconds = unix_timestamp_seconds();
+        if signature
+            .issued_at_epoch_seconds
+            .abs_diff(now_epoch_seconds)
+            > super::NODE_SIGNATURE_WINDOW_SECONDS
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request is expired or outside the allowed clock skew".to_owned(),
+            )
+            .into());
+        }
+        let replay_key = (node.clone(), signature.nonce.clone());
+        self.node_replay_nonces.retain(|_, accepted_at| {
+            now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
+        });
+        if self.node_replay_nonces.contains_key(&replay_key) {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request nonce has already been used".to_owned(),
+            )
+            .into());
+        }
+        verify_node_request_signature(
+            &identity.public_key,
+            node,
+            request_kind,
+            payload_digest,
+            &signature,
+        )
+        .map_err(CoordinatorError::Unauthorized)?;
+        if self
+            .node_replay_nonces
+            .keys()
+            .filter(|(retained_node, _)| retained_node == node)
+            .count()
+            >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request replay window is full; retry after the bounded signature window advances"
+                    .to_owned(),
+            )
+            .into());
+        }
+        self.node_replay_nonces
+            .insert(replay_key, now_epoch_seconds);
+        let seen_at = self.liveness_now_epoch_seconds();
+        self.node_last_seen_epoch_seconds
+            .insert(node.clone(), seen_at);
+        if let Some(descriptor) = self.node_descriptors.get_mut(node) {
+            descriptor.online = true;
+        }
+        Ok(())
+    }
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}

crates/clusterflux-coordinator/src/service/panels.rs

@@ -0,0 +1,307 @@
+use clusterflux_core::{
+    Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
+    ProcessId, ProjectId, RateLimit, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::artifact_id_from_path;
+use super::keys::panel_stop_key;
+use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+impl CoordinatorService {
+    pub(super) fn clear_operator_panel_state(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) {
+        let stop_key = panel_stop_key(tenant, project, process);
+        self.panel_snapshots.remove(&stop_key);
+        self.stopped_panels.remove(&stop_key);
+        self.panel_event_limits
+            .retain(|(event_tenant, event_project, event_process, _), _| {
+                event_tenant != tenant || event_project != project || event_process != process
+            });
+    }
+
+    pub(super) fn handle_render_operator_panel(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        max_download_bytes: u64,
+        stopped: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let stop_key = panel_stop_key(&tenant, &project, &process);
+
+        let panel = if stopped {
+            self.ensure_operator_panel_scope(&tenant, &project, &process)?;
+            self.stopped_panels.insert(stop_key);
+            let stop_key = panel_stop_key(&tenant, &project, &process);
+            let panel = match self.panel_snapshots.get(&stop_key).cloned() {
+                Some(panel) => stopped_panel_snapshot(panel),
+                None => self.render_operator_panel(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    UserId::new(actor_user),
+                    max_download_bytes,
+                    true,
+                )?,
+            };
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        } else {
+            self.stopped_panels.remove(&stop_key);
+            let panel = self.render_operator_panel(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                UserId::new(actor_user),
+                max_download_bytes,
+                false,
+            )?;
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        };
+        Ok(CoordinatorResponse::OperatorPanel { panel })
+    }
+
+    pub(super) fn handle_submit_panel_event(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        widget_id: String,
+        kind: PanelEventKind,
+        max_events: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let stop_key = panel_stop_key(&tenant, &project, &process);
+        let stopped = self.stopped_panels.contains(&stop_key);
+        let panel = if stopped {
+            match self.panel_snapshots.get(&stop_key).cloned() {
+                Some(panel) => stopped_panel_snapshot(panel),
+                None => {
+                    let panel = self.render_operator_panel(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        UserId::from("panel-user"),
+                        self.quota.download_limit(),
+                        true,
+                    )?;
+                    self.panel_snapshots.insert(stop_key.clone(), panel.clone());
+                    panel
+                }
+            }
+        } else {
+            let panel = self.render_operator_panel(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                UserId::from("panel-user"),
+                self.quota.download_limit(),
+                false,
+            )?;
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        };
+        let event = PanelEvent {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            widget_id: widget_id.clone(),
+            kind,
+        };
+        let limit_key = (tenant, project, process, widget_id);
+        let mut limit = self
+            .panel_event_limits
+            .get(&limit_key)
+            .cloned()
+            .unwrap_or(RateLimit {
+                max_events,
+                used_events: 0,
+            });
+        limit.max_events = max_events;
+        panel.accept_event(&event, &mut limit)?;
+        self.panel_event_limits.insert(limit_key, limit.clone());
+        Ok(CoordinatorResponse::PanelEventAccepted {
+            used_events: limit.used_events,
+            max_events: limit.max_events,
+        })
+    }
+
+    fn render_operator_panel(
+        &self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        actor_user: UserId,
+        max_download_bytes: u64,
+        stopped: bool,
+    ) -> Result<PanelState, CoordinatorServiceError> {
+        self.ensure_operator_panel_scope(&tenant, &project, &process)?;
+
+        let events = self
+            .task_events
+            .iter()
+            .filter(|event| {
+                event.tenant == tenant && event.project == project && event.process == process
+            })
+            .collect::<Vec<_>>();
+        let completed = events
+            .iter()
+            .filter(|event| event.status_code == Some(0))
+            .count() as u64;
+        let total = events.len().max(1) as u64;
+        let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::<u64>();
+        let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::<u64>();
+        let last_task = events.last().map(|event| event.task.clone());
+
+        let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone());
+        if stopped {
+            panel.freeze_program_ui_events();
+        }
+        panel.add_widget(PanelWidget {
+            id: "process-status".to_owned(),
+            label: "Process Status".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: if stopped {
+                    "stopped".to_owned()
+                } else {
+                    "running".to_owned()
+                },
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "task-progress".to_owned(),
+            label: "Tasks".to_owned(),
+            kind: PanelWidgetKind::Progress {
+                current: completed,
+                total,
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "task-summary".to_owned(),
+            label: "Task Summary".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: if events.is_empty() {
+                    "no task events recorded".to_owned()
+                } else {
+                    events
+                        .iter()
+                        .map(|event| {
+                            format!(
+                                "{} [{}]:{:?}:{}",
+                                event.task_definition, event.task, event.status_code, event.node
+                            )
+                        })
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                },
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "recent-logs".to_owned(),
+            label: "Recent Logs".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"),
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "debug-process".to_owned(),
+            label: "Debug Process".to_owned(),
+            kind: PanelWidgetKind::Button {
+                action: "debug-process".to_owned(),
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "cancel-process".to_owned(),
+            label: "Cancel Process".to_owned(),
+            kind: PanelWidgetKind::Button {
+                action: "cancel-process".to_owned(),
+            },
+        })?;
+        if last_task.is_some() {
+            panel.add_widget(PanelWidget {
+                id: "restart-selected-task".to_owned(),
+                label: "Restart Selected Task".to_owned(),
+                kind: PanelWidgetKind::Button {
+                    action: "restart-task".to_owned(),
+                },
+            })?;
+        }
+
+        let mut actions = vec![
+            clusterflux_core::ControlPlaneAction::DebugProcess,
+            clusterflux_core::ControlPlaneAction::CancelProcess,
+        ];
+        if let Some(task) = last_task.clone() {
+            actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
+        }
+        panel.set_control_plane_actions(actions);
+
+        if let Some(path) = events
+            .iter()
+            .rev()
+            .find_map(|event| event.artifact_path.as_ref())
+        {
+            let artifact = artifact_id_from_path(path);
+            let context = clusterflux_core::AuthContext {
+                tenant,
+                project,
+                actor: Actor::User(actor_user),
+            };
+            panel.add_download_widget_from_action(
+                "download-artifact",
+                "Download Artifact",
+                self.artifact_registry.download_action(
+                    &context,
+                    &artifact,
+                    &DownloadPolicy {
+                        max_bytes: max_download_bytes,
+                    },
+                ),
+            )?;
+        }
+
+        Ok(panel)
+    }
+
+    fn ensure_operator_panel_scope(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let active = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "operator panel requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, *tenant);
+        debug_assert_eq!(active.project, *project);
+        Ok(())
+    }
+}
+
+fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState {
+    panel.freeze_program_ui_events();
+    if let Some(status) = panel.widgets.get_mut("process-status") {
+        status.kind = PanelWidgetKind::Text {
+            value: "stopped".to_owned(),
+        };
+    }
+    panel
+}

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -0,0 +1,804 @@
+use std::collections::BTreeMap;
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
+    NodeDescriptor, NodeId, Placement, PlacementError, PlacementRequest, ProcessId, ProjectId,
+    Scheduler, TaskBoundaryValue, TaskCheckpoint, TaskDispatch, TaskInstanceId, TaskSpec, TenantId,
+    VfsManifest, VfsObject, VfsPath, WasmTaskInvocation,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key, task_restart_key};
+use super::{
+    CoordinatorResponse, CoordinatorService, CoordinatorServiceError, TaskAssignment, WorkflowActor,
+};
+
+use super::processes::*;
+use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
+
+fn select_task_placement(
+    candidates: &[Placement],
+    active_by_node: &BTreeMap<NodeId, usize>,
+) -> Option<Placement> {
+    candidates
+        .iter()
+        .min_by(|left, right| {
+            right
+                .score
+                .cmp(&left.score)
+                .then_with(|| {
+                    active_by_node
+                        .get(&left.node)
+                        .copied()
+                        .unwrap_or_default()
+                        .cmp(&active_by_node.get(&right.node).copied().unwrap_or_default())
+                })
+                .then_with(|| left.node.cmp(&right.node))
+        })
+        .cloned()
+}
+
+impl CoordinatorService {
+    fn place_workflow_task(
+        &self,
+        nodes: &[NodeDescriptor],
+        request: &PlacementRequest,
+    ) -> Result<Placement, PlacementError> {
+        let candidates = nodes
+            .iter()
+            .filter_map(|node| {
+                DefaultScheduler
+                    .place(std::slice::from_ref(node), request)
+                    .ok()
+            })
+            .collect::<Vec<_>>();
+        if candidates.is_empty() {
+            return DefaultScheduler.place(nodes, request);
+        }
+        let active_by_node = self
+            .active_tasks
+            .iter()
+            .filter(|(tenant, project, _, _, _)| {
+                tenant == &request.tenant && project == &request.project
+            })
+            .fold(BTreeMap::<NodeId, usize>::new(), |mut counts, key| {
+                *counts.entry(key.3.clone()).or_default() += 1;
+                counts
+            });
+        let mut selected = select_task_placement(&candidates, &active_by_node)
+            .expect("one or more compatible task-placement candidates should remain");
+        let selected_load = active_by_node
+            .get(&selected.node)
+            .copied()
+            .unwrap_or_default();
+        if candidates.iter().any(|candidate| {
+            candidate.score == selected.score
+                && active_by_node
+                    .get(&candidate.node)
+                    .copied()
+                    .unwrap_or_default()
+                    > selected_load
+        }) {
+            selected.reasons.push(format!(
+                "least active equal-locality node ({selected_load} active assignment(s))"
+            ));
+        }
+        Ok(selected)
+    }
+
+    pub(super) fn capture_task_restart_checkpoint(
+        &mut self,
+        assignment: &TaskAssignment,
+    ) -> Result<(), CoordinatorServiceError> {
+        let task_spec = &assignment.task_spec;
+        let environment_digest = task_spec.environment_digest.clone().unwrap_or_else(|| {
+            task_spec.environment.as_ref().map_or_else(
+                || Digest::sha256("clusterflux.environment.unconstrained.v1"),
+                |environment| {
+                    Digest::sha256(
+                        serde_json::to_vec(environment)
+                            .expect("serializable environment requirements"),
+                    )
+                },
+            )
+        });
+        let task_entrypoint = match &task_spec.dispatch {
+            clusterflux_core::TaskDispatch::CoordinatorNodeWasm { export, .. } => export
+                .clone()
+                .or_else(|| assignment_task_descriptor(assignment)?.get("export")?.as_str().map(str::to_owned))
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "cannot capture restart checkpoint for task `{}`: bundle descriptor omitted its Wasm export",
+                        task_spec.task_definition
+                    ))
+                })?,
+        };
+        let mut objects = BTreeMap::new();
+        let mut missing_required_artifact = false;
+        for artifact in &task_spec.required_artifacts {
+            let Some(metadata) = self.artifact_registry.metadata(artifact) else {
+                missing_required_artifact = true;
+                continue;
+            };
+            if metadata.tenant != assignment.tenant
+                || metadata.project != assignment.project
+                || metadata.retaining_nodes.is_empty()
+            {
+                missing_required_artifact = true;
+                continue;
+            }
+            let path = VfsPath::new(format!("/vfs/artifacts/{artifact}"))
+                .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
+            objects.insert(
+                path.clone(),
+                VfsObject {
+                    path,
+                    digest: metadata.digest.clone(),
+                    size: metadata.size,
+                    producer: metadata.producer_task.clone(),
+                    node: metadata.producer_node.clone(),
+                },
+            );
+        }
+        let checkpoint = TaskCheckpoint {
+            task: assignment.task.clone(),
+            boundary: CheckpointBoundary {
+                task_entrypoint,
+                serialized_args: Digest::sha256(serde_json::to_vec(&task_spec.args)?),
+                environment_digest,
+                vfs_epoch: task_spec.vfs_epoch,
+                task_abi: assignment_task_compatibility(assignment)
+                    .unwrap_or(Digest::sha256(serde_json::to_vec(&task_spec.dispatch)?)),
+            },
+            vfs_manifest: VfsManifest {
+                epoch: task_spec.vfs_epoch,
+                producer: assignment.task.clone(),
+                node: assignment.node.clone(),
+                objects,
+                large_bytes_uploaded: false,
+            },
+            depends_on_live_stack: false,
+            depends_on_live_socket: false,
+            depends_on_ephemeral_artifact_durability: missing_required_artifact,
+        };
+        let key = task_restart_key(
+            &assignment.tenant,
+            &assignment.project,
+            &assignment.process,
+            &assignment.task,
+        );
+        self.task_restart_checkpoint_order
+            .retain(|retained| retained != &key);
+        self.task_restart_checkpoints.insert(
+            key.clone(),
+            TaskRestartCheckpoint {
+                checkpoint,
+                assignment: assignment.clone(),
+            },
+        );
+        self.task_restart_checkpoint_order.push_back(key);
+        while self
+            .task_restart_checkpoint_order
+            .iter()
+            .filter(|(tenant, project, process, _)| {
+                tenant == &assignment.tenant
+                    && project == &assignment.project
+                    && process == &assignment.process
+            })
+            .count()
+            > super::MAX_RESTART_CHECKPOINTS_PER_PROCESS
+        {
+            let Some(index) = self.task_restart_checkpoint_order.iter().position(
+                |(tenant, project, process, _)| {
+                    tenant == &assignment.tenant
+                        && project == &assignment.project
+                        && process == &assignment.process
+                },
+            ) else {
+                break;
+            };
+            if let Some(expired) = self.task_restart_checkpoint_order.remove(index) {
+                self.task_restart_checkpoints.remove(&expired);
+            }
+        }
+        while self.task_restart_checkpoint_order.len() > super::MAX_RESTART_CHECKPOINTS_TOTAL {
+            if let Some(expired) = self.task_restart_checkpoint_order.pop_front() {
+                self.task_restart_checkpoints.remove(&expired);
+            }
+        }
+        Ok(())
+    }
+
+    pub(super) fn handle_schedule_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        environment: Option<clusterflux_core::EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<clusterflux_core::Capability>,
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let request = PlacementRequest {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            environment,
+            environment_digest,
+            environment_cache_required: false,
+            required_capabilities: required_capabilities.into_iter().collect(),
+            dependency_cache,
+            source_snapshot,
+            required_artifacts: required_artifacts
+                .into_iter()
+                .map(ArtifactId::new)
+                .collect(),
+            quota_available: self
+                .quota
+                .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
+                .is_ok(),
+            policy_allowed: self.admission.workflow_placement_allowed,
+            prefer_node: prefer_node.map(NodeId::new),
+        };
+        let nodes = self.live_node_descriptors();
+        let placement = DefaultScheduler.place(&nodes, &request)?;
+        Ok(CoordinatorResponse::TaskPlacement { placement })
+    }
+
+    pub(super) fn handle_launch_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        task_spec: TaskSpec,
+        _wait_for_node: bool,
+        _artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        task_spec
+            .validate_boundary_authority()
+            .map_err(CoordinatorServiceError::Protocol)?;
+        if matches!(
+            &task_spec.dispatch,
+            TaskDispatch::CoordinatorNodeWasm {
+                abi: clusterflux_core::WasmExportAbi::EntrypointV1,
+                ..
+            }
+        ) {
+            return self.handle_launch_coordinator_main(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                request_payload_digest,
+                task_spec,
+                wasm_module_base64,
+            );
+        }
+        Err(CoordinatorError::Unauthorized(
+            "external callers may launch only EntrypointV1; TaskV1 requires an authenticated live parent runtime or validated restart"
+                .to_owned(),
+        )
+        .into())
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_launch_child_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task_spec: TaskSpec,
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let parent_task = TaskInstanceId::new(parent_task);
+        if task_spec.process != process {
+            return Err(CoordinatorError::Unauthorized(
+                "child task must remain in its parent virtual process".to_owned(),
+            )
+            .into());
+        }
+        if !matches!(
+            task_spec.dispatch,
+            TaskDispatch::CoordinatorNodeWasm {
+                abi: clusterflux_core::WasmExportAbi::TaskV1,
+                ..
+            }
+        ) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task launch requires the TaskV1 ABI".to_owned(),
+            )
+            .into());
+        }
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let parent_key = task_control_key(&tenant, &project, &process, &node, &parent_task);
+        if !self.active_tasks.contains(&parent_key) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task launch requires a currently active parent task on the signed node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let actor = WorkflowActor {
+            kind: "task".to_owned(),
+            user: None,
+            agent: None,
+            credential_kind: CredentialKind::TaskCredential,
+            public_key_fingerprint: None,
+            authenticated_without_browser: true,
+            scopes: vec!["process:spawn-child".to_owned()],
+        };
+        self.handle_launch_task_with_actor(
+            tenant,
+            project,
+            actor,
+            task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        )
+    }
+
+    pub(super) fn handle_launch_task_with_actor(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: WorkflowActor,
+        task_spec: TaskSpec,
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        task_spec
+            .validate_boundary_authority()
+            .map_err(CoordinatorServiceError::Protocol)?;
+        if task_spec.tenant != tenant || task_spec.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification is outside the authenticated tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        if !task_spec.product_mode_uses_remote_dispatch() {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification must use the Wasm coordinator/node dispatch ABI".to_owned(),
+            )
+            .into());
+        }
+        if task_spec
+            .environment_id
+            .as_deref()
+            .is_some_and(|environment| environment.trim().is_empty() || environment.len() > 128)
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification environment id is invalid".to_owned(),
+            )
+            .into());
+        }
+        let process = task_spec.process.clone();
+        let task = task_spec.task_instance.clone();
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "task launch requires an active coordinator-side virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+        if self
+            .process_cancellations
+            .contains(&process_control_key(&tenant, &project, &process))
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "task launch is blocked because the virtual process is cancelling".to_owned(),
+            )
+            .into());
+        }
+        if self.task_instance_exists(&tenant, &project, &process, &task) {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "task instance {task} already exists in virtual process {process}; every spawn must use a fresh task-instance id"
+            )));
+        }
+        let in_flight = self
+            .active_tasks
+            .iter()
+            .filter(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant == &tenant && task_project == &project && task_process == &process
+            })
+            .count()
+            + self
+                .pending_task_launches
+                .iter()
+                .filter(|pending| {
+                    pending.tenant == tenant
+                        && pending.project == project
+                        && pending.process == process
+                })
+                .count();
+        if in_flight >= super::MAX_IN_FLIGHT_TASKS_PER_PROCESS {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "virtual process task limit of {} reached; join or cancel existing work before spawning more",
+                super::MAX_IN_FLIGHT_TASKS_PER_PROCESS
+            )));
+        }
+        if task_spec.vfs_epoch != active.coordinator_epoch {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task specification VFS epoch {} does not match active process epoch {}",
+                task_spec.vfs_epoch, active.coordinator_epoch
+            ))
+            .into());
+        }
+        let bundle_digest = task_spec.bundle_digest.as_ref().ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "Wasm task specification is missing its bundle digest".to_owned(),
+            )
+        })?;
+        if !bundle_digest.is_valid_sha256() {
+            return Err(CoordinatorError::Unauthorized(
+                "Wasm task specification has an invalid bundle digest".to_owned(),
+            )
+            .into());
+        }
+        let module = BASE64_STANDARD
+            .decode(&wasm_module_base64)
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "Wasm task module is not valid base64: {error}"
+                ))
+            })?;
+        let actual_digest = Digest::sha256(&module);
+        if &actual_digest != bundle_digest {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "Wasm task module digest does not match bundle digest: expected {bundle_digest}, actual {actual_digest}"
+            ))
+            .into());
+        }
+        WasmTaskInvocation::new(
+            task_spec.task_definition.clone(),
+            task.clone(),
+            task_spec.args.clone(),
+        )
+        .validate()
+        .map_err(CoordinatorServiceError::Protocol)?;
+        for artifact in &task_spec.required_artifacts {
+            let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
+                CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} is unavailable or has expired"
+                ))
+            })?;
+            if metadata.tenant != tenant || metadata.project != project {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} is outside the task tenant/project scope"
+                ))
+                .into());
+            }
+            if metadata.retaining_nodes.is_empty() {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} has no retaining node"
+                ))
+                .into());
+            }
+        }
+        VfsPath::new(&artifact_path)
+            .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        let request = PlacementRequest {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            environment: task_spec.environment.clone(),
+            environment_digest: task_spec.environment_digest.clone(),
+            environment_cache_required: task_spec.environment_id.is_some()
+                && task_spec.environment.is_none(),
+            required_capabilities: task_spec.required_capabilities.clone(),
+            dependency_cache: task_spec.dependency_cache.clone(),
+            source_snapshot: task_spec.source_snapshot.clone(),
+            required_artifacts: task_spec.required_artifacts.iter().cloned().collect(),
+            quota_available: self
+                .quota
+                .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
+                .is_ok(),
+            policy_allowed: self.admission.workflow_placement_allowed,
+            prefer_node: None,
+        };
+        let nodes = self.live_node_descriptors();
+        let placement = match self.place_workflow_task(&nodes, &request) {
+            Ok(placement) => placement,
+            Err(err) if wait_for_node => {
+                let reason = if err.message.is_empty() {
+                    "waiting for any capable node".to_owned()
+                } else {
+                    err.message
+                };
+                let charged_spawns =
+                    self.quota
+                        .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+                self.begin_task_attempt(&task_spec, None, Some(&artifact_path), true)?;
+                self.pending_task_launches.push_back(PendingTaskLaunch {
+                    tenant: tenant.clone(),
+                    project: project.clone(),
+                    process: process.clone(),
+                    task: task.clone(),
+                    request,
+                    epoch: active.coordinator_epoch,
+                    artifact_path,
+                    task_spec,
+                    wasm_module_base64,
+                });
+                return Ok(CoordinatorResponse::TaskQueued {
+                    process,
+                    task,
+                    actor,
+                    reason,
+                    charged_spawns,
+                    queued_tasks: self.pending_task_launches.len(),
+                });
+            }
+            Err(err) => return Err(err.into()),
+        };
+        let charged_spawns =
+            self.quota
+                .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        let assignment = TaskAssignment {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task: task.clone(),
+            node: placement.node.clone(),
+            epoch: active.coordinator_epoch,
+            artifact_path,
+            task_spec,
+            wasm_module_base64,
+        };
+        self.begin_task_attempt(
+            &assignment.task_spec,
+            Some(assignment.node.clone()),
+            Some(&assignment.artifact_path),
+            false,
+        )?;
+        self.capture_task_restart_checkpoint(&assignment)?;
+        let task_key = task_control_key(&tenant, &project, &process, &placement.node, &task);
+        self.task_placements
+            .insert(task_key.clone(), placement.clone());
+        self.active_tasks.insert(task_key);
+        self.task_assignments
+            .entry((tenant, project, placement.node.clone()))
+            .or_default()
+            .push_back(assignment.clone());
+        Ok(CoordinatorResponse::TaskLaunched {
+            process,
+            task,
+            actor,
+            placement,
+            assignment: Box::new(assignment),
+            charged_spawns,
+        })
+    }
+
+    fn task_instance_exists(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task_instance: &clusterflux_core::TaskInstanceId,
+    ) -> bool {
+        self.active_tasks.iter().any(
+            |(task_tenant, task_project, task_process, _, existing_instance)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && existing_instance == task_instance
+            },
+        ) || self.pending_task_launches.iter().any(|pending| {
+            &pending.tenant == tenant
+                && &pending.project == project
+                && &pending.process == process
+                && &pending.task == task_instance
+        }) || (!self.restart_launches.contains(&task_restart_key(
+            tenant,
+            project,
+            process,
+            task_instance,
+        )) && self.task_events.iter().any(|event| {
+            &event.tenant == tenant
+                && &event.project == project
+                && &event.process == process
+                && &event.task == task_instance
+        }))
+    }
+
+    pub(super) fn begin_task_attempt(
+        &mut self,
+        task_spec: &TaskSpec,
+        node: Option<NodeId>,
+        artifact_path: Option<&str>,
+        queued: bool,
+    ) -> Result<String, CoordinatorServiceError> {
+        let key = task_restart_key(
+            &task_spec.tenant,
+            &task_spec.project,
+            &task_spec.process,
+            &task_spec.task_instance,
+        );
+        while !self.task_attempts.contains_key(&key)
+            && self.task_attempts.len() >= super::MAX_TASK_ATTEMPT_HISTORIES
+        {
+            let removable = self.task_attempts.iter().find_map(|(candidate, attempts)| {
+                attempts
+                    .iter()
+                    .all(|attempt| {
+                        !matches!(
+                            &attempt.state,
+                            TaskAttemptState::Queued
+                                | TaskAttemptState::Running
+                                | TaskAttemptState::FailedAwaitingAction
+                        )
+                    })
+                    .then(|| candidate.clone())
+            });
+            let Some(removable) = removable else {
+                return Err(CoordinatorServiceError::Protocol(
+                    "task attempt history capacity is exhausted by active attempts".to_owned(),
+                ));
+            };
+            self.task_attempts.remove(&removable);
+        }
+        let attempts = self.task_attempts.entry(key).or_default();
+        for attempt in attempts.iter_mut() {
+            attempt.current = false;
+        }
+        let attempt_id = clusterflux_core::generate_opaque_token("ta")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let attempt_number = u32::try_from(attempts.len() + 1).unwrap_or(u32::MAX);
+        let mut argument_summary = task_spec
+            .args
+            .iter()
+            .map(|argument| {
+                let mut value = serde_json::to_string(argument)
+                    .unwrap_or_else(|_| "<invalid canonical argument>".to_owned());
+                value.truncate(value.len().min(1024));
+                value
+            })
+            .collect::<Vec<_>>();
+        argument_summary.truncate(64);
+        let mut handle_summary = task_spec
+            .required_artifacts
+            .iter()
+            .map(|artifact| format!("artifact:{artifact}"))
+            .collect::<Vec<_>>();
+        for argument in &task_spec.args {
+            if let TaskBoundaryValue::Structured(boundary) = argument {
+                handle_summary.extend(boundary.handles.iter().map(|handle| format!("{handle:?}")));
+            }
+        }
+        handle_summary.truncate(256);
+        attempts.push(TaskAttemptSnapshot {
+            process: task_spec.process.clone(),
+            task: task_spec.task_instance.clone(),
+            attempt_id: attempt_id.clone(),
+            attempt_number,
+            task_definition: task_spec.task_definition.clone(),
+            display_name: task_spec.task_definition.as_str().replace(['_', '-'], " "),
+            state: if queued {
+                TaskAttemptState::Queued
+            } else {
+                TaskAttemptState::Running
+            },
+            current: true,
+            node,
+            environment_id: task_spec.environment_id.clone(),
+            environment_digest: task_spec.environment_digest.clone(),
+            argument_summary,
+            handle_summary,
+            command_state: Some(if queued { "queued" } else { "running" }.to_owned()),
+            vfs_checkpoint: format!("vfs-epoch:{}", task_spec.vfs_epoch),
+            probe_symbol: None,
+            source_path: None,
+            source_line: None,
+            restart_compatible: true,
+            failure_policy: task_spec.failure_policy,
+            artifact_path: artifact_path.and_then(|path| VfsPath::new(path).ok()),
+            artifact_digest: None,
+            artifact_size_bytes: None,
+            status_code: None,
+            error: None,
+        });
+        if attempts.len() > 128 {
+            attempts.remove(0);
+        }
+        Ok(attempt_id)
+    }
+
+    pub(super) fn assign_task_attempt(&mut self, task_spec: &TaskSpec, node: NodeId) {
+        let key = task_restart_key(
+            &task_spec.tenant,
+            &task_spec.project,
+            &task_spec.process,
+            &task_spec.task_instance,
+        );
+        if let Some(attempt) = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+        {
+            attempt.node = Some(node);
+            attempt.state = TaskAttemptState::Running;
+            attempt.command_state = Some("running".to_owned());
+        }
+    }
+}
+
+fn assignment_task_compatibility(assignment: &TaskAssignment) -> Option<Digest> {
+    let descriptor = assignment_task_descriptor(assignment)?;
+    serde_json::from_value(descriptor.get("restart_compatibility_hash")?.clone()).ok()
+}
+
+fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option<serde_json::Value> {
+    let module = BASE64_STANDARD
+        .decode(&assignment.wasm_module_base64)
+        .ok()?;
+    let mut descriptors = super::main_runtime::task_descriptors(&module).ok()?;
+    descriptors.remove(assignment.task_spec.task_definition.as_str())
+}
+
+#[cfg(test)]
+mod placement_tests {
+    use super::*;
+
+    fn placement(node: &str, score: i64) -> Placement {
+        Placement {
+            node: NodeId::from(node),
+            score,
+            reasons: Vec::new(),
+        }
+    }
+
+    #[test]
+    fn equal_locality_prefers_the_least_active_node() {
+        let candidates = vec![placement("busy", 40), placement("idle", 40)];
+        let active = BTreeMap::from([(NodeId::from("busy"), 1)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("idle")
+        );
+    }
+
+    #[test]
+    fn locality_score_remains_stronger_than_load_balancing() {
+        let candidates = vec![placement("warm", 50), placement("cold", 40)];
+        let active = BTreeMap::from([(NodeId::from("warm"), 2)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("warm")
+        );
+    }
+}

crates/clusterflux-coordinator/src/service/processes.rs

@@ -0,0 +1,815 @@
+use std::collections::{BTreeSet, VecDeque};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    AgentId, AgentSignedRequest, CredentialKind, DefaultScheduler, Digest, NodeId,
+    PlacementRequest, ProcessId, ProjectId, RendezvousRequest, Scheduler, SourcePreparation,
+    TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key};
+use super::{
+    CoordinatorResponse, CoordinatorService, CoordinatorServiceError, SourcePreparationDisposition,
+    SourcePreparationStatus, TaskAssignment, TaskCancellationTarget, VirtualProcessStatus,
+    WorkflowActor,
+};
+
+#[derive(Clone, Debug)]
+pub(super) struct PendingTaskLaunch {
+    pub(super) tenant: TenantId,
+    pub(super) project: ProjectId,
+    pub(super) process: ProcessId,
+    pub(super) task: TaskInstanceId,
+    pub(super) request: PlacementRequest,
+    pub(super) epoch: u64,
+    pub(super) artifact_path: String,
+    pub(super) task_spec: TaskSpec,
+    pub(super) wasm_module_base64: String,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct TaskRestartCheckpoint {
+    pub(super) checkpoint: TaskCheckpoint,
+    pub(super) assignment: TaskAssignment,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_poll_task_assignment(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "task assignment poll is outside the enrolled tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        let assignment_key = (tenant.clone(), project.clone(), node.clone());
+        let assignment = self
+            .task_assignments
+            .get_mut(&assignment_key)
+            .and_then(VecDeque::pop_front);
+        if assignment.is_some() {
+            return Ok(CoordinatorResponse::TaskAssignment {
+                assignment: assignment.map(Box::new),
+            });
+        }
+        let assignment = self.assign_pending_task_to_node(&tenant, &project, &node)?;
+        Ok(CoordinatorResponse::TaskAssignment {
+            assignment: assignment.map(Box::new),
+        })
+    }
+
+    fn assign_pending_task_to_node(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        node: &NodeId,
+    ) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
+        let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
+            return Ok(None);
+        };
+        let mut remaining = VecDeque::new();
+        let mut selected = None;
+
+        while let Some(pending) = self.pending_task_launches.pop_front() {
+            if selected.is_some() {
+                remaining.push_back(pending);
+                continue;
+            }
+            if &pending.tenant != tenant || &pending.project != project {
+                remaining.push_back(pending);
+                continue;
+            }
+            if self.process_cancellations.contains(&process_control_key(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+            )) {
+                continue;
+            }
+            let Some(active) = self.coordinator.active_process(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+            ) else {
+                continue;
+            };
+            if active.tenant != pending.tenant
+                || active.project != pending.project
+                || active.coordinator_epoch != pending.epoch
+            {
+                continue;
+            }
+            let Ok(placement) =
+                DefaultScheduler.place(std::slice::from_ref(&descriptor), &pending.request)
+            else {
+                remaining.push_back(pending);
+                continue;
+            };
+            let assignment = TaskAssignment {
+                tenant: pending.tenant.clone(),
+                project: pending.project.clone(),
+                process: pending.process.clone(),
+                task: pending.task.clone(),
+                node: placement.node.clone(),
+                epoch: pending.epoch,
+                artifact_path: pending.artifact_path,
+                task_spec: pending.task_spec,
+                wasm_module_base64: pending.wasm_module_base64,
+            };
+            self.assign_task_attempt(&assignment.task_spec, assignment.node.clone());
+            self.capture_task_restart_checkpoint(&assignment)?;
+            let task_key = task_control_key(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+                &placement.node,
+                &pending.task,
+            );
+            self.task_placements.insert(task_key.clone(), placement);
+            self.active_tasks.insert(task_key);
+            selected = Some(assignment);
+        }
+
+        self.pending_task_launches = remaining;
+        Ok(selected)
+    }
+
+    pub(super) fn handle_request_rendezvous(
+        &mut self,
+        scope: clusterflux_core::DataPlaneScope,
+        source: clusterflux_core::NodeEndpoint,
+        destination: clusterflux_core::NodeEndpoint,
+        direct_connectivity: bool,
+        failure_reason: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let charged_rendezvous_attempts = self.quota.charge_rendezvous_attempt(
+            &scope.tenant,
+            &scope.project,
+            now_epoch_seconds,
+        )?;
+        let plan = self.transport.plan_authenticated_direct_bulk_transfer(
+            RendezvousRequest {
+                scope,
+                source,
+                destination,
+            },
+            direct_connectivity,
+            failure_reason,
+        )?;
+        Ok(CoordinatorResponse::RendezvousPlan {
+            plan,
+            charged_rendezvous_attempts,
+        })
+    }
+
+    pub(super) fn handle_request_source_preparation(
+        &mut self,
+        tenant: String,
+        project: String,
+        provider: clusterflux_core::SourceProviderKind,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let preparation = SourcePreparation::node_task(tenant.clone(), project.clone(), provider);
+        let request = PlacementRequest {
+            tenant,
+            project,
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: preparation.required_capabilities.clone(),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: Default::default(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+        let nodes = self.live_node_descriptors();
+        let disposition = match DefaultScheduler.place(&nodes, &request) {
+            Ok(placement) => SourcePreparationDisposition::Assigned {
+                node: placement.node,
+            },
+            Err(err) => SourcePreparationDisposition::Pending {
+                reason: if err.message.is_empty() {
+                    "waiting for any capable node to prepare source".to_owned()
+                } else {
+                    err.message
+                },
+            },
+        };
+        Ok(CoordinatorResponse::SourcePreparation {
+            status: SourcePreparationStatus {
+                preparation,
+                disposition,
+            },
+        })
+    }
+
+    pub(super) fn handle_complete_source_preparation(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        provider: clusterflux_core::SourceProviderKind,
+        source_snapshot: Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "source preparation completion is outside the enrolled tenant/project scope"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "source preparation completion requires a node capability report".to_owned(),
+            )
+        })?;
+        if !descriptor.source_snapshots.contains(&source_snapshot)
+            && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+        {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "node source snapshot retention limit of {} reached; refresh the node capability report",
+                super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+            )));
+        }
+        descriptor.source_snapshots.insert(source_snapshot.clone());
+        Ok(CoordinatorResponse::SourcePreparationCompleted {
+            node,
+            provider,
+            source_snapshot,
+        })
+    }
+
+    pub(super) fn handle_start_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        process: String,
+        restart: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        let actor = self.workflow_actor(
+            &tenant,
+            &project,
+            actor_user,
+            actor_agent,
+            agent_public_key_fingerprint,
+            agent_signature,
+            request_payload_digest,
+            "start_process",
+            &process,
+            None,
+        )?;
+        let replacing_existing = if let Some(active) = self
+            .coordinator
+            .active_process_for_project(&tenant, &project)
+        {
+            if active.id != process || !restart {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "project already has active virtual process {}; attach to or restart it, request cooperative cancellation, abort it, or use another Coordinator Project",
+                    active.id
+                ))
+                .into());
+            }
+            true
+        } else {
+            false
+        };
+        if !replacing_existing {
+            self.quota.ensure_process_admission(
+                &tenant,
+                self.coordinator.active_process_count_for_tenant(&tenant),
+            )?;
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let charged_spawns =
+            self.quota
+                .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        if replacing_existing {
+            self.main_runtime.interrupt_process(
+                &tenant,
+                &project,
+                &process,
+                "virtual process incarnation replaced",
+            );
+            self.main_runtime
+                .controls
+                .remove(&process_control_key(&tenant, &project, &process));
+        }
+        self.process_cancellations
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.process_aborts
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.clear_operator_panel_state(&tenant, &project, &process);
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_aborts
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.active_tasks
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_placements
+            .retain(|(task_tenant, task_project, task_process, _, _), _| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_assignments.retain(|_, assignments| {
+            assignments.retain(|assignment| {
+                assignment.tenant != tenant
+                    || assignment.project != project
+                    || assignment.process != process
+            });
+            !assignments.is_empty()
+        });
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        self.task_restart_checkpoints.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, _), _| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+            },
+        );
+        self.task_restart_checkpoint_order.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, _)| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+            },
+        );
+        self.task_events.retain(|event| {
+            event.tenant != tenant || event.project != project || event.process != process
+        });
+        self.task_attempts
+            .retain(|(attempt_tenant, attempt_project, attempt_process, _), _| {
+                attempt_tenant != &tenant
+                    || attempt_project != &project
+                    || attempt_process != &process
+            });
+        self.restart_launches
+            .retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
+                attempt_tenant != &tenant
+                    || attempt_project != &project
+                    || attempt_process != &process
+            });
+        self.debug_audit_events.retain(|event| {
+            event.tenant != tenant || event.project != project || event.process != process
+        });
+        self.coordinator
+            .start_process(tenant, project, process.clone());
+        Ok(CoordinatorResponse::ProcessStarted {
+            process,
+            epoch: self.coordinator.coordinator_epoch(),
+            actor,
+            charged_spawns,
+        })
+    }
+
+    pub(super) fn handle_reconnect_node(
+        &mut self,
+        node: String,
+        process: String,
+        epoch: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let node = NodeId::new(node);
+        let process = ProcessId::new(process);
+        self.coordinator
+            .reconnect_node(&node, Some((&process, epoch)))?;
+        Ok(CoordinatorResponse::NodeReconnected { node, process })
+    }
+
+    pub(super) fn handle_cancel_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.coordinator
+            .authorize_node_for_process(&node, &tenant, &project, &process)?;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "task cancellation requires an active virtual process".to_owned(),
+                )
+            })?;
+        if !active.connected_nodes.contains(&node) {
+            return Err(CoordinatorError::Unauthorized(
+                "task cancellation target node is not connected to the virtual process".to_owned(),
+            )
+            .into());
+        }
+        self.task_cancellations
+            .insert(task_control_key(&tenant, &project, &process, &node, &task));
+        Ok(CoordinatorResponse::TaskCancellationRequested {
+            process,
+            task,
+            node,
+        })
+    }
+
+    pub(super) fn handle_cancel_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let _actor_user = actor_user;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "process cancellation requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+        self.process_cancellations
+            .insert(process_control_key(&tenant, &project, &process));
+        self.main_runtime.interrupt_process(
+            &tenant,
+            &project,
+            &process,
+            "virtual process cancellation requested",
+        );
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        let mut cancelled_tasks = Vec::new();
+        let mut affected_nodes = BTreeSet::new();
+        for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
+            if task_tenant == &tenant && task_project == &project && task_process == &process {
+                self.task_cancellations
+                    .insert(task_control_key(&tenant, &project, &process, node, task));
+                affected_nodes.insert(node.clone());
+                cancelled_tasks.push(TaskCancellationTarget {
+                    process: process.clone(),
+                    task: task.clone(),
+                    node: node.clone(),
+                });
+            }
+        }
+        let process_key = process_control_key(&tenant, &project, &process);
+        if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
+            self.coordinator
+                .abort_process(&tenant, &project, &process)?;
+            self.clear_operator_panel_state(&tenant, &project, &process);
+        }
+        Ok(CoordinatorResponse::ProcessCancellationRequested {
+            process,
+            cancelled_tasks,
+            affected_nodes: affected_nodes.into_iter().collect(),
+        })
+    }
+
+    pub(super) fn handle_abort_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let _actor_user = actor_user;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "process abort requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+
+        let process_key = process_control_key(&tenant, &project, &process);
+        self.process_cancellations.remove(&process_key);
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.process_aborts.insert(process_key);
+        self.main_runtime
+            .interrupt_process(&tenant, &project, &process, "virtual process aborted");
+        self.main_runtime
+            .controls
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.clear_operator_panel_state(&tenant, &project, &process);
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        self.task_assignments.retain(|_, assignments| {
+            assignments.retain(|assignment| {
+                assignment.tenant != tenant
+                    || assignment.project != project
+                    || assignment.process != process
+            });
+            !assignments.is_empty()
+        });
+        let mut aborted_tasks = Vec::new();
+        let mut affected_nodes = BTreeSet::new();
+        for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
+            if task_tenant == &tenant && task_project == &project && task_process == &process {
+                self.task_aborts
+                    .insert(task_control_key(&tenant, &project, &process, node, task));
+                affected_nodes.insert(node.clone());
+                aborted_tasks.push(TaskCancellationTarget {
+                    process: process.clone(),
+                    task: task.clone(),
+                    node: node.clone(),
+                });
+            }
+        }
+
+        self.coordinator
+            .abort_process(&tenant, &project, &process)?;
+        let active_restart_tasks = aborted_tasks
+            .iter()
+            .map(|target| target.task.clone())
+            .collect::<BTreeSet<_>>();
+        self.task_restart_checkpoints.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task), _| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+                    || active_restart_tasks.contains(checkpoint_task)
+            },
+        );
+        self.task_restart_checkpoint_order.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task)| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+                    || active_restart_tasks.contains(checkpoint_task)
+            },
+        );
+        if aborted_tasks.is_empty() {
+            self.process_aborts
+                .remove(&process_control_key(&tenant, &project, &process));
+        }
+        Ok(CoordinatorResponse::ProcessAborted {
+            process,
+            aborted_tasks,
+            affected_nodes: affected_nodes.into_iter().collect(),
+        })
+    }
+
+    pub(super) fn handle_list_processes(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let processes = self
+            .coordinator
+            .active_processes_for_project(&tenant, &project)
+            .into_iter()
+            .map(|active| {
+                let process_key = process_control_key(&active.tenant, &active.project, &active.id);
+                let main = self.main_runtime.controls.get(&process_key);
+                let state = if self.process_cancellations.contains(&process_key) {
+                    "cancelling"
+                } else {
+                    main.map_or("running", |main| main.state.as_str())
+                };
+                let main_wait_state = main.and_then(|main| {
+                    if main.state != "running" {
+                        return None;
+                    }
+                    if self.pending_task_launches.iter().any(|pending| {
+                        pending.tenant == active.tenant
+                            && pending.project == active.project
+                            && pending.process == active.id
+                    }) {
+                        Some("waiting_for_node".to_owned())
+                    } else if self.main_runtime.is_waiting_for_task(
+                        &active.tenant,
+                        &active.project,
+                        &active.id,
+                    ) {
+                        Some("waiting_for_task".to_owned())
+                    } else {
+                        Some("executing".to_owned())
+                    }
+                });
+                VirtualProcessStatus {
+                    process: active.id,
+                    state: state.to_owned(),
+                    main_task_definition: main.map(|main| main.task_definition.clone()),
+                    main_task_instance: main.map(|main| main.task_instance.clone()),
+                    main_state: main.map(|main| main.state.clone()),
+                    main_wait_state,
+                    main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
+                    connected_nodes: active.connected_nodes.into_iter().collect(),
+                    coordinator_epoch: active.coordinator_epoch,
+                }
+            })
+            .collect();
+        Ok(CoordinatorResponse::ProcessStatuses { processes, actor })
+    }
+
+    pub(super) fn handle_poll_task_control(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let cancel_requested = self
+            .task_cancellations
+            .contains(&task_control_key(&tenant, &project, &process, &node, &task))
+            || self
+                .process_cancellations
+                .contains(&process_control_key(&tenant, &project, &process));
+        let abort_requested = self
+            .task_aborts
+            .contains(&task_control_key(&tenant, &project, &process, &node, &task))
+            || self
+                .process_aborts
+                .contains(&process_control_key(&tenant, &project, &process));
+        Ok(CoordinatorResponse::TaskControl {
+            process,
+            task,
+            cancel_requested,
+            abort_requested,
+        })
+    }
+
+    pub(super) fn workflow_actor(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        request_kind: &str,
+        process: &ProcessId,
+        task: Option<&TaskInstanceId>,
+    ) -> Result<WorkflowActor, CoordinatorServiceError> {
+        if let Some(agent) = actor_agent {
+            let agent = AgentId::new(agent);
+            let signature = agent_signature.ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent workflow dispatch requires a signed request proving private-key possession"
+                        .to_owned(),
+                )
+            })?;
+            let request_payload_digest = request_payload_digest.ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent workflow dispatch requires a canonical signed request payload"
+                        .to_owned(),
+                )
+            })?;
+            if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request nonce is missing or invalid".to_owned(),
+                )
+                .into());
+            }
+            let now_epoch_seconds = unix_timestamp_seconds();
+            let replay_key = (
+                tenant.clone(),
+                project.clone(),
+                agent.clone(),
+                signature.nonce.clone(),
+            );
+            const AGENT_SIGNATURE_WINDOW_SECONDS: u64 = 300;
+            self.agent_replay_nonces.retain(|_, issued_at| {
+                now_epoch_seconds <= issued_at.saturating_add(AGENT_SIGNATURE_WINDOW_SECONDS)
+            });
+            if self.agent_replay_nonces.contains_key(&replay_key) {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request nonce has already been used".to_owned(),
+                )
+                .into());
+            }
+            let canonical_scope = clusterflux_core::AgentWorkflowRequestScope::new(
+                tenant.clone(),
+                project.clone(),
+                request_kind,
+                process.clone(),
+                task.cloned(),
+            )
+            .map_err(CoordinatorError::Unauthorized)?;
+            let record = self.coordinator.authorize_agent_project_run(
+                canonical_scope.for_agent(&agent),
+                agent_public_key_fingerprint.as_ref(),
+                request_payload_digest,
+                &signature,
+                now_epoch_seconds,
+            )?;
+            if self
+                .agent_replay_nonces
+                .keys()
+                .filter(|(retained_tenant, retained_project, retained_agent, _)| {
+                    retained_tenant == tenant
+                        && retained_project == project
+                        && retained_agent == &agent
+                })
+                .count()
+                >= super::MAX_REPLAY_NONCES_PER_AUTHORITY
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request replay window is full; retry after the bounded signature window advances"
+                        .to_owned(),
+                )
+                .into());
+            }
+            self.agent_replay_nonces
+                .insert(replay_key, signature.issued_at_epoch_seconds);
+            return Ok(WorkflowActor {
+                kind: "agent".to_owned(),
+                user: Some(record.user),
+                agent: Some(agent),
+                credential_kind: CredentialKind::PublicKey,
+                public_key_fingerprint: Some(record.public_key_fingerprint),
+                authenticated_without_browser: true,
+                scopes: record.scopes,
+            });
+        }
+
+        let actor = UserId::new(actor_user.unwrap_or_else(|| "user".to_owned()));
+        Ok(WorkflowActor {
+            kind: "user".to_owned(),
+            user: Some(actor),
+            agent: None,
+            credential_kind: CredentialKind::BrowserSession,
+            public_key_fingerprint: None,
+            authenticated_without_browser: false,
+            scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
+        })
+    }
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}

crates/clusterflux-coordinator/src/service/protocol.rs

@@ -0,0 +1,675 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind,
+    DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements,
+    LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest,
+    PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation,
+    SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId,
+    UserId, VfsPath,
+};
+use serde::{Deserialize, Serialize};
+
+mod responses;
+pub use responses::*;
+
+use crate::{AgentPublicKeyRecord, ProjectRecord};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TaskReplacementBundle {
+    pub bundle_digest: Digest,
+    pub wasm_module_base64: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub source_snapshot: Option<Digest>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskFailureResolution {
+    AcceptFailure,
+    Cancel,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum CoordinatorRequest {
+    Ping,
+    Authenticated {
+        session_secret: String,
+        request: AuthenticatedCoordinatorRequest,
+    },
+    AuthStatus {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    AdminStatus {
+        tenant: String,
+        actor_user: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    },
+    SuspendTenant {
+        tenant: String,
+        actor_user: String,
+        target_tenant: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    },
+    CreateProject {
+        tenant: String,
+        actor_user: String,
+        project: String,
+        name: String,
+    },
+    SelectProject {
+        tenant: String,
+        actor_user: String,
+        project: String,
+    },
+    ListProjects {
+        tenant: String,
+        actor_user: String,
+    },
+    RegisterAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+        public_key: String,
+    },
+    ListAgentPublicKeys {
+        tenant: String,
+        project: String,
+        user: String,
+    },
+    RotateAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+    },
+    AttachNode {
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+    },
+    CreateNodeEnrollmentGrant {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        #[serde(default = "default_node_enrollment_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    ExchangeNodeEnrollmentGrant {
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+        enrollment_grant: String,
+    },
+    NodeHeartbeat {
+        node: String,
+        #[serde(default)]
+        node_signature: Option<NodeSignedRequest>,
+    },
+    SignedNode {
+        node: String,
+        node_signature: NodeSignedRequest,
+        request: Box<CoordinatorRequest>,
+    },
+    ReportNodeCapabilities {
+        tenant: String,
+        project: String,
+        node: String,
+        capabilities: NodeCapabilities,
+        cached_environment_digests: Vec<Digest>,
+        #[serde(default)]
+        dependency_cache_digests: Vec<Digest>,
+        source_snapshots: Vec<Digest>,
+        artifact_locations: Vec<String>,
+        direct_connectivity: bool,
+        online: bool,
+    },
+    ListNodeDescriptors {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    RevokeNodeCredential {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        node: String,
+    },
+    ScheduleTask {
+        tenant: String,
+        project: String,
+        environment: Option<EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<Capability>,
+        #[serde(default)]
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    },
+    LaunchTask {
+        tenant: String,
+        project: String,
+        #[serde(default)]
+        actor_user: Option<String>,
+        #[serde(default)]
+        actor_agent: Option<String>,
+        #[serde(default)]
+        agent_public_key_fingerprint: Option<Digest>,
+        #[serde(default)]
+        agent_signature: Option<AgentSignedRequest>,
+        task_spec: TaskSpec,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    LaunchChildTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task_spec: TaskSpec,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    JoinChildTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task: String,
+    },
+    PollTaskAssignment {
+        tenant: String,
+        project: String,
+        node: String,
+    },
+    PollArtifactTransfer {
+        tenant: String,
+        project: String,
+        node: String,
+    },
+    UploadArtifactTransferChunk {
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        offset: u64,
+        content_base64: String,
+        chunk_digest: Digest,
+        eof: bool,
+    },
+    FailArtifactTransfer {
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        message: String,
+    },
+    RequestRendezvous {
+        scope: DataPlaneScope,
+        source: NodeEndpoint,
+        destination: NodeEndpoint,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+    RequestSourcePreparation {
+        tenant: String,
+        project: String,
+        provider: SourceProviderKind,
+    },
+    CompleteSourcePreparation {
+        tenant: String,
+        project: String,
+        node: String,
+        provider: SourceProviderKind,
+        source_snapshot: Digest,
+    },
+    StartProcess {
+        tenant: String,
+        project: String,
+        #[serde(default)]
+        actor_user: Option<String>,
+        #[serde(default)]
+        actor_agent: Option<String>,
+        #[serde(default)]
+        agent_public_key_fingerprint: Option<Digest>,
+        #[serde(default)]
+        agent_signature: Option<AgentSignedRequest>,
+        process: String,
+        #[serde(default)]
+        restart: bool,
+    },
+    ReconnectNode {
+        node: String,
+        process: String,
+        epoch: u64,
+    },
+    CancelTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    CancelProcess {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    AbortProcess {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    ListProcesses {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    QuotaStatus {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    PollTaskControl {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    RestartTask {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        #[serde(default)]
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    SetDebugBreakpoints {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        probe_symbols: Vec<String>,
+    },
+    InspectDebugBreakpoints {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    CreateDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        stopped_task: String,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        epoch: u64,
+    },
+    PollDebugCommand {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    ReportDebugState {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        epoch: u64,
+        state: DebugAcknowledgementState,
+        #[serde(default)]
+        stack_frames: Vec<String>,
+        #[serde(default)]
+        local_values: Vec<(String, String)>,
+        #[serde(default)]
+        task_args: Vec<(String, String)>,
+        #[serde(default)]
+        handles: Vec<(String, String)>,
+        #[serde(default)]
+        command_status: Option<String>,
+        #[serde(default)]
+        recent_output: Vec<String>,
+        #[serde(default)]
+        message: Option<String>,
+    },
+    ReportDebugProbeHit {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        probe_symbol: String,
+    },
+    ReportTaskLog {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        #[serde(default)]
+        stdout_tail: String,
+        #[serde(default)]
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        backpressured: bool,
+    },
+    ReportVfsMetadata {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        large_bytes_uploaded: bool,
+    },
+    TaskCompleted {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        #[serde(default)]
+        terminal_state: Option<TaskTerminalState>,
+        status_code: Option<i32>,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        #[serde(default)]
+        stdout_tail: String,
+        #[serde(default)]
+        stderr_tail: String,
+        #[serde(default)]
+        stdout_truncated: bool,
+        #[serde(default)]
+        stderr_truncated: bool,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        #[serde(default)]
+        result: Option<TaskBoundaryValue>,
+    },
+    ListTaskEvents {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        #[serde(default)]
+        process: Option<String>,
+    },
+    ListTaskSnapshots {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    JoinTask {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+    },
+    RenderOperatorPanel {
+        tenant: String,
+        project: String,
+        process: String,
+        actor_user: String,
+        max_download_bytes: u64,
+        stopped: bool,
+    },
+    SubmitPanelEvent {
+        tenant: String,
+        project: String,
+        process: String,
+        widget_id: String,
+        kind: PanelEventKind,
+        max_events: u64,
+    },
+    CreateArtifactDownloadLink {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        #[serde(default = "default_download_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        token_digest: Digest,
+    },
+    ExportArtifactToNode {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+}
+
+impl CoordinatorRequest {
+    pub fn operation(&self) -> Result<String, String> {
+        serde_json::to_value(self)
+            .map_err(|err| format!("failed to encode coordinator request operation: {err}"))
+            .map(|value| clusterflux_core::coordinator_payload_operation(&value))
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum AuthenticatedCoordinatorRequest {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject {
+        project: String,
+        name: String,
+    },
+    SelectProject {
+        project: String,
+    },
+    ListProjects,
+    RegisterAgentPublicKey {
+        agent: String,
+        public_key: String,
+    },
+    ListAgentPublicKeys,
+    RotateAgentPublicKey {
+        agent: String,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        agent: String,
+    },
+    CreateNodeEnrollmentGrant {
+        #[serde(default = "default_node_enrollment_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    ListNodeDescriptors,
+    RevokeNodeCredential {
+        node: String,
+    },
+    StartProcess {
+        process: String,
+        #[serde(default)]
+        restart: bool,
+    },
+    ScheduleTask {
+        environment: Option<EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<Capability>,
+        #[serde(default)]
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    },
+    LaunchTask {
+        task_spec: Box<TaskSpec>,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    CancelProcess {
+        process: String,
+    },
+    AbortProcess {
+        process: String,
+    },
+    ListProcesses,
+    QuotaStatus,
+    RestartTask {
+        process: String,
+        task: String,
+        #[serde(default)]
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: String,
+    },
+    SetDebugBreakpoints {
+        process: String,
+        probe_symbols: Vec<String>,
+    },
+    InspectDebugBreakpoints {
+        process: String,
+    },
+    CreateDebugEpoch {
+        process: String,
+        stopped_task: String,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        process: String,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        process: String,
+        epoch: u64,
+    },
+    ListTaskEvents {
+        #[serde(default)]
+        process: Option<String>,
+    },
+    ListTaskSnapshots {
+        process: String,
+    },
+    JoinTask {
+        process: String,
+        task: String,
+    },
+    CreateArtifactDownloadLink {
+        artifact: String,
+        max_bytes: u64,
+        #[serde(default = "default_download_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        artifact: String,
+        token_digest: Digest,
+    },
+    ExportArtifactToNode {
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+}
+
+fn default_download_ttl_seconds() -> u64 {
+    900
+}
+
+fn default_node_enrollment_ttl_seconds() -> u64 {
+    900
+}

crates/clusterflux-coordinator/src/service/protocol/responses.rs

@@ -0,0 +1,545 @@
+use super::*;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskTerminalState {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+impl TaskTerminalState {
+    pub(crate) fn from_status_code(status_code: Option<i32>) -> Self {
+        match status_code {
+            Some(0) => Self::Completed,
+            _ => Self::Failed,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskExecutor {
+    CoordinatorMain,
+    Node,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCompletionEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub node: NodeId,
+    pub executor: TaskExecutor,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub task: TaskInstanceId,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub attempt_id: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub placement: Option<Placement>,
+    pub terminal_state: TaskTerminalState,
+    pub status_code: Option<i32>,
+    pub stdout_bytes: u64,
+    pub stderr_bytes: u64,
+    pub stdout_tail: String,
+    pub stderr_tail: String,
+    pub stdout_truncated: bool,
+    pub stderr_truncated: bool,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub result: Option<TaskBoundaryValue>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskAttemptState {
+    Queued,
+    Running,
+    FailedAwaitingAction,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAttemptSnapshot {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub attempt_number: u32,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub display_name: String,
+    pub state: TaskAttemptState,
+    pub current: bool,
+    pub node: Option<NodeId>,
+    pub environment_id: Option<String>,
+    pub environment_digest: Option<Digest>,
+    pub argument_summary: Vec<String>,
+    pub handle_summary: Vec<String>,
+    pub command_state: Option<String>,
+    pub vfs_checkpoint: String,
+    pub probe_symbol: Option<String>,
+    pub source_path: Option<String>,
+    pub source_line: Option<u32>,
+    pub restart_compatible: bool,
+    pub failure_policy: clusterflux_core::TaskFailurePolicy,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub status_code: Option<i32>,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAuditEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+    pub actor: UserId,
+    pub operation: String,
+    pub allowed: bool,
+    pub reason: String,
+    pub charged_debug_read_bytes: u64,
+    pub used_debug_read_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WorkflowActor {
+    pub kind: String,
+    pub user: Option<UserId>,
+    pub agent: Option<AgentId>,
+    pub credential_kind: CredentialKind,
+    pub public_key_fingerprint: Option<Digest>,
+    pub authenticated_without_browser: bool,
+    pub scopes: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAssignment {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+    pub epoch: u64,
+    pub artifact_path: String,
+    pub task_spec: TaskSpec,
+    pub wasm_module_base64: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactTransferAssignment {
+    pub transfer_id: String,
+    pub artifact: ArtifactId,
+    pub expected_digest: Digest,
+    pub expected_size_bytes: u64,
+    pub offset: u64,
+    pub max_chunk_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCancellationTarget {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DebugAcknowledgementState {
+    Frozen,
+    Running,
+    Failed,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipantAcknowledgement {
+    pub node: NodeId,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub epoch: u64,
+    pub state: DebugAcknowledgementState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct VirtualProcessStatus {
+    pub process: ProcessId,
+    pub state: String,
+    pub main_task_definition: Option<clusterflux_core::TaskDefinitionId>,
+    pub main_task_instance: Option<TaskInstanceId>,
+    pub main_state: Option<String>,
+    pub main_wait_state: Option<String>,
+    pub main_debug_epoch: Option<u64>,
+    pub connected_nodes: Vec<NodeId>,
+    pub coordinator_epoch: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SourcePreparationDisposition {
+    Pending { reason: String },
+    Assigned { node: NodeId },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourcePreparationStatus {
+    pub preparation: SourcePreparation,
+    pub disposition: SourcePreparationDisposition,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum CoordinatorResponse {
+    Pong {
+        epoch: u64,
+    },
+    AuthStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        authenticated: bool,
+        account_status: String,
+        suspended: bool,
+        disabled: bool,
+        deleted: bool,
+        manual_review: bool,
+        sanitized_reason: Option<String>,
+        next_actions: Vec<String>,
+        private_moderation_details_exposed: bool,
+        signup_failure_details_exposed: bool,
+    },
+    AdminStatus {
+        tenant: TenantId,
+        actor: UserId,
+        suspended: bool,
+        safe_default: String,
+    },
+    TenantSuspended {
+        tenant: TenantId,
+        actor: UserId,
+        policy: crate::ServicePolicyRecord,
+    },
+    ProjectCreated {
+        project: ProjectRecord,
+        actor: UserId,
+    },
+    ProjectSelected {
+        project: ProjectRecord,
+        actor: UserId,
+    },
+    Projects {
+        projects: Vec<ProjectRecord>,
+        actor: UserId,
+    },
+    CliSessionRevoked {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+    },
+    AgentPublicKey {
+        record: AgentPublicKeyRecord,
+        actor: UserId,
+    },
+    AgentPublicKeys {
+        records: Vec<AgentPublicKeyRecord>,
+        actor: UserId,
+    },
+    NodeAttached {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+    },
+    NodeEnrollmentGrantCreated {
+        tenant: TenantId,
+        project: ProjectId,
+        grant: String,
+        scope: String,
+        expires_at_epoch_seconds: u64,
+    },
+    NodeEnrollmentExchanged {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        credential: clusterflux_core::NodeCredential,
+    },
+    NodeHeartbeat {
+        node: NodeId,
+        epoch: u64,
+    },
+    NodeCapabilitiesRecorded {
+        node: NodeId,
+        node_descriptors: usize,
+    },
+    NodeDescriptors {
+        descriptors: Vec<NodeDescriptor>,
+        actor: UserId,
+    },
+    NodeCredentialRevoked {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        descriptor_removed: bool,
+        queued_assignments_removed: usize,
+    },
+    TaskPlacement {
+        placement: Placement,
+    },
+    TaskLaunched {
+        process: ProcessId,
+        task: TaskInstanceId,
+        actor: WorkflowActor,
+        placement: Placement,
+        assignment: Box<TaskAssignment>,
+        charged_spawns: u64,
+    },
+    MainLaunched {
+        process: ProcessId,
+        task_definition: clusterflux_core::TaskDefinitionId,
+        task_instance: TaskInstanceId,
+        actor: WorkflowActor,
+        state: String,
+    },
+    TaskQueued {
+        process: ProcessId,
+        task: TaskInstanceId,
+        actor: WorkflowActor,
+        reason: String,
+        charged_spawns: u64,
+        queued_tasks: usize,
+    },
+    TaskAssignment {
+        assignment: Option<Box<TaskAssignment>>,
+    },
+    ArtifactTransferAssignment {
+        transfer: Option<ArtifactTransferAssignment>,
+    },
+    ArtifactTransferChunkAccepted {
+        transfer_id: String,
+        next_offset: u64,
+        complete: bool,
+    },
+    ArtifactTransferFailed {
+        transfer_id: String,
+    },
+    RendezvousPlan {
+        plan: DirectBulkTransferPlan,
+        charged_rendezvous_attempts: u64,
+    },
+    SourcePreparation {
+        status: SourcePreparationStatus,
+    },
+    SourcePreparationCompleted {
+        node: NodeId,
+        provider: SourceProviderKind,
+        source_snapshot: Digest,
+    },
+    ProcessStarted {
+        process: ProcessId,
+        epoch: u64,
+        actor: WorkflowActor,
+        charged_spawns: u64,
+    },
+    NodeReconnected {
+        node: NodeId,
+        process: ProcessId,
+    },
+    TaskCancellationRequested {
+        process: ProcessId,
+        task: TaskInstanceId,
+        node: NodeId,
+    },
+    ProcessCancellationRequested {
+        process: ProcessId,
+        cancelled_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessAborted {
+        process: ProcessId,
+        aborted_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessStatuses {
+        processes: Vec<VirtualProcessStatus>,
+        actor: UserId,
+    },
+    QuotaStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        policy_label: Option<String>,
+        limits: ResourceLimits,
+        window_seconds: BTreeMap<LimitKind, u64>,
+        usage: BTreeMap<LimitKind, u64>,
+        window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+    },
+    TaskControl {
+        process: ProcessId,
+        task: TaskInstanceId,
+        cancel_requested: bool,
+        abort_requested: bool,
+    },
+    TaskRestart {
+        process: ProcessId,
+        task: TaskInstanceId,
+        restarted_task_instance: Option<clusterflux_core::TaskInstanceId>,
+        restarted_attempt_id: Option<String>,
+        actor: UserId,
+        accepted: bool,
+        clean_boundary_available: bool,
+        active_task: bool,
+        completed_event_observed: bool,
+        requires_whole_process_restart: bool,
+        message: String,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugCommand {
+        process: ProcessId,
+        task: TaskInstanceId,
+        epoch: Option<u64>,
+        command: Option<String>,
+    },
+    DebugStateRecorded {
+        process: ProcessId,
+        node: NodeId,
+        task: TaskInstanceId,
+        epoch: u64,
+        state: DebugAcknowledgementState,
+    },
+    DebugAttach {
+        process: ProcessId,
+        actor: UserId,
+        authorization: Authorization,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugBreakpoints {
+        process: ProcessId,
+        actor: UserId,
+        probe_symbols: Vec<String>,
+        hit_epoch: Option<u64>,
+        hit_task: Option<TaskInstanceId>,
+        hit_probe_symbol: Option<String>,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugProbeHit {
+        process: ProcessId,
+        node: NodeId,
+        task: TaskInstanceId,
+        probe_symbol: String,
+        breakpoint_matched: bool,
+        debug_epoch: Option<u64>,
+    },
+    DebugEpoch {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        affected_tasks: Vec<TaskCancellationTarget>,
+        all_stop_requested: bool,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugEpochStatus {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        expected_tasks: Vec<TaskCancellationTarget>,
+        acknowledgements: Vec<DebugParticipantAcknowledgement>,
+        fully_frozen: bool,
+        partially_frozen: bool,
+        fully_resumed: bool,
+        failed: bool,
+        failure_messages: Vec<String>,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    TaskLogRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        backpressured: bool,
+    },
+    VfsMetadataRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        artifact_path: Option<VfsPath>,
+        large_bytes_uploaded: bool,
+    },
+    TaskRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        events_recorded: usize,
+    },
+    TaskEvents {
+        events: Vec<TaskCompletionEvent>,
+    },
+    TaskSnapshots {
+        snapshots: Vec<TaskAttemptSnapshot>,
+    },
+    TaskFailureResolved {
+        process: ProcessId,
+        task: TaskInstanceId,
+        attempt_id: String,
+        resolution: TaskFailureResolution,
+    },
+    TaskJoined {
+        join: TaskJoinResult,
+    },
+    OperatorPanel {
+        panel: PanelState,
+    },
+    PanelEventAccepted {
+        used_events: u64,
+        max_events: u64,
+    },
+    ArtifactDownloadLink {
+        link: DownloadLink,
+    },
+    ArtifactDownloadLinkRevoked {
+        link: DownloadLink,
+    },
+    ArtifactDownloadStream {
+        link: DownloadLink,
+        streamed_bytes: u64,
+        charged_download_bytes: u64,
+        content_bytes_available: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_offset: Option<u64>,
+        #[serde(default)]
+        content_eof: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_base64: Option<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_source: Option<String>,
+    },
+    ArtifactExportPlan {
+        plan: DirectBulkTransferPlan,
+        source_node: NodeId,
+        receiver_node: NodeId,
+        artifact_size_bytes: u64,
+    },
+    Error {
+        message: String,
+    },
+}

crates/clusterflux-coordinator/src/service/quota.rs

@@ -0,0 +1,484 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{LimitError, LimitKind, ProjectId, ResourceLimits, ResourceMeter, TenantId};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CoordinatorQuotaConfiguration {
+    pub limits: ResourceLimits,
+    pub window_seconds: BTreeMap<LimitKind, u64>,
+    pub policy_label: Option<String>,
+    pub max_projects_per_tenant: usize,
+    pub max_nodes_per_tenant: usize,
+    pub max_active_processes_per_tenant: usize,
+}
+
+pub(super) struct CoordinatorQuotaStatus {
+    pub(super) policy_label: Option<String>,
+    pub(super) limits: ResourceLimits,
+    pub(super) window_seconds: BTreeMap<LimitKind, u64>,
+    pub(super) usage: BTreeMap<LimitKind, u64>,
+    pub(super) window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+}
+
+impl CoordinatorQuotaConfiguration {
+    pub fn new(
+        limits: ResourceLimits,
+        window_seconds: impl IntoIterator<Item = (LimitKind, u64)>,
+    ) -> Result<Self, String> {
+        let window_seconds = window_seconds.into_iter().collect::<BTreeMap<_, _>>();
+        if window_seconds.values().any(|seconds| *seconds == 0) {
+            return Err("quota windows must be at least one second".to_owned());
+        }
+        Ok(Self {
+            limits,
+            window_seconds,
+            policy_label: None,
+            max_projects_per_tenant: usize::MAX,
+            max_nodes_per_tenant: usize::MAX,
+            max_active_processes_per_tenant: usize::MAX,
+        })
+    }
+
+    pub fn with_policy_label(mut self, label: impl Into<String>) -> Self {
+        let label = label.into();
+        self.policy_label = (!label.trim().is_empty()).then_some(label);
+        self
+    }
+
+    pub fn with_admission_limits(
+        mut self,
+        max_projects_per_tenant: usize,
+        max_nodes_per_tenant: usize,
+        max_active_processes_per_tenant: usize,
+    ) -> Self {
+        self.max_projects_per_tenant = max_projects_per_tenant;
+        self.max_nodes_per_tenant = max_nodes_per_tenant;
+        self.max_active_processes_per_tenant = max_active_processes_per_tenant;
+        self
+    }
+
+    pub fn unlimited() -> Self {
+        Self {
+            limits: ResourceLimits::unlimited(),
+            window_seconds: LimitKind::ALL
+                .into_iter()
+                .map(|kind| (kind, u64::MAX))
+                .collect(),
+            policy_label: None,
+            max_projects_per_tenant: usize::MAX,
+            max_nodes_per_tenant: usize::MAX,
+            max_active_processes_per_tenant: usize::MAX,
+        }
+    }
+
+    pub fn window_seconds(&self, kind: LimitKind) -> u64 {
+        self.window_seconds
+            .get(&kind)
+            .copied()
+            .unwrap_or(u64::MAX)
+            .max(1)
+    }
+}
+
+impl Default for CoordinatorQuotaConfiguration {
+    fn default() -> Self {
+        Self::unlimited()
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct ProjectQuotaScope {
+    tenant: TenantId,
+    project: ProjectId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct MeterKey {
+    scope: ProjectQuotaScope,
+    kind: LimitKind,
+    window: u64,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct CoordinatorQuota {
+    configuration: CoordinatorQuotaConfiguration,
+    meters: BTreeMap<MeterKey, ResourceMeter>,
+}
+
+impl Default for CoordinatorQuota {
+    fn default() -> Self {
+        Self::new(CoordinatorQuotaConfiguration::default())
+    }
+}
+
+impl CoordinatorQuota {
+    pub(super) fn new(configuration: CoordinatorQuotaConfiguration) -> Self {
+        Self {
+            configuration,
+            meters: BTreeMap::new(),
+        }
+    }
+
+    pub(super) fn ensure_project_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_projects_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.project_limit: tenant {tenant} already has the maximum of {maximum} projects"
+            )));
+        }
+        Ok(())
+    }
+
+    pub(super) fn ensure_node_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_nodes_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.node_limit: tenant {tenant} already has the maximum of {maximum} nodes"
+            )));
+        }
+        Ok(())
+    }
+
+    pub(super) fn ensure_process_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_active_processes_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.active_process_limit: tenant {tenant} already has the maximum of {maximum} active processes"
+            )));
+        }
+        Ok(())
+    }
+
+    fn key(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> MeterKey {
+        MeterKey {
+            scope: ProjectQuotaScope {
+                tenant: tenant.clone(),
+                project: project.clone(),
+            },
+            kind,
+            window: now_epoch_seconds / self.configuration.window_seconds(kind),
+        }
+    }
+
+    fn meter(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> Option<&ResourceMeter> {
+        self.meters
+            .get(&self.key(tenant, project, kind, now_epoch_seconds))
+    }
+
+    fn meter_mut(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> &mut ResourceMeter {
+        let key = self.key(tenant, project, kind, now_epoch_seconds);
+        self.meters.retain(|existing, _| {
+            existing.scope != key.scope || existing.kind != kind || existing.window == key.window
+        });
+        self.meters.entry(key).or_default()
+    }
+
+    fn can_charge(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        amount: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.meter(tenant, project, kind, now_epoch_seconds)
+            .cloned()
+            .unwrap_or_default()
+            .can_charge(&self.configuration.limits, kind, amount)
+    }
+
+    fn charge(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        amount: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        let limits = self.configuration.limits.clone();
+        let meter = self.meter_mut(tenant, project, kind, now_epoch_seconds);
+        meter.charge(&limits, kind, amount)?;
+        Ok(meter.used(&kind))
+    }
+
+    fn used(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.meter(tenant, project, kind, now_epoch_seconds)
+            .map_or(0, |meter| meter.used(&kind))
+    }
+
+    pub(super) fn can_charge_workflow_spawn(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
+    }
+
+    pub(super) fn charge_api_call(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
+    }
+
+    pub(super) fn can_charge_log_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(
+            tenant,
+            project,
+            LimitKind::LogBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_log_bytes(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::LogBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_workflow_spawn(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_workflow_spawns(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::Spawn, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_api_calls(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::ApiCall, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_log_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::LogBytes, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn active_meter_count(&self) -> usize {
+        self.meters.len()
+    }
+
+    pub(super) fn charge_rendezvous_attempt(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::RendezvousAttempt,
+            1,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn can_charge_download(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_download(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn download_limit(&self) -> u64 {
+        self.configuration
+            .limits
+            .limit(&LimitKind::ArtifactDownloadBytes)
+    }
+
+    pub(super) fn used_download_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_debug_read(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::DebugReadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn used_debug_read_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(
+            tenant,
+            project,
+            LimitKind::DebugReadBytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn project_status(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> CoordinatorQuotaStatus {
+        let mut usage = BTreeMap::new();
+        let mut window_starts = BTreeMap::new();
+        for kind in LimitKind::ALL {
+            let seconds = self.configuration.window_seconds(kind);
+            usage.insert(kind, self.used(tenant, project, kind, now_epoch_seconds));
+            window_starts.insert(kind, (now_epoch_seconds / seconds).saturating_mul(seconds));
+        }
+        CoordinatorQuotaStatus {
+            policy_label: self.configuration.policy_label.clone(),
+            limits: self.configuration.limits.clone(),
+            window_seconds: self.configuration.window_seconds.clone(),
+            usage,
+            window_started_epoch_seconds: window_starts,
+        }
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_workflow_limits(&mut self, limits: ResourceLimits) {
+        self.configuration
+            .limits
+            .limits
+            .insert(LimitKind::Spawn, limits.limit(&LimitKind::Spawn));
+        self.meters.clear();
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_download_limits(&mut self, limits: ResourceLimits) {
+        self.configuration.limits.limits.insert(
+            LimitKind::ArtifactDownloadBytes,
+            limits.limit(&LimitKind::ArtifactDownloadBytes),
+        );
+        self.meters.clear();
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_rendezvous_limits(&mut self, limits: ResourceLimits) {
+        self.configuration.limits.limits.insert(
+            LimitKind::RendezvousAttempt,
+            limits.limit(&LimitKind::RendezvousAttempt),
+        );
+        self.meters.clear();
+    }
+}

crates/clusterflux-coordinator/src/service/relay.rs

@@ -0,0 +1,590 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{ProjectId, TenantId, UserId};
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ArtifactRelayConfiguration {
+    pub enabled: bool,
+    pub max_artifact_bytes: u64,
+    pub max_active_per_project: usize,
+    pub max_active_per_tenant: usize,
+    pub max_active_per_account: usize,
+    pub max_active_global: usize,
+    pub max_period_bytes_per_project: u64,
+    pub max_period_bytes_per_tenant: u64,
+    pub max_period_bytes_per_account: u64,
+    pub period_seconds: u64,
+    pub framing_overhead_bytes: u64,
+    pub max_tracked_scopes: usize,
+}
+
+impl ArtifactRelayConfiguration {
+    pub fn unlimited() -> Self {
+        Self {
+            enabled: true,
+            max_artifact_bytes: u64::MAX,
+            max_active_per_project: usize::MAX,
+            max_active_per_tenant: usize::MAX,
+            max_active_per_account: usize::MAX,
+            max_active_global: usize::MAX,
+            max_period_bytes_per_project: u64::MAX,
+            max_period_bytes_per_tenant: u64::MAX,
+            max_period_bytes_per_account: u64::MAX,
+            period_seconds: u64::MAX,
+            framing_overhead_bytes: 512,
+            max_tracked_scopes: usize::MAX,
+        }
+    }
+
+    pub fn validate(&self) -> Result<(), ArtifactRelayError> {
+        if self.period_seconds == 0
+            || self.max_active_per_project == 0
+            || self.max_active_per_tenant == 0
+            || self.max_active_per_account == 0
+            || self.max_active_global == 0
+            || self.max_tracked_scopes == 0
+        {
+            return Err(ArtifactRelayError::InvalidConfiguration);
+        }
+        Ok(())
+    }
+}
+
+impl Default for ArtifactRelayConfiguration {
+    fn default() -> Self {
+        Self::unlimited()
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ArtifactRelayUsage {
+    pub active_transfers: usize,
+    pub ingress_bytes: u64,
+    pub egress_bytes: u64,
+    pub abandoned_or_failed_bytes: u64,
+    pub reserved_bytes: u64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactRelayDurableState {
+    pub reservations: BTreeMap<String, ArtifactRelayReservationState>,
+    pub period: u64,
+    pub project_used: Vec<(TenantId, ProjectId, u64)>,
+    pub tenant_used: Vec<(TenantId, u64)>,
+    pub account_used: Vec<(TenantId, UserId, u64)>,
+    pub ingress_used: u64,
+    pub egress_used: u64,
+    pub abandoned_or_failed_used: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactRelayReservationState {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub account: UserId,
+    pub remaining_reserved_bytes: u64,
+    pub ingress_bytes: u64,
+    pub egress_bytes: u64,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum RelayFinishReason {
+    Completed,
+    Failed,
+    Cancelled,
+    Expired,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ArtifactRelayError {
+    #[error("artifact relay is disabled by hosted policy")]
+    Disabled,
+    #[error("artifact exceeds the configured relay size limit")]
+    ArtifactTooLarge,
+    #[error("artifact relay concurrency limit reached for {0}")]
+    Concurrency(&'static str),
+    #[error("artifact relay is temporarily at process capacity")]
+    Capacity,
+    #[error("artifact relay period byte budget exhausted for {0}")]
+    PeriodBudget(&'static str),
+    #[error("artifact relay retained scope state limit reached")]
+    StateLimit,
+    #[error("artifact relay reservation is missing or expired")]
+    MissingReservation,
+    #[error("artifact relay configuration contains a zero bound")]
+    InvalidConfiguration,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct RelayScope {
+    tenant: TenantId,
+    project: ProjectId,
+    account: UserId,
+}
+
+#[derive(Clone, Debug)]
+struct RelayReservation {
+    scope: RelayScope,
+    remaining_reserved_bytes: u64,
+    ingress_bytes: u64,
+    egress_bytes: u64,
+    expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct ArtifactRelayLedger {
+    configuration: ArtifactRelayConfiguration,
+    reservations: BTreeMap<String, RelayReservation>,
+    period: u64,
+    project_used: BTreeMap<(TenantId, ProjectId), u64>,
+    tenant_used: BTreeMap<TenantId, u64>,
+    account_used: BTreeMap<(TenantId, UserId), u64>,
+    ingress_used: u64,
+    egress_used: u64,
+    abandoned_or_failed_used: u64,
+}
+
+impl Default for ArtifactRelayLedger {
+    fn default() -> Self {
+        Self::new(ArtifactRelayConfiguration::default())
+    }
+}
+
+impl ArtifactRelayLedger {
+    pub(super) fn new(configuration: ArtifactRelayConfiguration) -> Self {
+        Self {
+            configuration,
+            reservations: BTreeMap::new(),
+            period: 0,
+            project_used: BTreeMap::new(),
+            tenant_used: BTreeMap::new(),
+            account_used: BTreeMap::new(),
+            ingress_used: 0,
+            egress_used: 0,
+            abandoned_or_failed_used: 0,
+        }
+    }
+
+    pub(super) fn configure(
+        &mut self,
+        configuration: ArtifactRelayConfiguration,
+    ) -> Result<(), ArtifactRelayError> {
+        configuration.validate()?;
+        self.configuration = configuration;
+        Ok(())
+    }
+
+    pub(super) fn from_durable(
+        configuration: ArtifactRelayConfiguration,
+        state: ArtifactRelayDurableState,
+    ) -> Self {
+        Self {
+            configuration,
+            reservations: state
+                .reservations
+                .into_iter()
+                .map(|(key, reservation)| {
+                    (
+                        key,
+                        RelayReservation {
+                            scope: RelayScope {
+                                tenant: reservation.tenant,
+                                project: reservation.project,
+                                account: reservation.account,
+                            },
+                            remaining_reserved_bytes: reservation.remaining_reserved_bytes,
+                            ingress_bytes: reservation.ingress_bytes,
+                            egress_bytes: reservation.egress_bytes,
+                            expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
+                        },
+                    )
+                })
+                .collect(),
+            period: state.period,
+            project_used: state
+                .project_used
+                .into_iter()
+                .map(|(tenant, project, bytes)| ((tenant, project), bytes))
+                .collect(),
+            tenant_used: state.tenant_used.into_iter().collect(),
+            account_used: state
+                .account_used
+                .into_iter()
+                .map(|(tenant, account, bytes)| ((tenant, account), bytes))
+                .collect(),
+            ingress_used: state.ingress_used,
+            egress_used: state.egress_used,
+            abandoned_or_failed_used: state.abandoned_or_failed_used,
+        }
+    }
+
+    pub(super) fn durable_state(&self) -> ArtifactRelayDurableState {
+        ArtifactRelayDurableState {
+            reservations: self
+                .reservations
+                .iter()
+                .map(|(key, reservation)| {
+                    (
+                        key.clone(),
+                        ArtifactRelayReservationState {
+                            tenant: reservation.scope.tenant.clone(),
+                            project: reservation.scope.project.clone(),
+                            account: reservation.scope.account.clone(),
+                            remaining_reserved_bytes: reservation.remaining_reserved_bytes,
+                            ingress_bytes: reservation.ingress_bytes,
+                            egress_bytes: reservation.egress_bytes,
+                            expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
+                        },
+                    )
+                })
+                .collect(),
+            period: self.period,
+            project_used: self
+                .project_used
+                .iter()
+                .map(|((tenant, project), bytes)| (tenant.clone(), project.clone(), *bytes))
+                .collect(),
+            tenant_used: self
+                .tenant_used
+                .iter()
+                .map(|(tenant, bytes)| (tenant.clone(), *bytes))
+                .collect(),
+            account_used: self
+                .account_used
+                .iter()
+                .map(|((tenant, account), bytes)| (tenant.clone(), account.clone(), *bytes))
+                .collect(),
+            ingress_used: self.ingress_used,
+            egress_used: self.egress_used,
+            abandoned_or_failed_used: self.abandoned_or_failed_used,
+        }
+    }
+
+    pub(super) fn reconcile_after_restart(&mut self, now_epoch_seconds: u64) {
+        let reservations = self.reservations.keys().cloned().collect::<Vec<_>>();
+        for key in reservations {
+            self.finish(&key, RelayFinishReason::Failed);
+        }
+        self.prepare_period(now_epoch_seconds);
+    }
+
+    fn prepare_period(&mut self, now_epoch_seconds: u64) {
+        let period = now_epoch_seconds / self.configuration.period_seconds.max(1);
+        if self.period != period {
+            self.period = period;
+            self.project_used.clear();
+            self.tenant_used.clear();
+            self.account_used.clear();
+            self.ingress_used = 0;
+            self.egress_used = 0;
+            self.abandoned_or_failed_used = 0;
+        }
+    }
+
+    pub(super) fn estimated_wire_bytes(&self, artifact_bytes: u64, chunk_bytes: u64) -> u64 {
+        let encoded = artifact_bytes
+            .saturating_add(2)
+            .saturating_div(3)
+            .saturating_mul(4);
+        let chunks = artifact_bytes
+            .saturating_add(chunk_bytes.saturating_sub(1))
+            .saturating_div(chunk_bytes.max(1))
+            .max(1);
+        encoded.saturating_mul(2).saturating_add(
+            chunks
+                .saturating_mul(2)
+                .saturating_mul(self.configuration.framing_overhead_bytes),
+        )
+    }
+
+    pub(super) fn framing_overhead_bytes(&self) -> u64 {
+        self.configuration.framing_overhead_bytes
+    }
+
+    pub(super) fn reserve(
+        &mut self,
+        key: String,
+        tenant: TenantId,
+        project: ProjectId,
+        account: UserId,
+        artifact_bytes: u64,
+        chunk_bytes: u64,
+        expires_at_epoch_seconds: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.expire(now_epoch_seconds);
+        self.prepare_period(now_epoch_seconds);
+        if !self.configuration.enabled {
+            return Err(ArtifactRelayError::Disabled);
+        }
+        if artifact_bytes > self.configuration.max_artifact_bytes {
+            return Err(ArtifactRelayError::ArtifactTooLarge);
+        }
+        let scope = RelayScope {
+            tenant,
+            project,
+            account,
+        };
+        self.check_concurrency(&scope)?;
+        self.check_tracked_scope_capacity(&scope)?;
+        let reserved = self.estimated_wire_bytes(artifact_bytes, chunk_bytes);
+        self.check_budget(&scope, reserved)?;
+        self.reservations.insert(
+            key,
+            RelayReservation {
+                scope,
+                remaining_reserved_bytes: reserved,
+                ingress_bytes: 0,
+                egress_bytes: 0,
+                expires_at_epoch_seconds,
+            },
+        );
+        Ok(())
+    }
+
+    pub(super) fn rekey(&mut self, from: &str, to: String) -> Result<(), ArtifactRelayError> {
+        let reservation = self
+            .reservations
+            .remove(from)
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        self.reservations.insert(to, reservation);
+        Ok(())
+    }
+
+    fn check_concurrency(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
+        if self.reservations.len() >= self.configuration.max_active_global {
+            return Err(ArtifactRelayError::Capacity);
+        }
+        let project = self
+            .reservations
+            .values()
+            .filter(|reservation| {
+                reservation.scope.tenant == scope.tenant
+                    && reservation.scope.project == scope.project
+            })
+            .count();
+        if project >= self.configuration.max_active_per_project {
+            return Err(ArtifactRelayError::Concurrency("project"));
+        }
+        let tenant = self
+            .reservations
+            .values()
+            .filter(|reservation| reservation.scope.tenant == scope.tenant)
+            .count();
+        if tenant >= self.configuration.max_active_per_tenant {
+            return Err(ArtifactRelayError::Concurrency("tenant"));
+        }
+        let account = self
+            .reservations
+            .values()
+            .filter(|reservation| {
+                reservation.scope.tenant == scope.tenant
+                    && reservation.scope.account == scope.account
+            })
+            .count();
+        if account >= self.configuration.max_active_per_account {
+            return Err(ArtifactRelayError::Concurrency("account"));
+        }
+        Ok(())
+    }
+
+    fn check_tracked_scope_capacity(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
+        let project_key = (scope.tenant.clone(), scope.project.clone());
+        let account_key = (scope.tenant.clone(), scope.account.clone());
+        let new_scopes = usize::from(!self.project_used.contains_key(&project_key))
+            + usize::from(!self.tenant_used.contains_key(&scope.tenant))
+            + usize::from(!self.account_used.contains_key(&account_key));
+        let tracked = self.project_used.len() + self.tenant_used.len() + self.account_used.len();
+        if tracked.saturating_add(new_scopes) > self.configuration.max_tracked_scopes {
+            return Err(ArtifactRelayError::StateLimit);
+        }
+        Ok(())
+    }
+
+    fn reserved_for(&self, scope: &RelayScope) -> (u64, u64, u64, u64) {
+        let mut project = 0_u64;
+        let mut tenant = 0_u64;
+        let mut account = 0_u64;
+        let mut global = 0_u64;
+        for reservation in self.reservations.values() {
+            let bytes = reservation.remaining_reserved_bytes;
+            global = global.saturating_add(bytes);
+            if reservation.scope.tenant == scope.tenant {
+                tenant = tenant.saturating_add(bytes);
+                if reservation.scope.project == scope.project {
+                    project = project.saturating_add(bytes);
+                }
+                if reservation.scope.account == scope.account {
+                    account = account.saturating_add(bytes);
+                }
+            }
+        }
+        (project, tenant, account, global)
+    }
+
+    fn check_budget(&self, scope: &RelayScope, additional: u64) -> Result<(), ArtifactRelayError> {
+        let (project_reserved, tenant_reserved, account_reserved, _) = self.reserved_for(scope);
+        let project_used = self
+            .project_used
+            .get(&(scope.tenant.clone(), scope.project.clone()))
+            .copied()
+            .unwrap_or(0);
+        let tenant_used = self.tenant_used.get(&scope.tenant).copied().unwrap_or(0);
+        let account_used = self
+            .account_used
+            .get(&(scope.tenant.clone(), scope.account.clone()))
+            .copied()
+            .unwrap_or(0);
+        for (used, reserved, limit, label) in [
+            (
+                project_used,
+                project_reserved,
+                self.configuration.max_period_bytes_per_project,
+                "project",
+            ),
+            (
+                tenant_used,
+                tenant_reserved,
+                self.configuration.max_period_bytes_per_tenant,
+                "tenant",
+            ),
+            (
+                account_used,
+                account_reserved,
+                self.configuration.max_period_bytes_per_account,
+                "account",
+            ),
+        ] {
+            if used.saturating_add(reserved).saturating_add(additional) > limit {
+                return Err(ArtifactRelayError::PeriodBudget(label));
+            }
+        }
+        Ok(())
+    }
+
+    fn charge(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        ingress: bool,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.prepare_period(now_epoch_seconds);
+        let reservation = self
+            .reservations
+            .get(key)
+            .cloned()
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        let additional = bytes.saturating_sub(reservation.remaining_reserved_bytes);
+        if additional > 0 {
+            self.check_budget(&reservation.scope, additional)?;
+        }
+        let reservation = self
+            .reservations
+            .get_mut(key)
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        reservation.remaining_reserved_bytes =
+            reservation.remaining_reserved_bytes.saturating_sub(bytes);
+        if ingress {
+            reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
+            self.ingress_used = self.ingress_used.saturating_add(bytes);
+        } else {
+            reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
+            self.egress_used = self.egress_used.saturating_add(bytes);
+        }
+        let project_key = (
+            reservation.scope.tenant.clone(),
+            reservation.scope.project.clone(),
+        );
+        let account_key = (
+            reservation.scope.tenant.clone(),
+            reservation.scope.account.clone(),
+        );
+        *self.project_used.entry(project_key).or_default() = self
+            .project_used
+            .get(&(
+                reservation.scope.tenant.clone(),
+                reservation.scope.project.clone(),
+            ))
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        *self
+            .tenant_used
+            .entry(reservation.scope.tenant.clone())
+            .or_default() = self
+            .tenant_used
+            .get(&reservation.scope.tenant)
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        *self.account_used.entry(account_key).or_default() = self
+            .account_used
+            .get(&(
+                reservation.scope.tenant.clone(),
+                reservation.scope.account.clone(),
+            ))
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        Ok(())
+    }
+
+    pub(super) fn charge_ingress(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.charge(key, bytes, true, now_epoch_seconds)
+    }
+
+    pub(super) fn charge_egress(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.charge(key, bytes, false, now_epoch_seconds)
+    }
+
+    pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
+        if let Some(reservation) = self.reservations.remove(key) {
+            if reason != RelayFinishReason::Completed {
+                self.abandoned_or_failed_used = self
+                    .abandoned_or_failed_used
+                    .saturating_add(reservation.ingress_bytes)
+                    .saturating_add(reservation.egress_bytes);
+            }
+        }
+    }
+
+    pub(super) fn expire(&mut self, now_epoch_seconds: u64) {
+        let expired = self
+            .reservations
+            .iter()
+            .filter(|(_, reservation)| reservation.expires_at_epoch_seconds < now_epoch_seconds)
+            .map(|(key, _)| key.clone())
+            .collect::<Vec<_>>();
+        for key in expired {
+            self.finish(&key, RelayFinishReason::Expired);
+        }
+    }
+
+    pub(super) fn usage(&self) -> ArtifactRelayUsage {
+        ArtifactRelayUsage {
+            active_transfers: self.reservations.len(),
+            ingress_bytes: self.ingress_used,
+            egress_bytes: self.egress_used,
+            abandoned_or_failed_bytes: self.abandoned_or_failed_used,
+            reserved_bytes: self
+                .reservations
+                .values()
+                .map(|reservation| reservation.remaining_reserved_bytes)
+                .fold(0_u64, u64::saturating_add),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/routing.rs

@@ -0,0 +1,831 @@
+use super::*;
+
+impl CoordinatorService {
+    pub fn handle_request(
+        &mut self,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.pump_main_runtime_commands();
+        let request_payload = serde_json::to_value(&request).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "failed to canonicalize coordinator request for authentication: {error}"
+            ))
+        })?;
+        let request_payload_digest =
+            clusterflux_core::signed_request_payload_digest(&request_payload);
+        match request {
+            CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong {
+                epoch: self.coordinator.coordinator_epoch(),
+            }),
+            CoordinatorRequest::Authenticated {
+                session_secret,
+                request,
+            } => self.handle_authenticated_request(session_secret, request),
+            CoordinatorRequest::AuthStatus {
+                tenant,
+                project,
+                actor_user,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(actor_user);
+                let account_state = self.coordinator.account_policy_state(&tenant);
+                Ok(CoordinatorResponse::AuthStatus {
+                    tenant,
+                    project,
+                    actor,
+                    authenticated: true,
+                    account_status: account_state.account_status,
+                    suspended: account_state.suspended,
+                    disabled: account_state.disabled,
+                    deleted: account_state.deleted,
+                    manual_review: account_state.manual_review,
+                    sanitized_reason: account_state.sanitized_reason,
+                    next_actions: account_state.next_actions,
+                    private_moderation_details_exposed: false,
+                    signup_failure_details_exposed: false,
+                })
+            }
+            CoordinatorRequest::AdminStatus {
+                tenant,
+                actor_user,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            } => self.handle_admin_status(
+                tenant,
+                actor_user,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            ),
+            CoordinatorRequest::SuspendTenant {
+                tenant,
+                actor_user,
+                target_tenant,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            } => self.handle_suspend_tenant(
+                tenant,
+                actor_user,
+                target_tenant,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            ),
+            CoordinatorRequest::CreateProject {
+                tenant,
+                actor_user,
+                project,
+                name,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let project = ProjectId::new(project);
+                self.coordinator.ensure_tenant_active(&tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the signed-in tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                if self.coordinator.project(&project).is_none() {
+                    self.quota.ensure_project_admission(
+                        &tenant,
+                        self.coordinator.project_count_for_tenant(&tenant),
+                    )?;
+                }
+                self.coordinator.upsert_tenant(tenant.clone());
+                self.coordinator.upsert_user(
+                    tenant.clone(),
+                    actor.clone(),
+                    CredentialKind::BrowserSession,
+                );
+                self.coordinator
+                    .upsert_project(tenant.clone(), project.clone(), name);
+                self.coordinator.grant_project_debug(
+                    tenant.clone(),
+                    project.clone(),
+                    actor.clone(),
+                );
+                self.persist_durable_state()?;
+                let project = self
+                    .coordinator
+                    .project(&project)
+                    .expect("project was just created")
+                    .clone();
+                Ok(CoordinatorResponse::ProjectCreated { project, actor })
+            }
+            CoordinatorRequest::SelectProject {
+                tenant,
+                actor_user,
+                project,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let project_id = ProjectId::new(project);
+                let project = self
+                    .coordinator
+                    .project(&project_id)
+                    .ok_or_else(|| {
+                        CoordinatorError::Unauthorized(
+                            "project is not visible to the signed-in user".to_owned(),
+                        )
+                    })?
+                    .clone();
+                if project.tenant != tenant {
+                    return Err(CoordinatorError::Unauthorized(
+                        "project is outside the signed-in tenant scope".to_owned(),
+                    )
+                    .into());
+                }
+                self.coordinator
+                    .upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::ProjectSelected { project, actor })
+            }
+            CoordinatorRequest::ListProjects { tenant, actor_user } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let context = clusterflux_core::AuthContext {
+                    tenant: tenant.clone(),
+                    project: ProjectId::from("__project_listing__"),
+                    actor: Actor::User(actor.clone()),
+                };
+                self.coordinator
+                    .upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::Projects {
+                    projects: self.coordinator.list_projects(&context),
+                    actor,
+                })
+            }
+            CoordinatorRequest::RegisterAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+                public_key,
+            }
+            | CoordinatorRequest::RotateAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+                public_key,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let agent = AgentId::new(agent);
+                self.coordinator.ensure_tenant_active(&tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the signed-in tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                self.coordinator.upsert_tenant(tenant.clone());
+                self.coordinator.upsert_user(
+                    tenant.clone(),
+                    actor.clone(),
+                    CredentialKind::CliDeviceSession,
+                );
+                self.coordinator
+                    .upsert_project(tenant.clone(), project.clone(), "local");
+                let record = self.coordinator.register_agent_public_key(
+                    tenant,
+                    project,
+                    actor.clone(),
+                    agent,
+                    public_key,
+                );
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::AgentPublicKey { record, actor })
+            }
+            CoordinatorRequest::ListAgentPublicKeys {
+                tenant,
+                project,
+                user,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let context = clusterflux_core::AuthContext {
+                    tenant,
+                    project,
+                    actor: Actor::User(actor.clone()),
+                };
+                Ok(CoordinatorResponse::AgentPublicKeys {
+                    records: self.coordinator.list_agent_public_keys(&context),
+                    actor,
+                })
+            }
+            CoordinatorRequest::RevokeAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let agent = AgentId::new(agent);
+                let context = clusterflux_core::AuthContext {
+                    tenant,
+                    project,
+                    actor: Actor::User(actor.clone()),
+                };
+                let record = self.coordinator.revoke_agent_public_key(&context, &agent)?;
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::AgentPublicKey { record, actor })
+            }
+            CoordinatorRequest::AttachNode {
+                tenant,
+                project,
+                node,
+                public_key,
+            } => self.handle_attach_node(tenant, project, node, public_key),
+            CoordinatorRequest::CreateNodeEnrollmentGrant {
+                tenant,
+                project,
+                actor_user,
+                ttl_seconds,
+            } => self.handle_create_node_enrollment_grant(tenant, project, actor_user, ttl_seconds),
+            CoordinatorRequest::ExchangeNodeEnrollmentGrant {
+                tenant,
+                project,
+                node,
+                public_key,
+                enrollment_grant,
+            } => self.handle_exchange_node_enrollment_grant(
+                tenant,
+                project,
+                node,
+                public_key,
+                enrollment_grant,
+            ),
+            CoordinatorRequest::NodeHeartbeat {
+                node,
+                node_signature,
+            } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
+            CoordinatorRequest::SignedNode {
+                node,
+                node_signature,
+                request,
+            } => self.handle_signed_node_request(node, node_signature, *request),
+            CoordinatorRequest::ReportNodeCapabilities { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::ListNodeDescriptors {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_list_node_descriptors(tenant, project, actor_user),
+            CoordinatorRequest::RevokeNodeCredential {
+                tenant,
+                project,
+                actor_user,
+                node,
+            } => self.handle_revoke_node_credential(tenant, project, actor_user, node),
+            CoordinatorRequest::ScheduleTask {
+                tenant,
+                project,
+                environment,
+                environment_digest,
+                required_capabilities,
+                dependency_cache,
+                source_snapshot,
+                required_artifacts,
+                prefer_node,
+            } => self.handle_schedule_task(
+                tenant,
+                project,
+                environment,
+                environment_digest,
+                required_capabilities,
+                dependency_cache,
+                source_snapshot,
+                required_artifacts,
+                prefer_node,
+            ),
+            CoordinatorRequest::LaunchTask {
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            } => self.handle_launch_task(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                Some(&request_payload_digest),
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            ),
+            CoordinatorRequest::LaunchChildTask { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::JoinChildTask { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::PollTaskAssignment { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::RequestRendezvous {
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_request_rendezvous(
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            ),
+            CoordinatorRequest::RequestSourcePreparation {
+                tenant,
+                project,
+                provider,
+            } => self.handle_request_source_preparation(tenant, project, provider),
+            CoordinatorRequest::CompleteSourcePreparation { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::StartProcess {
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                process,
+                restart,
+            } => self.handle_start_process(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                Some(&request_payload_digest),
+                process,
+                restart,
+            ),
+            CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::CancelTask {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_cancel_task(tenant, project, process, node, task),
+            CoordinatorRequest::CancelProcess {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_cancel_process(tenant, project, actor_user, process),
+            CoordinatorRequest::AbortProcess {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_abort_process(tenant, project, actor_user, process),
+            CoordinatorRequest::ListProcesses {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_list_processes(tenant, project, actor_user),
+            CoordinatorRequest::QuotaStatus {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_quota_status(tenant, project, actor_user),
+            CoordinatorRequest::PollTaskControl { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::PollArtifactTransfer { .. }
+            | CoordinatorRequest::UploadArtifactTransferChunk { .. }
+            | CoordinatorRequest::FailArtifactTransfer { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::RestartTask {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                replacement_bundle,
+            } => self.handle_restart_task(
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                replacement_bundle,
+            ),
+            CoordinatorRequest::ResolveTaskFailure {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                resolution,
+            } => self.handle_resolve_task_failure(
+                tenant, project, actor_user, process, task, resolution,
+            ),
+            request @ (CoordinatorRequest::DebugAttach { .. }
+            | CoordinatorRequest::SetDebugBreakpoints { .. }
+            | CoordinatorRequest::InspectDebugBreakpoints { .. }
+            | CoordinatorRequest::CreateDebugEpoch { .. }
+            | CoordinatorRequest::ResumeDebugEpoch { .. }
+            | CoordinatorRequest::InspectDebugEpoch { .. }) => self.handle_debug_request(request),
+            CoordinatorRequest::PollDebugCommand { .. }
+            | CoordinatorRequest::ReportDebugState { .. }
+            | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ListTaskEvents {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_list_task_events(tenant, project, actor_user, process),
+            CoordinatorRequest::ListTaskSnapshots {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_list_task_snapshots(tenant, project, actor_user, process),
+            CoordinatorRequest::JoinTask {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+            } => self.handle_join_task(tenant, project, actor_user, process, task),
+            CoordinatorRequest::RenderOperatorPanel {
+                tenant,
+                project,
+                process,
+                actor_user,
+                max_download_bytes,
+                stopped,
+            } => self.handle_render_operator_panel(
+                tenant,
+                project,
+                actor_user,
+                process,
+                max_download_bytes,
+                stopped,
+            ),
+            CoordinatorRequest::SubmitPanelEvent {
+                tenant,
+                project,
+                process,
+                widget_id,
+                kind,
+                max_events,
+            } => self
+                .handle_submit_panel_event(tenant, project, process, widget_id, kind, max_events),
+            CoordinatorRequest::CreateArtifactDownloadLink {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            } => self.handle_create_artifact_download_link(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            ),
+            CoordinatorRequest::OpenArtifactDownloadStream {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            } => self.handle_open_artifact_download_stream(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            ),
+            CoordinatorRequest::RevokeArtifactDownloadLink {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                token_digest,
+            } => self.handle_revoke_artifact_download_link(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                token_digest,
+            ),
+            CoordinatorRequest::ExportArtifactToNode {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                receiver_node,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_export_artifact_to_node(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                receiver_node,
+                direct_connectivity,
+                failure_reason,
+            ),
+        }
+    }
+
+    fn handle_authenticated_request(
+        &mut self,
+        session_secret: String,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = if matches!(&request, AuthenticatedCoordinatorRequest::AuthStatus) {
+            self.coordinator
+                .authenticate_cli_session_for_status(&session_secret)?
+        } else {
+            self.coordinator.authenticate_cli_session(&session_secret)?
+        };
+        let authorized = authorize_authenticated_user_operation(&context, &request)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .charge_api_call(&context.tenant, &context.project, now_epoch_seconds)?;
+        let _authorized_operation = authorized.operation;
+        let actor = authorized.actor;
+        match request {
+            AuthenticatedCoordinatorRequest::AuthStatus => {
+                let account_state = self.coordinator.account_policy_state(&context.tenant);
+                Ok(CoordinatorResponse::AuthStatus {
+                    tenant: context.tenant,
+                    project: context.project,
+                    actor,
+                    authenticated: true,
+                    account_status: account_state.account_status,
+                    suspended: account_state.suspended,
+                    disabled: account_state.disabled,
+                    deleted: account_state.deleted,
+                    manual_review: account_state.manual_review,
+                    sanitized_reason: account_state.sanitized_reason,
+                    next_actions: account_state.next_actions,
+                    private_moderation_details_exposed: false,
+                    signup_failure_details_exposed: false,
+                })
+            }
+            AuthenticatedCoordinatorRequest::RevokeCliSession => {
+                self.coordinator.revoke_cli_session(&session_secret)?;
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::CliSessionRevoked {
+                    tenant: context.tenant,
+                    project: context.project,
+                    actor,
+                })
+            }
+            AuthenticatedCoordinatorRequest::CreateProject { project, name } => {
+                let project = ProjectId::new(project);
+                self.coordinator.ensure_tenant_active(&context.tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != context.tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the authenticated tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                if self.coordinator.project(&project).is_none() {
+                    self.quota.ensure_project_admission(
+                        &context.tenant,
+                        self.coordinator.project_count_for_tenant(&context.tenant),
+                    )?;
+                }
+                self.coordinator
+                    .upsert_project(context.tenant.clone(), project.clone(), name);
+                self.coordinator.grant_project_debug(
+                    context.tenant.clone(),
+                    project.clone(),
+                    actor.clone(),
+                );
+                self.persist_durable_state()?;
+                let project = self
+                    .coordinator
+                    .project(&project)
+                    .expect("project was just created")
+                    .clone();
+                Ok(CoordinatorResponse::ProjectCreated { project, actor })
+            }
+            AuthenticatedCoordinatorRequest::SelectProject { project } => {
+                let project_id = ProjectId::new(project);
+                let project = self
+                    .coordinator
+                    .project(&project_id)
+                    .ok_or_else(|| {
+                        CoordinatorError::Unauthorized(
+                            "project is not visible to the authenticated user".to_owned(),
+                        )
+                    })?
+                    .clone();
+                if project.tenant != context.tenant {
+                    return Err(CoordinatorError::Unauthorized(
+                        "project is outside the authenticated tenant scope".to_owned(),
+                    )
+                    .into());
+                }
+                Ok(CoordinatorResponse::ProjectSelected { project, actor })
+            }
+            AuthenticatedCoordinatorRequest::ListProjects => Ok(CoordinatorResponse::Projects {
+                projects: self.coordinator.list_projects(&context),
+                actor,
+            }),
+            request @ (AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. }
+            | AuthenticatedCoordinatorRequest::ListAgentPublicKeys
+            | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. }
+            | AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. }) => {
+                self.handle_authenticated_agent_key_request(&context, &actor, request)
+            }
+            AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { ttl_seconds } => self
+                .handle_create_node_enrollment_grant(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    ttl_seconds,
+                ),
+            AuthenticatedCoordinatorRequest::ListNodeDescriptors => self
+                .handle_list_node_descriptors(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                ),
+            AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
+                .handle_revoke_node_credential(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    node,
+                ),
+            AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
+                .handle_start_process(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    Some(actor.as_str().to_owned()),
+                    None,
+                    None,
+                    None,
+                    None,
+                    process,
+                    restart,
+                ),
+            request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
+                self.handle_authenticated_schedule_task(&context, request)
+            }
+            request @ AuthenticatedCoordinatorRequest::LaunchTask { .. } => {
+                self.handle_authenticated_launch_task(&context, &actor, request)
+            }
+            AuthenticatedCoordinatorRequest::CancelProcess { process } => self
+                .handle_cancel_process(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+            ),
+            AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+            ),
+            AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+            ),
+            AuthenticatedCoordinatorRequest::RestartTask {
+                process,
+                task,
+                replacement_bundle,
+            } => self.handle_restart_task(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                replacement_bundle,
+            ),
+            AuthenticatedCoordinatorRequest::ResolveTaskFailure {
+                process,
+                task,
+                resolution,
+            } => self.handle_resolve_task_failure(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                resolution,
+            ),
+            request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. }
+            | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. }
+            | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. }
+            | AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. }
+            | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. }
+            | AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. }) => self
+                .handle_authenticated_debug_request(
+                    &context.tenant,
+                    &context.project,
+                    &actor,
+                    request,
+                ),
+            AuthenticatedCoordinatorRequest::ListTaskEvents { process } => self
+                .handle_list_task_events(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => self
+                .handle_list_task_snapshots(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+            ),
+            AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            } => self.handle_create_artifact_download_link(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            ),
+            AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            } => self.handle_open_artifact_download_stream(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            ),
+            AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink {
+                artifact,
+                token_digest,
+            } => self.handle_revoke_artifact_download_link(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                token_digest,
+            ),
+            request @ AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
+                self.handle_authenticated_artifact_export(&context, &actor, request)
+            }
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/signed_nodes.rs

@@ -0,0 +1,365 @@
+use clusterflux_core::{NodeId, NodeSignedRequest};
+
+use crate::CoordinatorError;
+
+use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+impl CoordinatorService {
+    pub(super) fn handle_signed_node_request(
+        &mut self,
+        signed_node: String,
+        node_signature: NodeSignedRequest,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let request_kind = signed_node_request_kind(&request)?;
+        let request_node = signed_node_request_node(&request)?;
+        let request_payload = serde_json::to_value(&request).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "failed to canonicalize signed node request: {error}"
+            ))
+        })?;
+        let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
+        let signed_node = NodeId::new(signed_node);
+        if request_node != signed_node {
+            return Err(CoordinatorError::Unauthorized(
+                "signed node request node does not match the wrapped request node".to_owned(),
+            )
+            .into());
+        }
+        self.authenticate_node_request(
+            &signed_node,
+            Some(node_signature),
+            request_kind,
+            &payload_digest,
+        )?;
+        match request {
+            CoordinatorRequest::ReportNodeCapabilities {
+                tenant,
+                project,
+                node,
+                capabilities,
+                cached_environment_digests,
+                dependency_cache_digests,
+                source_snapshots,
+                artifact_locations,
+                direct_connectivity,
+                online,
+            } => self.handle_report_node_capabilities(
+                tenant,
+                project,
+                node,
+                capabilities,
+                cached_environment_digests,
+                dependency_cache_digests,
+                source_snapshots,
+                artifact_locations,
+                direct_connectivity,
+                online,
+            ),
+            CoordinatorRequest::PollTaskAssignment {
+                tenant,
+                project,
+                node,
+            } => self.handle_poll_task_assignment(tenant, project, node),
+            CoordinatorRequest::PollArtifactTransfer {
+                tenant,
+                project,
+                node,
+            } => self.handle_poll_artifact_transfer(tenant, project, node),
+            CoordinatorRequest::UploadArtifactTransferChunk {
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                offset,
+                content_base64,
+                chunk_digest,
+                eof,
+            } => self.handle_upload_artifact_transfer_chunk(
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                offset,
+                content_base64,
+                chunk_digest,
+                eof,
+            ),
+            CoordinatorRequest::FailArtifactTransfer {
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                message,
+            } => self.handle_fail_artifact_transfer(
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                message,
+            ),
+            CoordinatorRequest::LaunchChildTask {
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            } => self.handle_launch_child_task(
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            ),
+            CoordinatorRequest::JoinChildTask {
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task,
+            } => self.handle_join_child_task(tenant, project, process, node, parent_task, task),
+            CoordinatorRequest::CompleteSourcePreparation {
+                tenant,
+                project,
+                node,
+                provider,
+                source_snapshot,
+            } => self.handle_complete_source_preparation(
+                tenant,
+                project,
+                node,
+                provider,
+                source_snapshot,
+            ),
+            CoordinatorRequest::ReconnectNode {
+                node,
+                process,
+                epoch,
+            } => self.handle_reconnect_node(node, process, epoch),
+            CoordinatorRequest::PollTaskControl {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_poll_task_control(tenant, project, process, node, task),
+            CoordinatorRequest::PollDebugCommand {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_poll_debug_command(tenant, project, process, node, task),
+            CoordinatorRequest::ReportDebugState {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                epoch,
+                state,
+                stack_frames,
+                local_values,
+                task_args,
+                handles,
+                command_status,
+                recent_output,
+                message,
+            } => self.handle_report_debug_state(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                epoch,
+                state,
+                stack_frames,
+                local_values,
+                task_args,
+                handles,
+                command_status,
+                recent_output,
+                message,
+            ),
+            CoordinatorRequest::ReportDebugProbeHit {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                probe_symbol,
+            } => self.handle_report_debug_probe_hit(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                probe_symbol,
+            ),
+            CoordinatorRequest::ReportTaskLog {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                backpressured,
+            } => self.handle_report_task_log(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                backpressured,
+            ),
+            CoordinatorRequest::ReportVfsMetadata {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                large_bytes_uploaded,
+            } => self.handle_report_vfs_metadata(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                large_bytes_uploaded,
+            ),
+            CoordinatorRequest::TaskCompleted {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                terminal_state,
+                status_code,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                result,
+            } => self.handle_task_completed(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                terminal_state,
+                status_code,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                result,
+            ),
+            _ => self.reject_unsigned_node_request(),
+        }
+    }
+
+    pub(super) fn reject_unsigned_node_request(
+        &self,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        Err(CoordinatorError::Unauthorized(
+            "node-originated request requires signed_node envelope proof".to_owned(),
+        )
+        .into())
+    }
+}
+
+fn signed_node_request_kind(
+    request: &CoordinatorRequest,
+) -> Result<&'static str, CoordinatorServiceError> {
+    match request {
+        CoordinatorRequest::ReportNodeCapabilities { .. } => Ok("report_node_capabilities"),
+        CoordinatorRequest::PollTaskAssignment { .. } => Ok("poll_task_assignment"),
+        CoordinatorRequest::PollArtifactTransfer { .. } => Ok("poll_artifact_transfer"),
+        CoordinatorRequest::UploadArtifactTransferChunk { .. } => {
+            Ok("upload_artifact_transfer_chunk")
+        }
+        CoordinatorRequest::FailArtifactTransfer { .. } => Ok("fail_artifact_transfer"),
+        CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
+        CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
+        CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
+        CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
+        CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
+        CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
+        CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
+        CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
+        CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
+        CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
+        CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
+        _ => Err(CoordinatorError::Unauthorized(
+            "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
+        )
+        .into()),
+    }
+}
+
+fn signed_node_request_node(
+    request: &CoordinatorRequest,
+) -> Result<NodeId, CoordinatorServiceError> {
+    match request {
+        CoordinatorRequest::ReportNodeCapabilities { node, .. }
+        | CoordinatorRequest::PollTaskAssignment { node, .. }
+        | CoordinatorRequest::PollArtifactTransfer { node, .. }
+        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
+        | CoordinatorRequest::FailArtifactTransfer { node, .. }
+        | CoordinatorRequest::LaunchChildTask { node, .. }
+        | CoordinatorRequest::JoinChildTask { node, .. }
+        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
+        | CoordinatorRequest::ReconnectNode { node, .. }
+        | CoordinatorRequest::PollTaskControl { node, .. }
+        | CoordinatorRequest::PollDebugCommand { node, .. }
+        | CoordinatorRequest::ReportDebugState { node, .. }
+        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
+        | CoordinatorRequest::ReportTaskLog { node, .. }
+        | CoordinatorRequest::ReportVfsMetadata { node, .. }
+        | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
+        _ => Err(CoordinatorError::Unauthorized(
+            "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
+        )
+        .into()),
+    }
+}

crates/clusterflux-coordinator/src/service/tcp.rs

@@ -0,0 +1,200 @@
+use std::io::{BufRead, BufReader, Write};
+use std::net::{SocketAddr, TcpListener, TcpStream};
+use std::sync::{Arc, Mutex};
+
+use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ClientAuthorityMode {
+    Strict,
+    LocalTrustedLoopback,
+}
+
+impl CoordinatorService {
+    pub fn serve_tcp(self, listener: TcpListener) -> Result<(), CoordinatorServiceError> {
+        if !listener.local_addr()?.ip().is_loopback() {
+            return Err(CoordinatorServiceError::Protocol(
+                "the native coordinator transport is plaintext and restricted to loopback; expose a remote coordinator only through a secure transport"
+                    .to_owned(),
+            ));
+        }
+        self.serve_tcp_with_authority(listener, ClientAuthorityMode::Strict)
+    }
+
+    pub fn serve_tcp_local_trusted(
+        self,
+        listener: TcpListener,
+    ) -> Result<(), CoordinatorServiceError> {
+        if !listener.local_addr()?.ip().is_loopback() {
+            return Err(CoordinatorServiceError::Protocol(
+                "local trusted request mode is restricted to a loopback listener".to_owned(),
+            ));
+        }
+        self.serve_tcp_with_authority(listener, ClientAuthorityMode::LocalTrustedLoopback)
+    }
+
+    fn serve_tcp_with_authority(
+        self,
+        listener: TcpListener,
+        authority_mode: ClientAuthorityMode,
+    ) -> Result<(), CoordinatorServiceError> {
+        let shared = Arc::new(Mutex::new(self));
+        for stream in listener.incoming() {
+            let stream = stream?;
+            let service = Arc::clone(&shared);
+            std::thread::spawn(move || {
+                if let Err(err) = handle_shared_stream(service, stream, authority_mode) {
+                    eprintln!("coordinator stream failed: {err}");
+                }
+            });
+        }
+        Ok(())
+    }
+
+    pub fn handle_stream(&mut self, stream: TcpStream) -> Result<(), CoordinatorServiceError> {
+        self.handle_stream_with_authority(stream, ClientAuthorityMode::Strict)
+    }
+
+    #[cfg(test)]
+    pub(super) fn handle_stream_local_trusted(
+        &mut self,
+        stream: TcpStream,
+    ) -> Result<(), CoordinatorServiceError> {
+        self.handle_stream_with_authority(stream, ClientAuthorityMode::LocalTrustedLoopback)
+    }
+
+    fn handle_stream_with_authority(
+        &mut self,
+        stream: TcpStream,
+        authority_mode: ClientAuthorityMode,
+    ) -> Result<(), CoordinatorServiceError> {
+        let mut reader = BufReader::new(stream.try_clone()?);
+        let mut writer = stream;
+        loop {
+            let mut line = String::new();
+            if reader.read_line(&mut line)? == 0 {
+                return Ok(());
+            }
+            if line.trim().is_empty() {
+                continue;
+            }
+            let response = match decode_wire_request(&line) {
+                Ok(request) => match authorize_client_request(&request, authority_mode)
+                    .and_then(|()| self.handle_request(request))
+                {
+                    Ok(response) => response,
+                    Err(err) => CoordinatorResponse::Error {
+                        message: err.to_string(),
+                    },
+                },
+                Err(err) => CoordinatorResponse::Error {
+                    message: err.to_string(),
+                },
+            };
+            serde_json::to_writer(&mut writer, &response)?;
+            writer.write_all(b"\n")?;
+            writer.flush()?;
+        }
+    }
+}
+
+fn handle_shared_stream(
+    service: Arc<Mutex<CoordinatorService>>,
+    stream: TcpStream,
+    authority_mode: ClientAuthorityMode,
+) -> Result<(), CoordinatorServiceError> {
+    let mut reader = BufReader::new(stream.try_clone()?);
+    let mut writer = stream;
+    loop {
+        let mut line = String::new();
+        if reader.read_line(&mut line)? == 0 {
+            return Ok(());
+        }
+        if line.trim().is_empty() {
+            continue;
+        }
+        let response = match decode_wire_request(&line) {
+            Ok(request) => match authorize_client_request(&request, authority_mode) {
+                Ok(()) => match service.lock() {
+                    Ok(mut service) => match service.handle_request(request) {
+                        Ok(response) => response,
+                        Err(err) => CoordinatorResponse::Error {
+                            message: err.to_string(),
+                        },
+                    },
+                    Err(_) => CoordinatorResponse::Error {
+                        message: "coordinator service lock poisoned".to_owned(),
+                    },
+                },
+                Err(err) => CoordinatorResponse::Error {
+                    message: err.to_string(),
+                },
+            },
+            Err(err) => CoordinatorResponse::Error {
+                message: err.to_string(),
+            },
+        };
+        serde_json::to_writer(&mut writer, &response)?;
+        writer.write_all(b"\n")?;
+        writer.flush()?;
+    }
+}
+
+pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), CoordinatorServiceError> {
+    let listener = TcpListener::bind(addr)?;
+    let addr = listener.local_addr()?;
+    Ok((listener, addr))
+}
+
+fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
+    serde_json::from_str::<super::CoordinatorWireRequest>(line)?
+        .into_request()
+        .map_err(CoordinatorServiceError::Protocol)
+}
+
+fn authorize_client_request(
+    request: &CoordinatorRequest,
+    authority_mode: ClientAuthorityMode,
+) -> Result<(), CoordinatorServiceError> {
+    if authority_mode == ClientAuthorityMode::LocalTrustedLoopback {
+        return Ok(());
+    }
+    match request {
+        CoordinatorRequest::Ping
+        | CoordinatorRequest::Authenticated { .. }
+        | CoordinatorRequest::ExchangeNodeEnrollmentGrant { .. }
+        | CoordinatorRequest::SignedNode { .. }
+        | CoordinatorRequest::NodeHeartbeat {
+            node_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::StartProcess {
+            actor_agent: Some(_),
+            agent_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::LaunchTask {
+            actor_agent: Some(_),
+            agent_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::AdminStatus { .. }
+        | CoordinatorRequest::SuspendTenant { .. } => Ok(()),
+        _ => Err(CoordinatorServiceError::Protocol(
+            "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
+                .to_owned(),
+        )),
+    }
+}
+
+#[cfg(test)]
+mod transport_boundary_tests {
+    use super::*;
+
+    #[test]
+    fn native_plaintext_service_refuses_non_loopback_listener() {
+        let (listener, _) = bind_listener("0.0.0.0:0").unwrap();
+        let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
+        assert!(error.to_string().contains("restricted to loopback"));
+    }
+}

crates/clusterflux-coordinator/src/service/wire_protocol.rs

@@ -0,0 +1,59 @@
+use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use super::CoordinatorRequest;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum CoordinatorWireRequest {
+    Envelope(CoordinatorRequestEnvelope),
+}
+
+impl CoordinatorWireRequest {
+    pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        match self {
+            Self::Envelope(envelope) => envelope.into_request(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct CoordinatorRequestEnvelope {
+    #[serde(rename = "type")]
+    pub envelope_type: String,
+    pub protocol_version: u64,
+    pub request_id: String,
+    pub operation: String,
+    #[serde(default)]
+    pub authentication: Option<Value>,
+    pub payload: CoordinatorRequest,
+}
+
+impl CoordinatorRequestEnvelope {
+    pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
+            return Err(format!(
+                "unsupported coordinator wire request type {}; expected {}",
+                self.envelope_type, COORDINATOR_WIRE_REQUEST_TYPE
+            ));
+        }
+        if self.protocol_version != COORDINATOR_PROTOCOL_VERSION {
+            return Err(format!(
+                "unsupported coordinator protocol version {}; expected {}",
+                self.protocol_version, COORDINATOR_PROTOCOL_VERSION
+            ));
+        }
+        if self.request_id.trim().is_empty() {
+            return Err("coordinator wire request_id must be non-empty".to_owned());
+        }
+        let payload_operation = self.payload.operation()?;
+        if self.operation != payload_operation {
+            return Err(format!(
+                "coordinator wire operation {} does not match payload operation {}",
+                self.operation, payload_operation
+            ));
+        }
+        Ok(self.payload)
+    }
+}

crates/clusterflux-coordinator/src/sessions.rs

@@ -0,0 +1,194 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{Actor, AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId};
+
+use crate::{CliSessionRecord, Coordinator, CoordinatorError, CredentialRecord};
+
+impl Coordinator {
+    pub fn issue_cli_session(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        user: UserId,
+        session_secret: &str,
+        expires_at_epoch_seconds: Option<u64>,
+    ) -> CliSessionRecord {
+        self.upsert_tenant(tenant.clone());
+        self.upsert_user(
+            tenant.clone(),
+            user.clone(),
+            CredentialKind::CliDeviceSession,
+        );
+        let session_digest = Digest::sha256(session_secret);
+        let record = CliSessionRecord {
+            session_digest: session_digest.clone(),
+            tenant: tenant.clone(),
+            project: project.clone(),
+            user: user.clone(),
+            credential_kind: CredentialKind::CliDeviceSession,
+            expires_at_epoch_seconds,
+            revoked: false,
+        };
+        self.durable
+            .cli_sessions
+            .insert(session_digest.clone(), record.clone());
+        let credential_subject = format!("cli-session:{}", session_digest.as_str());
+        self.durable.credentials.insert(
+            credential_subject.clone(),
+            CredentialRecord {
+                subject: credential_subject,
+                tenant,
+                project: Some(project),
+                kind: CredentialKind::CliDeviceSession,
+                public_key_fingerprint: None,
+            },
+        );
+        record
+    }
+
+    pub fn authenticate_cli_session(
+        &self,
+        session_secret: &str,
+    ) -> Result<AuthContext, CoordinatorError> {
+        self.authenticate_cli_session_at_with_tenant_policy(
+            session_secret,
+            unix_timestamp_seconds(),
+            true,
+        )
+    }
+
+    pub fn authenticate_cli_session_for_status(
+        &self,
+        session_secret: &str,
+    ) -> Result<AuthContext, CoordinatorError> {
+        self.authenticate_cli_session_at_with_tenant_policy(
+            session_secret,
+            unix_timestamp_seconds(),
+            false,
+        )
+    }
+
+    pub fn authenticate_cli_session_at(
+        &self,
+        session_secret: &str,
+        now_epoch_seconds: u64,
+    ) -> Result<AuthContext, CoordinatorError> {
+        self.authenticate_cli_session_at_with_tenant_policy(session_secret, now_epoch_seconds, true)
+    }
+
+    fn authenticate_cli_session_at_with_tenant_policy(
+        &self,
+        session_secret: &str,
+        now_epoch_seconds: u64,
+        require_active_tenant: bool,
+    ) -> Result<AuthContext, CoordinatorError> {
+        if session_secret.trim().is_empty() {
+            return Err(CoordinatorError::Unauthorized(
+                "CLI session credential is missing".to_owned(),
+            ));
+        }
+        let session_digest = Digest::sha256(session_secret);
+        let record = self
+            .durable
+            .cli_sessions
+            .get(&session_digest)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "CLI session credential is not recognized".to_owned(),
+                )
+            })?;
+        if record.revoked {
+            return Err(CoordinatorError::Unauthorized(
+                "CLI session credential has been revoked".to_owned(),
+            ));
+        }
+        if record
+            .expires_at_epoch_seconds
+            .is_some_and(|expires_at| expires_at <= now_epoch_seconds)
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "CLI session credential has expired; run clusterflux login --browser again"
+                    .to_owned(),
+            ));
+        }
+        if record.credential_kind != CredentialKind::CliDeviceSession {
+            return Err(CoordinatorError::Unauthorized(
+                "credential is not a CLI session".to_owned(),
+            ));
+        }
+        if require_active_tenant {
+            self.ensure_tenant_active(&record.tenant)?;
+        }
+        Ok(AuthContext {
+            tenant: record.tenant.clone(),
+            project: record.project.clone(),
+            actor: Actor::User(record.user.clone()),
+        })
+    }
+
+    pub fn revoke_cli_session(
+        &mut self,
+        session_secret: &str,
+    ) -> Result<CliSessionRecord, CoordinatorError> {
+        self.authenticate_cli_session(session_secret)?;
+        let session_digest = Digest::sha256(session_secret);
+        let record = self
+            .durable
+            .cli_sessions
+            .get_mut(&session_digest)
+            .expect("authenticated session must exist");
+        record.revoked = true;
+        Ok(record.clone())
+    }
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use crate::InMemoryDurableStore;
+
+    use super::*;
+
+    #[test]
+    fn repeated_logins_for_one_user_create_distinct_session_credentials() {
+        let store = InMemoryDurableStore::default();
+        let mut coordinator = Coordinator::boot(&store, 1);
+        let tenant = TenantId::from("tenant");
+        let user = UserId::from("user");
+
+        coordinator.issue_cli_session(
+            tenant.clone(),
+            ProjectId::from("project-one"),
+            user.clone(),
+            "session-one",
+            None,
+        );
+        coordinator.issue_cli_session(
+            tenant,
+            ProjectId::from("project-two"),
+            user,
+            "session-two",
+            None,
+        );
+
+        let subjects = coordinator
+            .durable
+            .credentials
+            .values()
+            .map(|credential| credential.subject.clone())
+            .collect::<BTreeSet<_>>();
+        assert_eq!(coordinator.durable.cli_sessions.len(), 2);
+        assert_eq!(subjects.len(), 2);
+        assert!(subjects
+            .iter()
+            .all(|subject| subject.starts_with("cli-session:sha256:")));
+    }
+}

crates/clusterflux-core/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "clusterflux-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+ed25519-dalek.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+hex.workspace = true
+sha2.workspace = true
+syn.workspace = true
+thiserror.workspace = true
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+getrandom.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true

crates/clusterflux-core/src/auth.rs

@@ -0,0 +1,829 @@
+#[cfg(not(target_arch = "wasm32"))]
+use base64::engine::general_purpose::URL_SAFE_NO_PAD;
+use base64::{engine::general_purpose::STANDARD, Engine as _};
+use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::{
+    AgentId, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId,
+};
+
+pub fn admin_request_proof(
+    admin_token: &str,
+    operation: &str,
+    tenant: &str,
+    actor_user: &str,
+    target_tenant: &str,
+    nonce: &str,
+    issued_at_epoch_seconds: u64,
+) -> Digest {
+    admin_request_proof_from_token_digest(
+        &Digest::sha256(admin_token),
+        operation,
+        tenant,
+        actor_user,
+        target_tenant,
+        nonce,
+        issued_at_epoch_seconds,
+    )
+}
+
+pub fn admin_request_proof_from_token_digest(
+    admin_token_digest: &Digest,
+    operation: &str,
+    tenant: &str,
+    actor_user: &str,
+    target_tenant: &str,
+    nonce: &str,
+    issued_at_epoch_seconds: u64,
+) -> Digest {
+    let issued_at_epoch_seconds = issued_at_epoch_seconds.to_string();
+    Digest::from_parts([
+        b"clusterflux-admin-request-proof:v1".as_slice(),
+        admin_token_digest.as_str().as_bytes(),
+        operation.as_bytes(),
+        tenant.as_bytes(),
+        actor_user.as_bytes(),
+        target_tenant.as_bytes(),
+        nonce.as_bytes(),
+        issued_at_epoch_seconds.as_bytes(),
+    ])
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Actor {
+    User(UserId),
+    Agent(AgentId),
+    Node(NodeId),
+    Task(TaskInstanceId),
+}
+
+impl Actor {
+    pub fn kind(&self) -> IdentityKind {
+        match self {
+            Self::User(_) => IdentityKind::User,
+            Self::Agent(_) => IdentityKind::Agent,
+            Self::Node(_) => IdentityKind::Node,
+            Self::Task(_) => IdentityKind::Task,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum IdentityKind {
+    User,
+    Agent,
+    Node,
+    Project,
+    Task,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum CredentialKind {
+    BrowserSession,
+    CliDeviceSession,
+    PublicKey,
+    NodeCredential,
+    TaskCredential,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AuthContext {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub actor: Actor,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Action {
+    CreateProject,
+    AttachNode,
+    CreateNodeEnrollmentGrant,
+    ExchangeNodeEnrollmentGrant,
+    LoginBrowser,
+    LoginCli,
+    EnrollAgent,
+    List,
+    Inspect,
+    Mutate,
+    ClaimTask,
+    DebugAttach,
+    DebugRead,
+    DownloadArtifact,
+    PublishArtifact,
+    RunNativeCommand,
+    RunContainer,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BrowserLoginFlow {
+    pub authorization_url: String,
+    pub callback_path: String,
+    pub state: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PublicKeyIdentity {
+    pub subject: Actor,
+    pub public_key: String,
+    pub fingerprint: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AgentSignedRequest {
+    pub nonce: String,
+    pub issued_at_epoch_seconds: u64,
+    pub signature: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeSignedRequest {
+    pub nonce: String,
+    pub issued_at_epoch_seconds: u64,
+    pub signature: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EnrollmentGrant {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub grant_id: String,
+    pub scope: String,
+    pub expires_at_epoch_seconds: u64,
+    pub consumed: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeCredential {
+    pub node: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub public_key_fingerprint: Digest,
+    pub scope: String,
+    pub capability_policy_digest: Digest,
+    pub credential_kind: CredentialKind,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum EnrollmentError {
+    Expired,
+    AlreadyConsumed,
+    WrongScope,
+}
+
+impl EnrollmentGrant {
+    pub fn exchange_for_node_identity(
+        &mut self,
+        node: NodeId,
+        public_key: &str,
+        requested_scope: &str,
+        now_epoch_seconds: u64,
+    ) -> Result<NodeCredential, EnrollmentError> {
+        if self.consumed {
+            return Err(EnrollmentError::AlreadyConsumed);
+        }
+        if now_epoch_seconds > self.expires_at_epoch_seconds {
+            return Err(EnrollmentError::Expired);
+        }
+        if requested_scope != self.scope {
+            return Err(EnrollmentError::WrongScope);
+        }
+        self.consumed = true;
+        let capability_policy_digest =
+            node_capability_policy_digest(&self.tenant, &self.project, &self.scope);
+        Ok(NodeCredential {
+            node,
+            tenant: self.tenant.clone(),
+            project: self.project.clone(),
+            public_key_fingerprint: Digest::sha256(public_key),
+            scope: self.scope.clone(),
+            capability_policy_digest,
+            credential_kind: CredentialKind::NodeCredential,
+        })
+    }
+}
+
+pub fn node_capability_policy_digest(
+    tenant: &TenantId,
+    project: &ProjectId,
+    scope: &str,
+) -> Digest {
+    Digest::from_parts([
+        b"node-capability-policy:v1".as_slice(),
+        tenant.as_str().as_bytes(),
+        project.as_str().as_bytes(),
+        scope.as_bytes(),
+    ])
+}
+
+pub fn agent_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
+    let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
+    let private_key: [u8; 32] = private_key
+        .try_into()
+        .map_err(|_| "agent private key must be 32 bytes".to_owned())?;
+    let signing_key = SigningKey::from_bytes(&private_key);
+    Ok(format!(
+        "ed25519:{}",
+        STANDARD.encode(signing_key.verifying_key().to_bytes())
+    ))
+}
+
+pub fn node_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
+    agent_ed25519_public_key_from_private_key(private_key)
+}
+
+pub fn derive_ed25519_private_key_from_seed(seed: &str) -> String {
+    let digest = Digest::sha256(seed);
+    let hex = digest.as_str().trim_start_matches("sha256:");
+    let bytes = hex::decode(hex).expect("sha256 digest hex should decode");
+    format!("ed25519:{}", STANDARD.encode(bytes))
+}
+
+/// Generates a new Ed25519 private key from the operating system CSPRNG.
+///
+/// Seed-derived keys remain available for deterministic test fixtures, but
+/// must not be used for persisted user, Agent, or Node credentials.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn generate_ed25519_private_key() -> Result<String, String> {
+    let mut bytes = [0_u8; 32];
+    getrandom::fill(&mut bytes)
+        .map_err(|err| format!("operating system random source failed: {err}"))?;
+    Ok(format!("ed25519:{}", STANDARD.encode(bytes)))
+}
+
+/// Generates an opaque, URL-safe 256-bit token suitable for one-time grants,
+/// session secrets, and nonces. The label is non-secret domain separation.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn generate_opaque_token(label: &str) -> Result<String, String> {
+    let mut bytes = [0_u8; 32];
+    getrandom::fill(&mut bytes)
+        .map_err(|err| format!("operating system random source failed: {err}"))?;
+    Ok(format!("{label}_{}", URL_SAFE_NO_PAD.encode(bytes)))
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct AgentWorkflowScope<'a> {
+    pub tenant: &'a TenantId,
+    pub project: &'a ProjectId,
+    pub agent: &'a AgentId,
+    pub request_kind: &'a str,
+    pub process: &'a ProcessId,
+    pub task: Option<&'a TaskInstanceId>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct AgentWorkflowRequestScope {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub request_kind: String,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+}
+
+impl AgentWorkflowRequestScope {
+    pub fn new(
+        tenant: TenantId,
+        project: ProjectId,
+        request_kind: impl Into<String>,
+        process: ProcessId,
+        task: Option<TaskInstanceId>,
+    ) -> Result<Self, String> {
+        let request_kind = request_kind.into();
+        match request_kind.as_str() {
+            "start_process" if task.is_some() => {
+                return Err("start_process agent scope must not contain a task instance".to_owned())
+            }
+            "launch_task" if task.is_none() => {
+                return Err("launch_task agent scope requires a task instance".to_owned())
+            }
+            "start_process" | "launch_task" => {}
+            _ => {
+                return Err(format!(
+                    "request kind `{request_kind}` is not an agent workflow operation"
+                ))
+            }
+        }
+        Ok(Self {
+            tenant,
+            project,
+            request_kind,
+            process,
+            task,
+        })
+    }
+
+    pub fn for_agent<'a>(&'a self, agent: &'a AgentId) -> AgentWorkflowScope<'a> {
+        AgentWorkflowScope {
+            tenant: &self.tenant,
+            project: &self.project,
+            agent,
+            request_kind: &self.request_kind,
+            process: &self.process,
+            task: self.task.as_ref(),
+        }
+    }
+}
+
+pub fn agent_workflow_request_scope_from_payload(
+    payload: &Value,
+) -> Result<AgentWorkflowRequestScope, String> {
+    let object = payload
+        .as_object()
+        .ok_or_else(|| "agent workflow request payload must be an object".to_owned())?;
+    let request_kind = object
+        .get("type")
+        .and_then(Value::as_str)
+        .ok_or_else(|| "agent workflow request type is missing".to_owned())?;
+    let tenant = object
+        .get("tenant")
+        .and_then(Value::as_str)
+        .ok_or_else(|| "agent workflow tenant is missing".to_owned())?;
+    let project = object
+        .get("project")
+        .and_then(Value::as_str)
+        .ok_or_else(|| "agent workflow project is missing".to_owned())?;
+    let (process, task) = match request_kind {
+        "start_process" => (
+            object
+                .get("process")
+                .and_then(Value::as_str)
+                .ok_or_else(|| "start_process agent scope is missing process".to_owned())?,
+            None,
+        ),
+        "launch_task" => {
+            let task_spec = object
+                .get("task_spec")
+                .and_then(Value::as_object)
+                .ok_or_else(|| "launch_task agent scope is missing task_spec".to_owned())?;
+            let process = task_spec
+                .get("process")
+                .and_then(Value::as_str)
+                .ok_or_else(|| "launch_task agent scope is missing process".to_owned())?;
+            let task = task_spec
+                .get("task_instance")
+                .and_then(Value::as_str)
+                .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
+            (process, Some(TaskInstanceId::from(task)))
+        }
+        _ => {
+            return Err(format!(
+                "request kind `{request_kind}` is not an agent workflow operation"
+            ))
+        }
+    };
+    AgentWorkflowRequestScope::new(
+        TenantId::from(tenant),
+        ProjectId::from(project),
+        request_kind,
+        ProcessId::from(process),
+        task,
+    )
+}
+
+pub fn sign_agent_workflow_request(
+    private_key: &str,
+    scope: AgentWorkflowScope<'_>,
+    payload_digest: &Digest,
+    nonce: String,
+    issued_at_epoch_seconds: u64,
+) -> Result<AgentSignedRequest, String> {
+    let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
+    let private_key: [u8; 32] = private_key
+        .try_into()
+        .map_err(|_| "agent private key must be 32 bytes".to_owned())?;
+    let signing_key = SigningKey::from_bytes(&private_key);
+    let message =
+        agent_workflow_signature_message(scope, payload_digest, &nonce, issued_at_epoch_seconds);
+    let signature: Signature = signing_key.sign(&message);
+    Ok(AgentSignedRequest {
+        nonce,
+        issued_at_epoch_seconds,
+        signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
+    })
+}
+
+pub fn verify_agent_workflow_signature(
+    public_key: &str,
+    scope: AgentWorkflowScope<'_>,
+    payload_digest: &Digest,
+    signed_request: &AgentSignedRequest,
+) -> Result<(), String> {
+    let public_key = decode_ed25519_key(public_key, 32, "agent public key")?;
+    let public_key: [u8; 32] = public_key
+        .try_into()
+        .map_err(|_| "agent public key must be 32 bytes".to_owned())?;
+    let verifying_key = VerifyingKey::from_bytes(&public_key)
+        .map_err(|_| "agent public key is not a valid Ed25519 verifying key".to_owned())?;
+    let signature = decode_ed25519_key(&signed_request.signature, 64, "agent signature")?;
+    let signature: [u8; 64] = signature
+        .try_into()
+        .map_err(|_| "agent signature must be 64 bytes".to_owned())?;
+    let signature = Signature::from_bytes(&signature);
+    let message = agent_workflow_signature_message(
+        scope,
+        payload_digest,
+        &signed_request.nonce,
+        signed_request.issued_at_epoch_seconds,
+    );
+    verifying_key
+        .verify(&message, &signature)
+        .map_err(|_| "agent signature does not verify against the registered public key".to_owned())
+}
+
+pub fn sign_node_request(
+    private_key: &str,
+    node: &NodeId,
+    request_kind: &str,
+    payload_digest: &Digest,
+    nonce: String,
+    issued_at_epoch_seconds: u64,
+) -> Result<NodeSignedRequest, String> {
+    let private_key = decode_ed25519_key(private_key, 32, "node private key")?;
+    let private_key: [u8; 32] = private_key
+        .try_into()
+        .map_err(|_| "node private key must be 32 bytes".to_owned())?;
+    let signing_key = SigningKey::from_bytes(&private_key);
+    let message = node_request_signature_message(
+        node,
+        request_kind,
+        payload_digest,
+        &nonce,
+        issued_at_epoch_seconds,
+    );
+    let signature: Signature = signing_key.sign(&message);
+    Ok(NodeSignedRequest {
+        nonce,
+        issued_at_epoch_seconds,
+        signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
+    })
+}
+
+pub fn verify_node_request_signature(
+    public_key: &str,
+    node: &NodeId,
+    request_kind: &str,
+    payload_digest: &Digest,
+    signed_request: &NodeSignedRequest,
+) -> Result<(), String> {
+    let public_key = decode_ed25519_key(public_key, 32, "node public key")?;
+    let public_key: [u8; 32] = public_key
+        .try_into()
+        .map_err(|_| "node public key must be 32 bytes".to_owned())?;
+    let verifying_key = VerifyingKey::from_bytes(&public_key)
+        .map_err(|_| "node public key is not a valid Ed25519 verifying key".to_owned())?;
+    let signature = decode_ed25519_key(&signed_request.signature, 64, "node signature")?;
+    let signature: [u8; 64] = signature
+        .try_into()
+        .map_err(|_| "node signature must be 64 bytes".to_owned())?;
+    let signature = Signature::from_bytes(&signature);
+    let message = node_request_signature_message(
+        node,
+        request_kind,
+        payload_digest,
+        &signed_request.nonce,
+        signed_request.issued_at_epoch_seconds,
+    );
+    verifying_key
+        .verify(&message, &signature)
+        .map_err(|_| "node signature does not verify against the enrolled public key".to_owned())
+}
+
+fn decode_ed25519_key(value: &str, expected_len: usize, kind: &str) -> Result<Vec<u8>, String> {
+    let encoded = value
+        .strip_prefix("ed25519:")
+        .ok_or_else(|| format!("{kind} must use ed25519:<base64> encoding"))?;
+    let bytes = STANDARD
+        .decode(encoded)
+        .map_err(|_| format!("{kind} is not valid base64"))?;
+    if bytes.len() != expected_len {
+        return Err(format!("{kind} must be {expected_len} bytes"));
+    }
+    Ok(bytes)
+}
+
+fn agent_workflow_signature_message(
+    scope: AgentWorkflowScope<'_>,
+    payload_digest: &Digest,
+    nonce: &str,
+    issued_at_epoch_seconds: u64,
+) -> Vec<u8> {
+    let issued_at = issued_at_epoch_seconds.to_string();
+    let task = scope.task.map(TaskInstanceId::as_str).unwrap_or("");
+    let parts = [
+        "clusterflux-agent-workflow-signature:v2",
+        scope.tenant.as_str(),
+        scope.project.as_str(),
+        scope.agent.as_str(),
+        scope.request_kind,
+        scope.process.as_str(),
+        task,
+        payload_digest.as_str(),
+        nonce,
+        &issued_at,
+    ];
+    let mut message = Vec::new();
+    for part in parts {
+        message.extend_from_slice(part.len().to_string().as_bytes());
+        message.push(b':');
+        message.extend_from_slice(part.as_bytes());
+        message.push(b'\n');
+    }
+    message
+}
+
+fn node_request_signature_message(
+    node: &NodeId,
+    request_kind: &str,
+    payload_digest: &Digest,
+    nonce: &str,
+    issued_at_epoch_seconds: u64,
+) -> Vec<u8> {
+    let issued_at = issued_at_epoch_seconds.to_string();
+    let parts = [
+        "clusterflux-node-request-signature:v2",
+        node.as_str(),
+        request_kind,
+        payload_digest.as_str(),
+        nonce,
+        &issued_at,
+    ];
+    let mut message = Vec::new();
+    for part in parts {
+        message.extend_from_slice(part.len().to_string().as_bytes());
+        message.push(b':');
+        message.extend_from_slice(part.as_bytes());
+        message.push(b'\n');
+    }
+    message
+}
+
+/// Computes the stable digest covered by node and agent request signatures.
+///
+/// The proof field itself is excluded, and explicit JSON nulls are normalized
+/// with omitted optional fields because the wire protocol deserializes both to
+/// the same request. Every semantically meaningful key and value remains bound
+/// by the signature.
+pub fn signed_request_payload_digest(value: &Value) -> Digest {
+    fn canonicalize(value: &Value, top_level: bool) -> Value {
+        match value {
+            Value::Object(object) => Value::Object(
+                object
+                    .iter()
+                    .filter(|(key, value)| {
+                        !value.is_null()
+                            && (!top_level
+                                || !matches!(key.as_str(), "agent_signature" | "node_signature"))
+                    })
+                    .map(|(key, value)| (key.clone(), canonicalize(value, false)))
+                    .collect(),
+            ),
+            Value::Array(values) => Value::Array(
+                values
+                    .iter()
+                    .map(|value| canonicalize(value, false))
+                    .collect(),
+            ),
+            value => value.clone(),
+        }
+    }
+
+    let canonical = canonicalize(value, true);
+    let bytes = serde_json::to_vec(&canonical)
+        .expect("canonical JSON request values are always serializable");
+    Digest::sha256(bytes)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Scope {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: Option<ProcessId>,
+    pub task: Option<TaskInstanceId>,
+    pub node: Option<NodeId>,
+    pub artifact: Option<ArtifactId>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Authorization {
+    pub allowed: bool,
+    pub reason: String,
+}
+
+impl Authorization {
+    pub fn allow(reason: impl Into<String>) -> Self {
+        Self {
+            allowed: true,
+            reason: reason.into(),
+        }
+    }
+
+    pub fn deny(reason: impl Into<String>) -> Self {
+        Self {
+            allowed: false,
+            reason: reason.into(),
+        }
+    }
+}
+
+pub fn same_tenant_project(context: &AuthContext, scope: &Scope) -> Authorization {
+    if context.tenant != scope.tenant {
+        return Authorization::deny("tenant mismatch");
+    }
+    if context.project != scope.project {
+        return Authorization::deny("project mismatch");
+    }
+    Authorization::allow("same tenant and project")
+}
+
+pub fn task_credentials_do_not_contain_user_session(
+    task: &Actor,
+    credentials: &[CredentialKind],
+) -> Authorization {
+    if !matches!(task, Actor::Task(_)) {
+        return Authorization::deny("credential check requires task actor");
+    }
+    if credentials.iter().any(|credential| {
+        matches!(
+            credential,
+            CredentialKind::BrowserSession | CredentialKind::CliDeviceSession
+        )
+    }) {
+        return Authorization::deny(
+            "user OAuth/session tokens must not be passed to nodes as task credentials",
+        );
+    }
+    Authorization::allow("task credentials are scoped runtime credentials")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tenant_project_scope_denies_cross_tenant_access() {
+        let context = AuthContext {
+            tenant: TenantId::from("tenant-a"),
+            project: ProjectId::from("project-a"),
+            actor: Actor::User(UserId::from("user-a")),
+        };
+        let scope = Scope {
+            tenant: TenantId::from("tenant-b"),
+            project: ProjectId::from("project-a"),
+            process: None,
+            task: None,
+            node: None,
+            artifact: None,
+        };
+
+        assert!(!same_tenant_project(&context, &scope).allowed);
+    }
+
+    #[test]
+    fn node_enrollment_exchanges_short_lived_grant_once() {
+        let mut grant = EnrollmentGrant {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            grant_id: "grant".to_owned(),
+            scope: "node:attach".to_owned(),
+            expires_at_epoch_seconds: 100,
+            consumed: false,
+        };
+
+        let credential = grant
+            .exchange_for_node_identity(NodeId::from("node"), "public-key", "node:attach", 99)
+            .unwrap();
+
+        assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
+        assert_eq!(credential.tenant, TenantId::from("tenant"));
+        assert_eq!(credential.project, ProjectId::from("project"));
+        assert_eq!(credential.node, NodeId::from("node"));
+        assert_eq!(credential.scope, "node:attach");
+        assert_eq!(
+            credential.capability_policy_digest,
+            node_capability_policy_digest(
+                &TenantId::from("tenant"),
+                &ProjectId::from("project"),
+                "node:attach"
+            )
+        );
+        assert_eq!(
+            grant.exchange_for_node_identity(
+                NodeId::from("node2"),
+                "public-key",
+                "node:attach",
+                99
+            ),
+            Err(EnrollmentError::AlreadyConsumed)
+        );
+    }
+
+    #[test]
+    fn node_capability_policy_digest_is_scoped() {
+        let base = node_capability_policy_digest(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            "node:attach",
+        );
+        let other_project = node_capability_policy_digest(
+            &TenantId::from("tenant"),
+            &ProjectId::from("other"),
+            "node:attach",
+        );
+        let other_scope = node_capability_policy_digest(
+            &TenantId::from("tenant"),
+            &ProjectId::from("project"),
+            "node:limited",
+        );
+
+        assert!(base.is_valid_sha256());
+        assert_ne!(base, other_project);
+        assert_ne!(base, other_scope);
+    }
+
+    #[test]
+    fn generated_ed25519_private_keys_are_random_and_valid() {
+        let first = generate_ed25519_private_key().unwrap();
+        let second = generate_ed25519_private_key().unwrap();
+
+        assert_ne!(first, second);
+        assert!(agent_ed25519_public_key_from_private_key(&first)
+            .unwrap()
+            .starts_with("ed25519:"));
+        assert!(node_ed25519_public_key_from_private_key(&second)
+            .unwrap()
+            .starts_with("ed25519:"));
+    }
+
+    #[test]
+    fn generated_opaque_tokens_are_random_and_url_safe() {
+        let first = generate_opaque_token("grant").unwrap();
+        let second = generate_opaque_token("grant").unwrap();
+
+        assert_ne!(first, second);
+        assert!(first.starts_with("grant_"));
+        assert!(first
+            .bytes()
+            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')));
+    }
+
+    #[test]
+    fn task_credentials_reject_user_session_tokens() {
+        for credential in [
+            CredentialKind::BrowserSession,
+            CredentialKind::CliDeviceSession,
+        ] {
+            let authz = task_credentials_do_not_contain_user_session(
+                &Actor::Task(TaskInstanceId::from("task")),
+                &[CredentialKind::TaskCredential, credential],
+            );
+
+            assert!(!authz.allowed);
+            assert!(authz.reason.contains("must not be passed"));
+        }
+
+        let scoped = task_credentials_do_not_contain_user_session(
+            &Actor::Task(TaskInstanceId::from("task")),
+            &[
+                CredentialKind::TaskCredential,
+                CredentialKind::NodeCredential,
+            ],
+        );
+        assert!(scoped.allowed);
+    }
+
+    #[test]
+    fn identities_remain_distinct_for_authorization() {
+        assert_eq!(Actor::User(UserId::from("user")).kind(), IdentityKind::User);
+        assert_eq!(
+            Actor::Agent(AgentId::from("agent")).kind(),
+            IdentityKind::Agent
+        );
+        assert_eq!(Actor::Node(NodeId::from("node")).kind(), IdentityKind::Node);
+        assert_eq!(
+            Actor::Task(TaskInstanceId::from("task")).kind(),
+            IdentityKind::Task
+        );
+
+        let scope = Scope {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            process: Some(ProcessId::from("process")),
+            task: Some(TaskInstanceId::from("task")),
+            node: Some(NodeId::from("node")),
+            artifact: Some(ArtifactId::from("artifact")),
+        };
+        assert_eq!(scope.process, Some(ProcessId::from("process")));
+        assert_eq!(scope.artifact, Some(ArtifactId::from("artifact")));
+
+        assert_ne!(
+            CredentialKind::BrowserSession,
+            CredentialKind::CliDeviceSession
+        );
+        assert_ne!(CredentialKind::PublicKey, CredentialKind::NodeCredential);
+        assert_ne!(
+            CredentialKind::NodeCredential,
+            CredentialKind::TaskCredential
+        );
+    }
+}

crates/clusterflux-core/src/bundle.rs

@@ -0,0 +1,459 @@
+use serde::{Deserialize, Serialize};
+use syn::parse::Parser;
+use syn::{Expr, Item, Lit, Meta, Token};
+
+use crate::{Digest, EnvironmentResource, SourceTransferPolicy, TaskDefinitionId};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SelectedInput {
+    pub path: String,
+    pub digest: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleIdentityInputs {
+    pub wasm_code: Digest,
+    pub task_abi: Digest,
+    pub entrypoints: Vec<String>,
+    pub default_entrypoint: String,
+    pub environments: Vec<EnvironmentResource>,
+    pub source_provider_manifest: Digest,
+    pub source_transfer_policy: SourceTransferPolicy,
+    pub selected_inputs: Vec<SelectedInput>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleMetadata {
+    pub identity: Digest,
+    pub wasm_code: Digest,
+    pub task_metadata: BundleTaskMetadata,
+    pub source_metadata: BundleSourceMetadata,
+    pub debug_metadata: BundleDebugMetadata,
+    pub large_input_policy: BundleLargeInputPolicy,
+    pub restart_compatibility: BundleRestartCompatibility,
+    pub environments: Vec<EnvironmentResource>,
+    pub selected_inputs: Vec<SelectedInput>,
+    pub embeds_full_container_images: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleTaskMetadata {
+    pub task_abi: Digest,
+    pub entrypoints: Vec<String>,
+    pub default_entrypoint: String,
+    pub boundary: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleSourceMetadata {
+    pub source_provider_manifest: Digest,
+    pub transfer_policy: SourceTransferPolicy,
+    pub selected_inputs: Vec<SelectedInput>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleDebugMetadata {
+    pub available: bool,
+    pub source_level_breakpoints: bool,
+    pub dap_virtual_process: bool,
+    pub variables_pane_supported: bool,
+    pub probes: Vec<BundleDebugProbe>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleDebugProbe {
+    pub id: String,
+    pub source_path: String,
+    pub line_start: u32,
+    pub line_end: u32,
+    pub function: String,
+    pub task: TaskDefinitionId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleLargeInputPolicy {
+    pub selected_inputs_are_content_digests: bool,
+    pub selected_input_bytes_included: bool,
+    pub full_repository_bytes_included: bool,
+    pub silent_task_argument_serialization: bool,
+    pub supported_handle_types: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct BundleRestartCompatibility {
+    pub source_edits_can_restart_from_clean_task_boundary: bool,
+    pub requires_clean_checkpoint_boundary: bool,
+    pub compares_task_abi: Digest,
+    pub compares_environment_digests: bool,
+    pub compares_serialized_args: bool,
+    pub discards_unflushed_task_local_changes: bool,
+    pub incompatible_changes_require_whole_process_restart: bool,
+}
+
+impl BundleIdentityInputs {
+    pub fn identity(&self) -> Digest {
+        let mut parts = vec![
+            b"bundle:v1".to_vec(),
+            self.wasm_code.as_str().as_bytes().to_vec(),
+            self.task_abi.as_str().as_bytes().to_vec(),
+            self.source_provider_manifest.as_str().as_bytes().to_vec(),
+            self.default_entrypoint.as_bytes().to_vec(),
+            format!("{:?}", self.source_transfer_policy).into_bytes(),
+        ];
+
+        let mut entrypoints = self.entrypoints.clone();
+        entrypoints.sort();
+        for entrypoint in entrypoints {
+            parts.push(entrypoint.into_bytes());
+        }
+
+        let mut environments = self.environments.clone();
+        environments.sort_by(|left, right| left.name.cmp(&right.name));
+        for environment in environments {
+            parts.push(environment.name.as_bytes().to_vec());
+            parts.push(format!("{:?}", environment.kind).into_bytes());
+            parts.push(environment.digest.as_str().as_bytes().to_vec());
+        }
+
+        let mut inputs = self.selected_inputs.clone();
+        inputs.sort_by(|left, right| left.path.cmp(&right.path));
+        for input in inputs {
+            parts.push(input.path.into_bytes());
+            parts.push(input.digest.as_str().as_bytes().to_vec());
+        }
+
+        Digest::from_parts(parts)
+    }
+
+    pub fn inspectable_metadata(&self) -> BundleMetadata {
+        let mut entrypoints = self.entrypoints.clone();
+        entrypoints.sort();
+
+        BundleMetadata {
+            identity: self.identity(),
+            wasm_code: self.wasm_code.clone(),
+            task_metadata: BundleTaskMetadata {
+                task_abi: self.task_abi.clone(),
+                entrypoints,
+                default_entrypoint: self.default_entrypoint.clone(),
+                boundary: "clusterflux_task_exports".to_owned(),
+            },
+            source_metadata: BundleSourceMetadata {
+                source_provider_manifest: self.source_provider_manifest.clone(),
+                transfer_policy: self.source_transfer_policy.clone(),
+                selected_inputs: self.selected_inputs.clone(),
+            },
+            debug_metadata: BundleDebugMetadata {
+                available: true,
+                source_level_breakpoints: true,
+                dap_virtual_process: true,
+                variables_pane_supported: true,
+                probes: Vec::new(),
+            },
+            large_input_policy: BundleLargeInputPolicy {
+                selected_inputs_are_content_digests: true,
+                selected_input_bytes_included: false,
+                full_repository_bytes_included: false,
+                silent_task_argument_serialization: false,
+                supported_handle_types: vec![
+                    "SourceSnapshot".to_owned(),
+                    "Blob".to_owned(),
+                    "Artifact".to_owned(),
+                    "VFS".to_owned(),
+                ],
+            },
+            restart_compatibility: BundleRestartCompatibility {
+                source_edits_can_restart_from_clean_task_boundary: true,
+                requires_clean_checkpoint_boundary: true,
+                compares_task_abi: self.task_abi.clone(),
+                compares_environment_digests: true,
+                compares_serialized_args: true,
+                discards_unflushed_task_local_changes: true,
+                incompatible_changes_require_whole_process_restart: true,
+            },
+            environments: self.environments.clone(),
+            selected_inputs: self.selected_inputs.clone(),
+            embeds_full_container_images: false,
+        }
+    }
+}
+
+pub fn discover_source_debug_probes(
+    source_path: impl Into<String>,
+    source: &str,
+) -> Vec<BundleDebugProbe> {
+    let source_path = source_path.into();
+    let lines = source.lines().collect::<Vec<_>>();
+    let function_starts = lines
+        .iter()
+        .enumerate()
+        .filter_map(|(index, line)| {
+            parse_rust_function_name(line).map(|function| (index, function))
+        })
+        .collect::<Vec<_>>();
+    let Ok(file) = syn::parse_file(source) else {
+        return Vec::new();
+    };
+
+    file.items
+        .iter()
+        .filter_map(|item| {
+            let Item::Fn(function) = item else {
+                return None;
+            };
+            let function_name = function.sig.ident.to_string();
+            let task = clusterflux_probe_task(function)?;
+            let (function_index, (line_index, _)) = function_starts
+                .iter()
+                .enumerate()
+                .find(|(_, (_, candidate))| candidate == &function_name)?;
+            let line_start = (*line_index + 1) as u32;
+            let next_start = function_starts
+                .get(function_index + 1)
+                .map(|(next_index, _)| *next_index)
+                .unwrap_or(lines.len());
+            let line_end = next_start.max(*line_index + 1) as u32;
+            Some(debug_probe(
+                &source_path,
+                line_start,
+                line_end,
+                &function_name,
+                task,
+            ))
+        })
+        .collect()
+}
+
+fn debug_probe(
+    source_path: &str,
+    line_start: u32,
+    line_end: u32,
+    function: &str,
+    task: TaskDefinitionId,
+) -> BundleDebugProbe {
+    let id = Digest::from_parts([
+        b"bundle-debug-probe:v1".as_slice(),
+        source_path.as_bytes(),
+        function.as_bytes(),
+        task.as_str().as_bytes(),
+        line_start.to_string().as_bytes(),
+        line_end.to_string().as_bytes(),
+    ])
+    .as_str()
+    .to_owned();
+    BundleDebugProbe {
+        id,
+        source_path: source_path.to_owned(),
+        line_start,
+        line_end,
+        function: function.to_owned(),
+        task,
+    }
+}
+
+fn parse_rust_function_name(line: &str) -> Option<String> {
+    let start = line.find("fn ")? + 3;
+    let rest = &line[start..];
+    let name = rest
+        .chars()
+        .take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
+        .collect::<String>();
+    (!name.is_empty()).then_some(name)
+}
+
+fn clusterflux_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> {
+    for attribute in &function.attrs {
+        let mut segments = attribute.path().segments.iter();
+        let Some(namespace) = segments.next() else {
+            continue;
+        };
+        let Some(kind) = segments.next() else {
+            continue;
+        };
+        if namespace.ident != "clusterflux" || segments.next().is_some() {
+            continue;
+        }
+        let function_name = function.sig.ident.to_string();
+        let default_name = match kind.ident.to_string().as_str() {
+            "main" => function_name
+                .strip_suffix("_main")
+                .unwrap_or(&function_name)
+                .to_owned(),
+            "task" => function_name,
+            _ => continue,
+        };
+        let declared_name = match &attribute.meta {
+            Meta::List(list) => {
+                let parser = syn::punctuated::Punctuated::<Meta, Token![,]>::parse_terminated;
+                parser
+                    .parse2(list.tokens.clone())
+                    .ok()
+                    .and_then(|items| {
+                        items.into_iter().find_map(|item| {
+                            let Meta::NameValue(name_value) = item else {
+                                return None;
+                            };
+                            if !name_value.path.is_ident("name") {
+                                return None;
+                            }
+                            let Expr::Lit(value) = name_value.value else {
+                                return None;
+                            };
+                            let Lit::Str(value) = value.lit else {
+                                return None;
+                            };
+                            Some(value.value())
+                        })
+                    })
+                    .unwrap_or(default_name)
+            }
+            _ => default_name,
+        };
+        return Some(TaskDefinitionId::new(declared_name));
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use crate::{EnvironmentKind, EnvironmentRequirements};
+
+    use super::*;
+
+    fn env(digest: &str) -> EnvironmentResource {
+        EnvironmentResource {
+            name: "linux".to_owned(),
+            kind: EnvironmentKind::Containerfile,
+            recipe_path: PathBuf::from("envs/linux/Containerfile"),
+            context_path: PathBuf::from("envs/linux"),
+            digest: Digest::sha256(digest),
+            requirements: EnvironmentRequirements::linux_container(),
+        }
+    }
+
+    #[test]
+    fn bundle_identity_changes_when_environment_recipe_changes() {
+        let base = BundleIdentityInputs {
+            wasm_code: Digest::sha256("wasm"),
+            task_abi: Digest::sha256("abi"),
+            entrypoints: vec!["build".to_owned()],
+            default_entrypoint: "build".to_owned(),
+            environments: vec![env("recipe-a")],
+            source_provider_manifest: Digest::sha256("source"),
+            source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
+            selected_inputs: vec![],
+        };
+        let mut changed = base.clone();
+        changed.environments = vec![env("recipe-b")];
+
+        assert_ne!(base.identity(), changed.identity());
+    }
+
+    #[test]
+    fn bundle_metadata_is_inspectable_and_does_not_vendor_images_by_default() {
+        let inputs = BundleIdentityInputs {
+            wasm_code: Digest::sha256("wasm"),
+            task_abi: Digest::sha256("abi"),
+            entrypoints: vec!["build".to_owned(), "test".to_owned()],
+            default_entrypoint: "build".to_owned(),
+            environments: vec![env("recipe")],
+            source_provider_manifest: Digest::sha256("source"),
+            source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
+            selected_inputs: vec![SelectedInput {
+                path: "inputs/config.json".to_owned(),
+                digest: Digest::sha256("config"),
+            }],
+        };
+
+        let metadata = inputs.inspectable_metadata();
+
+        assert!(metadata.wasm_code.as_str().starts_with("sha256:"));
+        assert_eq!(metadata.task_metadata.default_entrypoint, "build");
+        assert!(metadata
+            .task_metadata
+            .entrypoints
+            .contains(&"test".to_owned()));
+        assert!(metadata.debug_metadata.dap_virtual_process);
+        assert!(
+            metadata
+                .source_metadata
+                .transfer_policy
+                .local_source_bytes_remain_node_local
+        );
+        assert!(
+            metadata
+                .large_input_policy
+                .selected_inputs_are_content_digests
+        );
+        assert!(!metadata.large_input_policy.selected_input_bytes_included);
+        assert!(!metadata.large_input_policy.full_repository_bytes_included);
+        assert!(
+            !metadata
+                .large_input_policy
+                .silent_task_argument_serialization
+        );
+        assert!(metadata
+            .large_input_policy
+            .supported_handle_types
+            .contains(&"Artifact".to_owned()));
+        assert!(
+            metadata
+                .restart_compatibility
+                .source_edits_can_restart_from_clean_task_boundary
+        );
+        assert!(
+            metadata
+                .restart_compatibility
+                .requires_clean_checkpoint_boundary
+        );
+        assert_eq!(
+            metadata.restart_compatibility.compares_task_abi,
+            inputs.task_abi
+        );
+        assert!(
+            metadata
+                .restart_compatibility
+                .incompatible_changes_require_whole_process_restart
+        );
+        assert_eq!(metadata.environments.len(), 1);
+        assert!(!metadata.embeds_full_container_images);
+    }
+
+    #[test]
+    fn source_debug_probe_metadata_maps_function_ranges_to_tasks() {
+        let probes = discover_source_debug_probes(
+            "src/build.rs",
+            r#"#[clusterflux::main]
+fn build_main() {
+    let linux = compile_linux();
+}
+
+#[clusterflux::task]
+fn compile_linux() {
+    println!("linux");
+}
+
+fn helper_without_runtime_probe() {}
+
+#[clusterflux::task(name = "release")]
+fn package_release() {
+    println!("package");
+}
+"#,
+        );
+
+        assert_eq!(probes.len(), 3);
+        assert_eq!(probes[0].source_path, "src/build.rs");
+        assert_eq!(probes[0].function, "build_main");
+        assert_eq!(probes[0].task, TaskDefinitionId::from("build"));
+        assert_eq!(probes[0].line_start, 2);
+        assert_eq!(probes[0].line_end, 6);
+        assert_eq!(probes[1].function, "compile_linux");
+        assert_eq!(probes[1].task, TaskDefinitionId::from("compile_linux"));
+        assert_eq!(probes[2].function, "package_release");
+        assert_eq!(probes[2].task, TaskDefinitionId::from("release"));
+        assert!(probes.iter().all(|probe| probe.id.starts_with("sha256:")));
+    }
+}

crates/clusterflux-core/src/capability.rs

@@ -0,0 +1,218 @@
+use std::collections::BTreeSet;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum Capability {
+    Command,
+    Containers,
+    RootlessPodman,
+    SourceFilesystem,
+    SourceGit,
+    HostFilesystem,
+    Network,
+    Secrets,
+    InboundPorts,
+    ArbitrarySyscalls,
+    VfsArtifacts,
+    WindowsCommandDev,
+    QuicDirect,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum EnvironmentBackend {
+    Container,
+    NixFlake,
+    WindowsCommandDev,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum Os {
+    Linux,
+    Windows,
+    Macos,
+    Other(String),
+}
+
+impl Os {
+    pub fn current() -> Self {
+        match std::env::consts::OS {
+            "linux" => Self::Linux,
+            "windows" => Self::Windows,
+            "macos" => Self::Macos,
+            other => Self::Other(other.to_owned()),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeCapabilities {
+    pub os: Os,
+    pub arch: String,
+    pub capabilities: BTreeSet<Capability>,
+    pub environment_backends: BTreeSet<EnvironmentBackend>,
+    pub source_providers: BTreeSet<String>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum CapabilityReportError {
+    #[error("node architecture `{0}` is invalid")]
+    InvalidArchitecture(String),
+    #[error("node OS label `{0}` is invalid")]
+    InvalidOsLabel(String),
+    #[error("source provider id `{0}` is invalid")]
+    InvalidSourceProvider(String),
+}
+
+impl NodeCapabilities {
+    pub fn detect_current() -> Self {
+        let os = Os::current();
+        let mut capabilities = BTreeSet::from([
+            Capability::Command,
+            Capability::SourceFilesystem,
+            Capability::VfsArtifacts,
+        ]);
+        let mut environment_backends = BTreeSet::new();
+
+        match os {
+            Os::Linux => {
+                if rootless_podman_available() {
+                    capabilities.insert(Capability::Containers);
+                    capabilities.insert(Capability::RootlessPodman);
+                    environment_backends.insert(EnvironmentBackend::Container);
+                }
+            }
+            Os::Windows => {
+                capabilities.insert(Capability::WindowsCommandDev);
+                environment_backends.insert(EnvironmentBackend::WindowsCommandDev);
+            }
+            Os::Macos | Os::Other(_) => {}
+        }
+
+        Self {
+            os,
+            arch: std::env::consts::ARCH.to_owned(),
+            capabilities,
+            environment_backends,
+            source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
+        }
+    }
+
+    pub fn with_capability(mut self, capability: Capability) -> Self {
+        self.capabilities.insert(capability);
+        self
+    }
+
+    pub fn has_all(&self, required: &BTreeSet<Capability>) -> bool {
+        required
+            .iter()
+            .all(|capability| self.capabilities.contains(capability))
+    }
+
+    pub fn validate_public_report(&self) -> Result<(), CapabilityReportError> {
+        if !valid_capability_label(&self.arch) {
+            return Err(CapabilityReportError::InvalidArchitecture(
+                self.arch.clone(),
+            ));
+        }
+        if let Os::Other(label) = &self.os {
+            if !valid_capability_label(label) {
+                return Err(CapabilityReportError::InvalidOsLabel(label.clone()));
+            }
+        }
+        for provider in &self.source_providers {
+            if !valid_source_provider_id(provider) {
+                return Err(CapabilityReportError::InvalidSourceProvider(
+                    provider.clone(),
+                ));
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn rootless_podman_available() -> bool {
+    const ATTEMPTS: usize = 3;
+    for attempt in 0..ATTEMPTS {
+        match std::process::Command::new("podman")
+            .args(["info", "--format", "{{.Host.Security.Rootless}}"])
+            .output()
+        {
+            Ok(output) if rootless_podman_probe_succeeded(&output) => return true,
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return false,
+            Ok(_) | Err(_) if attempt + 1 < ATTEMPTS => {
+                std::thread::sleep(std::time::Duration::from_millis(250));
+            }
+            Ok(_) | Err(_) => {}
+        }
+    }
+    false
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn rootless_podman_probe_succeeded(output: &std::process::Output) -> bool {
+    output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == "true"
+}
+
+#[cfg(target_arch = "wasm32")]
+fn rootless_podman_available() -> bool {
+    false
+}
+
+fn valid_capability_label(label: &str) -> bool {
+    !label.is_empty()
+        && label.len() <= 64
+        && label.bytes().all(
+            |byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.'),
+        )
+}
+
+fn valid_source_provider_id(provider: &str) -> bool {
+    !provider.is_empty()
+        && provider.len() <= 64
+        && provider
+            .bytes()
+            .all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn capabilities() -> NodeCapabilities {
+        NodeCapabilities {
+            os: Os::Linux,
+            arch: "x86_64".to_owned(),
+            capabilities: BTreeSet::from([Capability::Command]),
+            environment_backends: BTreeSet::new(),
+            source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
+        }
+    }
+
+    #[test]
+    fn capability_reports_validate_hostile_strings() {
+        assert!(capabilities().validate_public_report().is_ok());
+
+        let mut invalid_arch = capabilities();
+        invalid_arch.arch = "x86_64\nmalicious".to_owned();
+        assert_eq!(
+            invalid_arch.validate_public_report(),
+            Err(CapabilityReportError::InvalidArchitecture(
+                "x86_64\nmalicious".to_owned()
+            ))
+        );
+
+        let mut invalid_provider = capabilities();
+        invalid_provider
+            .source_providers
+            .insert("../checkout".to_owned());
+        assert_eq!(
+            invalid_provider.validate_public_report(),
+            Err(CapabilityReportError::InvalidSourceProvider(
+                "../checkout".to_owned()
+            ))
+        );
+    }
+}

crates/clusterflux-core/src/checkpoint.rs

@@ -0,0 +1,284 @@
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{Digest, TaskInstanceId, VfsManifest};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CheckpointBoundary {
+    pub task_entrypoint: String,
+    pub serialized_args: Digest,
+    pub environment_digest: Digest,
+    pub vfs_epoch: u64,
+    pub task_abi: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCheckpoint {
+    pub task: TaskInstanceId,
+    pub boundary: CheckpointBoundary,
+    pub vfs_manifest: VfsManifest,
+    pub depends_on_live_stack: bool,
+    pub depends_on_live_socket: bool,
+    pub depends_on_ephemeral_artifact_durability: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RestartRequest {
+    pub task: TaskInstanceId,
+    pub entrypoint: String,
+    pub serialized_args: Digest,
+    pub environment_digest: Digest,
+    pub task_abi: Digest,
+    pub source_edited: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum RestartDecision {
+    RestartTask {
+        task: TaskInstanceId,
+        from_vfs_epoch: u64,
+        discard_unflushed_changes: bool,
+    },
+    RestartWholeVirtualProcess {
+        message: String,
+    },
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct RestartPolicy;
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum CompatibilityFailure {
+    #[error("task entrypoint changed")]
+    Entrypoint,
+    #[error("serialized task arguments changed")]
+    Args,
+    #[error("environment digest changed")]
+    Environment,
+    #[error("task ABI changed")]
+    TaskAbi,
+    #[error("checkpoint depends on unsupported live stack migration")]
+    LiveStack,
+    #[error("checkpoint depends on unsupported live socket checkpointing")]
+    LiveSocket,
+    #[error("checkpoint incorrectly treats ephemeral artifacts as durable")]
+    EphemeralArtifactDurability,
+}
+
+impl RestartPolicy {
+    pub fn decide(&self, checkpoint: &TaskCheckpoint, request: &RestartRequest) -> RestartDecision {
+        match compatibility_failure(checkpoint, request) {
+            None => RestartDecision::RestartTask {
+                task: checkpoint.task.clone(),
+                from_vfs_epoch: checkpoint.boundary.vfs_epoch,
+                discard_unflushed_changes: true,
+            },
+            Some(failure) => RestartDecision::RestartWholeVirtualProcess {
+                message: format!(
+                    "cannot restart selected task `{}` from checkpoint: {failure}; restart the whole virtual process",
+                    checkpoint.task
+                ),
+            },
+        }
+    }
+}
+
+fn compatibility_failure(
+    checkpoint: &TaskCheckpoint,
+    request: &RestartRequest,
+) -> Option<CompatibilityFailure> {
+    if checkpoint.depends_on_live_stack {
+        return Some(CompatibilityFailure::LiveStack);
+    }
+    if checkpoint.depends_on_live_socket {
+        return Some(CompatibilityFailure::LiveSocket);
+    }
+    if checkpoint.depends_on_ephemeral_artifact_durability {
+        return Some(CompatibilityFailure::EphemeralArtifactDurability);
+    }
+    if checkpoint.boundary.task_entrypoint != request.entrypoint {
+        return Some(CompatibilityFailure::Entrypoint);
+    }
+    if checkpoint.boundary.serialized_args != request.serialized_args {
+        return Some(CompatibilityFailure::Args);
+    }
+    if checkpoint.boundary.environment_digest != request.environment_digest {
+        return Some(CompatibilityFailure::Environment);
+    }
+    if checkpoint.boundary.task_abi != request.task_abi {
+        return Some(CompatibilityFailure::TaskAbi);
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use crate::{
+        EnvironmentKind, EnvironmentRequirements, EnvironmentResource, NodeId, VfsOverlay, VfsPath,
+    };
+
+    use super::*;
+
+    fn env(digest_input: &str) -> EnvironmentResource {
+        EnvironmentResource {
+            name: "linux".to_owned(),
+            kind: EnvironmentKind::Containerfile,
+            recipe_path: PathBuf::from("envs/linux/Containerfile"),
+            context_path: PathBuf::from("envs/linux"),
+            digest: Digest::sha256(digest_input),
+            requirements: EnvironmentRequirements::linux_container(),
+        }
+    }
+
+    fn checkpoint() -> (TaskCheckpoint, EnvironmentResource) {
+        let environment = env("env");
+        let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node"));
+        overlay.write(
+            VfsPath::new("/vfs/artifacts/app").unwrap(),
+            Digest::sha256("app"),
+            3,
+        );
+        let manifest = overlay.flush();
+        (
+            TaskCheckpoint {
+                task: TaskInstanceId::from("task"),
+                boundary: CheckpointBoundary {
+                    task_entrypoint: "compile_linux".to_owned(),
+                    serialized_args: Digest::sha256("args"),
+                    environment_digest: environment.digest.clone(),
+                    vfs_epoch: manifest.epoch,
+                    task_abi: Digest::sha256("abi"),
+                },
+                vfs_manifest: manifest,
+                depends_on_live_stack: false,
+                depends_on_live_socket: false,
+                depends_on_ephemeral_artifact_durability: false,
+            },
+            environment,
+        )
+    }
+
+    fn restart_request(environment: EnvironmentResource) -> RestartRequest {
+        RestartRequest {
+            task: TaskInstanceId::from("task"),
+            entrypoint: "compile_linux".to_owned(),
+            serialized_args: Digest::sha256("args"),
+            environment_digest: environment.digest,
+            task_abi: Digest::sha256("abi"),
+            source_edited: true,
+        }
+    }
+
+    fn assert_whole_process_restart(decision: RestartDecision, expected_reason: &str) {
+        match decision {
+            RestartDecision::RestartWholeVirtualProcess { message } => {
+                assert!(
+                    message.contains(expected_reason),
+                    "restart message `{message}` did not include `{expected_reason}`"
+                );
+                assert!(
+                    message.contains("restart the whole virtual process"),
+                    "restart message `{message}` did not direct a whole-process restart"
+                );
+            }
+            RestartDecision::RestartTask { .. } => {
+                panic!("incompatible checkpoint unexpectedly restarted selected task")
+            }
+        }
+    }
+
+    #[test]
+    fn compatible_restart_uses_task_boundary_and_discards_unflushed_changes() {
+        let (checkpoint, environment) = checkpoint();
+        let request = restart_request(environment);
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_eq!(
+            decision,
+            RestartDecision::RestartTask {
+                task: TaskInstanceId::from("task"),
+                from_vfs_epoch: 1,
+                discard_unflushed_changes: true
+            }
+        );
+    }
+
+    #[test]
+    fn incompatible_environment_requires_whole_process_restart() {
+        let (checkpoint, _) = checkpoint();
+        let request = restart_request(env("changed-env"));
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "environment digest changed");
+    }
+
+    #[test]
+    fn incompatible_entrypoint_requires_whole_process_restart() {
+        let (checkpoint, environment) = checkpoint();
+        let mut request = restart_request(environment);
+        request.entrypoint = "package_linux".to_owned();
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "task entrypoint changed");
+    }
+
+    #[test]
+    fn incompatible_serialized_args_require_whole_process_restart() {
+        let (checkpoint, environment) = checkpoint();
+        let mut request = restart_request(environment);
+        request.serialized_args = Digest::sha256("changed-args");
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "serialized task arguments changed");
+    }
+
+    #[test]
+    fn incompatible_task_abi_requires_whole_process_restart() {
+        let (checkpoint, environment) = checkpoint();
+        let mut request = restart_request(environment);
+        request.task_abi = Digest::sha256("changed-abi");
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "task ABI changed");
+    }
+
+    #[test]
+    fn restart_never_claims_live_stack_migration() {
+        let (mut checkpoint, environment) = checkpoint();
+        checkpoint.depends_on_live_stack = true;
+        let request = restart_request(environment);
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "live stack");
+    }
+
+    #[test]
+    fn restart_never_claims_live_socket_checkpointing() {
+        let (mut checkpoint, environment) = checkpoint();
+        checkpoint.depends_on_live_socket = true;
+        let request = restart_request(environment);
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "live socket");
+    }
+
+    #[test]
+    fn restart_never_depends_on_ephemeral_artifact_durability() {
+        let (mut checkpoint, environment) = checkpoint();
+        checkpoint.depends_on_ephemeral_artifact_durability = true;
+        let request = restart_request(environment);
+
+        let decision = RestartPolicy.decide(&checkpoint, &request);
+
+        assert_whole_process_restart(decision, "ephemeral artifacts");
+    }
+}

crates/clusterflux-core/src/debug.rs

@@ -0,0 +1,337 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{ProcessId, TaskInstanceId};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DebugParticipantKind {
+    WasmTask,
+    ControlledNativeCommand,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DebugRuntimeState {
+    Running,
+    Frozen,
+    Completed,
+    Failed(String),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipant {
+    pub task: TaskInstanceId,
+    pub name: String,
+    pub kind: DebugParticipantKind,
+    pub can_freeze: bool,
+    pub state: DebugRuntimeState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DebugStopReason {
+    Breakpoint { task: TaskInstanceId, line: u32 },
+    PauseRequest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ThreadInspection {
+    pub task: TaskInstanceId,
+    pub name: String,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugEpoch {
+    pub process: ProcessId,
+    pub epoch: u64,
+    pub reason: DebugStopReason,
+    participants: BTreeMap<TaskInstanceId, DebugParticipant>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum DebugEpochError {
+    #[error("no participant could freeze; `{task}` rejected the freeze request")]
+    CannotFreeze { task: TaskInstanceId },
+    #[error("participant `{0}` is not part of this debug epoch")]
+    UnknownParticipant(TaskInstanceId),
+    #[error("participant `{0}` did not acknowledge frozen state in this debug epoch")]
+    ParticipantNotFrozen(TaskInstanceId),
+}
+
+impl DebugEpoch {
+    pub fn all_stop(
+        process: ProcessId,
+        epoch: u64,
+        reason: DebugStopReason,
+        participants: Vec<DebugParticipant>,
+    ) -> Result<Self, DebugEpochError> {
+        let first_rejected = participants
+            .iter()
+            .find(|participant| {
+                matches!(participant.state, DebugRuntimeState::Running) && !participant.can_freeze
+            })
+            .map(|participant| participant.task.clone());
+        let participants: BTreeMap<_, _> = participants
+            .into_iter()
+            .map(|mut participant| {
+                if matches!(participant.state, DebugRuntimeState::Running) {
+                    participant.state = if participant.can_freeze {
+                        DebugRuntimeState::Frozen
+                    } else {
+                        DebugRuntimeState::Failed(
+                            "participant did not acknowledge frozen state before the debug deadline"
+                                .to_owned(),
+                        )
+                    };
+                }
+                (participant.task.clone(), participant)
+            })
+            .collect();
+
+        if !participants
+            .values()
+            .any(|participant| participant.state == DebugRuntimeState::Frozen)
+        {
+            return Err(DebugEpochError::CannotFreeze {
+                task: first_rejected.unwrap_or_else(|| TaskInstanceId::from("debug-epoch")),
+            });
+        }
+
+        Ok(Self {
+            process,
+            epoch,
+            reason,
+            participants,
+        })
+    }
+
+    pub fn pause(
+        process: ProcessId,
+        epoch: u64,
+        participants: Vec<DebugParticipant>,
+    ) -> Result<Self, DebugEpochError> {
+        Self::all_stop(process, epoch, DebugStopReason::PauseRequest, participants)
+    }
+
+    pub fn continue_all(&mut self) {
+        for participant in self.participants.values_mut() {
+            if participant.state == DebugRuntimeState::Frozen {
+                participant.state = DebugRuntimeState::Running;
+            }
+        }
+    }
+
+    pub fn inspection(&self, task: &TaskInstanceId) -> Result<ThreadInspection, DebugEpochError> {
+        let participant = self
+            .participants
+            .get(task)
+            .ok_or_else(|| DebugEpochError::UnknownParticipant(task.clone()))?;
+        if participant.state != DebugRuntimeState::Frozen {
+            return Err(DebugEpochError::ParticipantNotFrozen(task.clone()));
+        }
+        Ok(ThreadInspection {
+            task: participant.task.clone(),
+            name: participant.name.clone(),
+            stack_frames: participant.stack_frames.clone(),
+            local_values: participant.local_values.clone(),
+            task_args: participant.task_args.clone(),
+            handles: participant.handles.clone(),
+            command_status: participant.command_status.clone(),
+            recent_output: participant.recent_output.clone(),
+        })
+    }
+
+    pub fn participant_state(&self, task: &TaskInstanceId) -> Option<&DebugRuntimeState> {
+        self.participants
+            .get(task)
+            .map(|participant| &participant.state)
+    }
+
+    pub fn thread_names(&self) -> Vec<String> {
+        self.participants
+            .values()
+            .map(|participant| participant.name.clone())
+            .collect()
+    }
+
+    pub fn all_threads_stopped(&self) -> bool {
+        !self.participants.is_empty()
+            && self
+                .participants
+                .values()
+                .all(|participant| participant.state == DebugRuntimeState::Frozen)
+    }
+
+    pub fn partially_frozen(&self) -> bool {
+        !self.all_threads_stopped()
+            && self
+                .participants
+                .values()
+                .any(|participant| participant.state == DebugRuntimeState::Frozen)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn participant(task: &str, kind: DebugParticipantKind, can_freeze: bool) -> DebugParticipant {
+        DebugParticipant {
+            task: TaskInstanceId::from(task),
+            name: task.to_owned(),
+            kind,
+            can_freeze,
+            state: DebugRuntimeState::Running,
+            stack_frames: vec![format!("{task}::run")],
+            local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())],
+            task_args: vec![("target".to_owned(), "linux".to_owned())],
+            handles: vec![("artifact".to_owned(), "artifact-1".to_owned())],
+            command_status: Some("running".to_owned()),
+            recent_output: vec!["building".to_owned()],
+        }
+    }
+
+    #[test]
+    fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks() {
+        let epoch = DebugEpoch::all_stop(
+            ProcessId::from("process"),
+            1,
+            DebugStopReason::Breakpoint {
+                task: TaskInstanceId::from("compile-linux"),
+                line: 42,
+            },
+            vec![
+                participant("main", DebugParticipantKind::WasmTask, true),
+                participant(
+                    "compile-linux",
+                    DebugParticipantKind::ControlledNativeCommand,
+                    true,
+                ),
+            ],
+        )
+        .unwrap();
+
+        assert_eq!(
+            epoch.participant_state(&TaskInstanceId::from("main")),
+            Some(&DebugRuntimeState::Frozen)
+        );
+        assert_eq!(
+            epoch.participant_state(&TaskInstanceId::from("compile-linux")),
+            Some(&DebugRuntimeState::Frozen)
+        );
+    }
+
+    #[test]
+    fn debug_epoch_reports_failure_when_no_participant_can_freeze() {
+        let error = DebugEpoch::pause(
+            ProcessId::from("process"),
+            1,
+            vec![participant(
+                "compile-linux",
+                DebugParticipantKind::ControlledNativeCommand,
+                false,
+            )],
+        )
+        .unwrap_err();
+
+        assert!(matches!(error, DebugEpochError::CannotFreeze { .. }));
+    }
+
+    #[test]
+    fn debug_epoch_keeps_frozen_participants_when_another_participant_fails() {
+        let epoch = DebugEpoch::pause(
+            ProcessId::from("process"),
+            1,
+            vec![
+                participant("main", DebugParticipantKind::WasmTask, true),
+                participant(
+                    "native",
+                    DebugParticipantKind::ControlledNativeCommand,
+                    false,
+                ),
+            ],
+        )
+        .unwrap();
+
+        assert!(epoch.partially_frozen());
+        assert!(!epoch.all_threads_stopped());
+        assert!(matches!(
+            epoch.participant_state(&TaskInstanceId::from("native")),
+            Some(DebugRuntimeState::Failed(_))
+        ));
+        assert!(matches!(
+            epoch.inspection(&TaskInstanceId::from("native")),
+            Err(DebugEpochError::ParticipantNotFrozen(_))
+        ));
+    }
+
+    #[test]
+    fn continue_resumes_every_frozen_participant() {
+        let mut epoch = DebugEpoch::pause(
+            ProcessId::from("process"),
+            1,
+            vec![
+                participant("main", DebugParticipantKind::WasmTask, true),
+                participant("task", DebugParticipantKind::WasmTask, true),
+            ],
+        )
+        .unwrap();
+
+        epoch.continue_all();
+
+        assert_eq!(
+            epoch.participant_state(&TaskInstanceId::from("main")),
+            Some(&DebugRuntimeState::Running)
+        );
+        assert_eq!(
+            epoch.participant_state(&TaskInstanceId::from("task")),
+            Some(&DebugRuntimeState::Running)
+        );
+    }
+
+    #[test]
+    fn inspection_exposes_stack_args_handles_command_status_and_output() {
+        let epoch = DebugEpoch::pause(
+            ProcessId::from("process"),
+            1,
+            vec![participant(
+                "compile-linux",
+                DebugParticipantKind::ControlledNativeCommand,
+                true,
+            )],
+        )
+        .unwrap();
+
+        let inspection = epoch
+            .inspection(&TaskInstanceId::from("compile-linux"))
+            .unwrap();
+
+        assert_eq!(inspection.stack_frames, vec!["compile-linux::run"]);
+        assert_eq!(
+            inspection.local_values[0],
+            ("wasm_local_0".to_owned(), "I32(41)".to_owned())
+        );
+        assert_eq!(
+            inspection.task_args[0],
+            ("target".to_owned(), "linux".to_owned())
+        );
+        assert_eq!(
+            inspection.handles[0],
+            ("artifact".to_owned(), "artifact-1".to_owned())
+        );
+        assert_eq!(inspection.command_status, Some("running".to_owned()));
+        assert_eq!(inspection.recent_output, vec!["building"]);
+    }
+}

crates/clusterflux-core/src/digest.rs

@@ -0,0 +1,68 @@
+use serde::{Deserialize, Serialize};
+use sha2::{Digest as ShaDigest, Sha256};
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct Digest(String);
+
+impl Digest {
+    pub fn sha256(bytes: impl AsRef<[u8]>) -> Self {
+        let mut hasher = Sha256::new();
+        hasher.update(bytes.as_ref());
+        Self(format!("sha256:{}", hex::encode(hasher.finalize())))
+    }
+
+    pub fn from_parts(parts: impl IntoIterator<Item = impl AsRef<[u8]>>) -> Self {
+        let mut hasher = Sha256::new();
+        for part in parts {
+            let part = part.as_ref();
+            hasher.update((part.len() as u64).to_be_bytes());
+            hasher.update(part);
+        }
+        Self(format!("sha256:{}", hex::encode(hasher.finalize())))
+    }
+
+    pub fn from_sha256_hex(hex_digest: impl Into<String>) -> Result<Self, String> {
+        let digest = Self(format!("sha256:{}", hex_digest.into()));
+        if digest.is_valid_sha256() {
+            Ok(digest)
+        } else {
+            Err(
+                "SHA-256 digest must contain exactly 64 lowercase hexadecimal characters"
+                    .to_owned(),
+            )
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    pub fn is_valid_sha256(&self) -> bool {
+        let Some(hex) = self.0.strip_prefix("sha256:") else {
+            return false;
+        };
+        hex.len() == 64
+            && hex
+                .bytes()
+                .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'))
+    }
+}
+
+impl std::fmt::Display for Digest {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn digest_validates_strict_sha256_syntax() {
+        assert!(Digest::sha256("bytes").is_valid_sha256());
+        assert!(!Digest("sha1:abc".to_owned()).is_valid_sha256());
+        assert!(!Digest("sha256:ABCDEF".to_owned()).is_valid_sha256());
+        assert!(!Digest("sha256:not-hex".to_owned()).is_valid_sha256());
+    }
+}

crates/clusterflux-core/src/environment.rs

@@ -0,0 +1,288 @@
+use std::collections::BTreeSet;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{Capability, Digest, Os};
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum EnvironmentKind {
+    Containerfile,
+    Dockerfile,
+    NixFlake,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EnvironmentRequirements {
+    pub os: Option<Os>,
+    pub arch: Option<String>,
+    pub capabilities: BTreeSet<Capability>,
+}
+
+impl EnvironmentRequirements {
+    pub fn linux_container() -> Self {
+        Self {
+            os: Some(Os::Linux),
+            arch: None,
+            capabilities: BTreeSet::from([Capability::Containers, Capability::RootlessPodman]),
+        }
+    }
+
+    pub fn windows_command_dev() -> Self {
+        Self {
+            os: Some(Os::Windows),
+            arch: None,
+            capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
+        }
+    }
+
+    pub fn unconstrained() -> Self {
+        Self {
+            os: None,
+            arch: None,
+            capabilities: BTreeSet::new(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EnvironmentResource {
+    pub name: String,
+    pub kind: EnvironmentKind,
+    pub recipe_path: PathBuf,
+    pub context_path: PathBuf,
+    pub digest: Digest,
+    pub requirements: EnvironmentRequirements,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EnvironmentReference {
+    pub name: String,
+    pub byte_offset: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct EnvironmentDiagnostic {
+    pub reference: EnvironmentReference,
+    pub message: String,
+}
+
+#[derive(Debug, Error)]
+pub enum EnvironmentError {
+    #[error("failed to read environment resources under {path}: {source}")]
+    Read {
+        path: PathBuf,
+        #[source]
+        source: std::io::Error,
+    },
+}
+
+pub fn discover_environments(
+    project_root: &Path,
+) -> Result<Vec<EnvironmentResource>, EnvironmentError> {
+    let envs_dir = project_root.join("envs");
+    if !envs_dir.exists() {
+        return Ok(Vec::new());
+    }
+
+    let mut resources = Vec::new();
+    let entries = fs::read_dir(&envs_dir).map_err(|source| EnvironmentError::Read {
+        path: envs_dir.clone(),
+        source,
+    })?;
+
+    for entry in entries {
+        let entry = entry.map_err(|source| EnvironmentError::Read {
+            path: envs_dir.clone(),
+            source,
+        })?;
+        let path = entry.path();
+        if !path.is_dir() {
+            continue;
+        }
+        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
+            continue;
+        };
+
+        if let Some(resource) = discover_one(project_root, name, &path)? {
+            resources.push(resource);
+        }
+    }
+
+    resources.sort_by(|left, right| left.name.cmp(&right.name));
+    Ok(resources)
+}
+
+pub fn diagnose_environment_references(
+    source: &str,
+    environments: &[EnvironmentResource],
+) -> Vec<EnvironmentDiagnostic> {
+    let known = environments
+        .iter()
+        .map(|environment| environment.name.as_str())
+        .collect::<BTreeSet<_>>();
+
+    find_env_macro_references(source)
+        .into_iter()
+        .filter(|reference| !known.contains(reference.name.as_str()))
+        .map(|reference| EnvironmentDiagnostic {
+            message: format!(
+                "missing Clusterflux environment `{}`; expected envs/{}/Containerfile or envs/{}/Dockerfile",
+                reference.name, reference.name, reference.name
+            ),
+            reference,
+        })
+        .collect()
+}
+
+fn discover_one(
+    project_root: &Path,
+    name: &str,
+    env_dir: &Path,
+) -> Result<Option<EnvironmentResource>, EnvironmentError> {
+    let candidates = [
+        ("Containerfile", EnvironmentKind::Containerfile),
+        ("Dockerfile", EnvironmentKind::Dockerfile),
+        ("flake.nix", EnvironmentKind::NixFlake),
+    ];
+
+    for (file_name, kind) in candidates {
+        let recipe_path = env_dir.join(file_name);
+        if !recipe_path.exists() {
+            continue;
+        }
+
+        let recipe_bytes = fs::read(&recipe_path).map_err(|source| EnvironmentError::Read {
+            path: recipe_path.clone(),
+            source,
+        })?;
+        let relative_recipe = recipe_path
+            .strip_prefix(project_root)
+            .unwrap_or(&recipe_path)
+            .to_string_lossy();
+        let digest = Digest::from_parts([
+            b"environment:v1".as_slice(),
+            name.as_bytes(),
+            format!("{kind:?}").as_bytes(),
+            relative_recipe.as_bytes(),
+            recipe_bytes.as_slice(),
+        ]);
+        let requirements = match kind {
+            EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile
+                if name.eq_ignore_ascii_case("windows") =>
+            {
+                EnvironmentRequirements::windows_command_dev()
+            }
+            EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile => {
+                EnvironmentRequirements::linux_container()
+            }
+            EnvironmentKind::NixFlake => EnvironmentRequirements::unconstrained(),
+        };
+
+        return Ok(Some(EnvironmentResource {
+            name: name.to_owned(),
+            kind,
+            recipe_path,
+            context_path: env_dir.to_path_buf(),
+            digest,
+            requirements,
+        }));
+    }
+
+    Ok(None)
+}
+
+fn find_env_macro_references(source: &str) -> Vec<EnvironmentReference> {
+    let mut references = Vec::new();
+    let mut cursor = 0;
+
+    while let Some(index) = source[cursor..].find("env!(") {
+        let start = cursor + index;
+        let mut pos = start + "env!(".len();
+        while source[pos..].starts_with(char::is_whitespace) {
+            pos += source[pos..]
+                .chars()
+                .next()
+                .map(char::len_utf8)
+                .unwrap_or(1);
+        }
+        if !source[pos..].starts_with('"') {
+            cursor = pos;
+            continue;
+        }
+        pos += 1;
+        let name_start = pos;
+        while pos < source.len() && !source[pos..].starts_with('"') {
+            pos += source[pos..]
+                .chars()
+                .next()
+                .map(char::len_utf8)
+                .unwrap_or(1);
+        }
+        if pos < source.len() {
+            references.push(EnvironmentReference {
+                name: source[name_start..pos].to_owned(),
+                byte_offset: start,
+            });
+        }
+        cursor = pos.saturating_add(1);
+    }
+
+    references
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs;
+
+    use super::*;
+
+    #[test]
+    fn discovers_containerfile_environments_by_logical_name() {
+        let temp = tempfile::tempdir().unwrap();
+        let linux = temp.path().join("envs/linux");
+        fs::create_dir_all(&linux).unwrap();
+        fs::write(linux.join("Containerfile"), "FROM alpine\n").unwrap();
+
+        let envs = discover_environments(temp.path()).unwrap();
+
+        assert_eq!(envs.len(), 1);
+        assert_eq!(envs[0].name, "linux");
+        assert_eq!(envs[0].kind, EnvironmentKind::Containerfile);
+        assert!(!envs[0].digest.as_str().is_empty());
+    }
+
+    #[test]
+    fn missing_env_macro_reference_reports_clear_diagnostic() {
+        let source = r#"fn main() { let _ = env!("windows"); }"#;
+        let diagnostics = diagnose_environment_references(source, &[]);
+
+        assert_eq!(diagnostics.len(), 1);
+        assert!(diagnostics[0]
+            .message
+            .contains("envs/windows/Containerfile"));
+    }
+
+    #[test]
+    fn windows_environment_name_uses_windows_development_requirements() {
+        let temp = tempfile::tempdir().unwrap();
+        let windows = temp.path().join("envs/windows");
+        fs::create_dir_all(&windows).unwrap();
+        fs::write(
+            windows.join("Dockerfile"),
+            "# user-attached windows dev contract\n",
+        )
+        .unwrap();
+
+        let envs = discover_environments(temp.path()).unwrap();
+
+        assert_eq!(envs[0].name, "windows");
+        assert_eq!(envs[0].requirements.os, Some(Os::Windows));
+        assert!(envs[0]
+            .requirements
+            .capabilities
+            .contains(&Capability::WindowsCommandDev));
+    }
+}

crates/clusterflux-core/src/ids.rs

@@ -0,0 +1,45 @@
+use serde::{Deserialize, Serialize};
+
+macro_rules! id_type {
+    ($name:ident) => {
+        #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+        pub struct $name(String);
+
+        impl $name {
+            pub fn new(value: impl Into<String>) -> Self {
+                let value = value.into();
+                assert!(
+                    !value.trim().is_empty(),
+                    concat!(stringify!($name), " cannot be empty")
+                );
+                Self(value)
+            }
+
+            pub fn as_str(&self) -> &str {
+                &self.0
+            }
+        }
+
+        impl From<&str> for $name {
+            fn from(value: &str) -> Self {
+                Self::new(value)
+            }
+        }
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                f.write_str(&self.0)
+            }
+        }
+    };
+}
+
+id_type!(AgentId);
+id_type!(ArtifactId);
+id_type!(NodeId);
+id_type!(ProcessId);
+id_type!(ProjectId);
+id_type!(TaskDefinitionId);
+id_type!(TaskInstanceId);
+id_type!(TenantId);
+id_type!(UserId);

crates/clusterflux-core/src/lib.rs

@@ -0,0 +1,102 @@
+pub mod artifact;
+pub mod auth;
+pub mod bundle;
+pub mod capability;
+pub mod checkpoint;
+pub mod debug;
+pub mod digest;
+pub mod environment;
+pub mod execution;
+pub mod ids;
+pub mod limits;
+pub mod operator_panel;
+pub mod policy;
+pub mod project;
+pub mod scheduler;
+pub mod source;
+pub mod transport;
+pub mod vfs;
+pub mod wire;
+
+pub use artifact::{
+    ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
+    ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
+    DownloadStreamRequest, RetentionPolicy, StorageLocation,
+};
+pub use auth::{
+    admin_request_proof, admin_request_proof_from_token_digest,
+    agent_ed25519_public_key_from_private_key, agent_workflow_request_scope_from_payload,
+    derive_ed25519_private_key_from_seed, node_capability_policy_digest,
+    node_ed25519_public_key_from_private_key, sign_agent_workflow_request, sign_node_request,
+    signed_request_payload_digest, verify_agent_workflow_signature, verify_node_request_signature,
+    Action, Actor, AgentSignedRequest, AgentWorkflowRequestScope, AgentWorkflowScope, AuthContext,
+    Authorization, BrowserLoginFlow, CredentialKind, EnrollmentError, EnrollmentGrant,
+    IdentityKind, NodeCredential, NodeSignedRequest, PublicKeyIdentity, Scope,
+};
+#[cfg(not(target_arch = "wasm32"))]
+pub use auth::{generate_ed25519_private_key, generate_opaque_token};
+pub use bundle::{
+    discover_source_debug_probes, BundleDebugMetadata, BundleDebugProbe, BundleIdentityInputs,
+    BundleLargeInputPolicy, BundleMetadata, BundleRestartCompatibility, BundleSourceMetadata,
+    BundleTaskMetadata, SelectedInput,
+};
+pub use capability::{Capability, CapabilityReportError, EnvironmentBackend, NodeCapabilities, Os};
+pub use checkpoint::{
+    CheckpointBoundary, CompatibilityFailure, RestartDecision, RestartPolicy, RestartRequest,
+    TaskCheckpoint,
+};
+pub use debug::{
+    DebugEpoch, DebugEpochError, DebugParticipant, DebugParticipantKind, DebugRuntimeState,
+    DebugStopReason, ThreadInspection,
+};
+pub use digest::Digest;
+pub use environment::{
+    diagnose_environment_references, discover_environments, EnvironmentDiagnostic, EnvironmentKind,
+    EnvironmentReference, EnvironmentRequirements, EnvironmentResource,
+};
+pub use execution::{
+    CommandBackendKind, CommandInvocation, CommandNetworkPolicy, CommandPlan, GuestRuntimeKind,
+    NativeCommandPolicy, StructuredTaskBoundary, TaskBoundaryHandle, TaskBoundaryValue,
+    TaskDispatch, TaskFailurePolicy, TaskJoinResult, TaskJoinState, TaskSpec, WasmExportAbi,
+    WasmHostCommandRequest, WasmHostCommandResult, WasmHostDebugProbeRequest,
+    WasmHostDebugProbeResult, WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult,
+    WasmHostTaskControlRequest, WasmHostTaskControlResult, WasmHostTaskHandle,
+    WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
+    WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation,
+    WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
+};
+pub use ids::{
+    AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
+    UserId,
+};
+pub use limits::{
+    LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,
+    ResourceMeter, TaskArgumentBudget, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
+};
+pub use operator_panel::{
+    ControlPlaneAction, PanelError, PanelEvent, PanelEventKind, PanelState, PanelWidget,
+    PanelWidgetKind, RateLimit,
+};
+pub use policy::{
+    CapabilityPolicy, Decision, LocalTrustedPolicy, PolicyReason, ResourceRequest, ServicePolicy,
+};
+pub use project::{Entrypoint, ProjectModel, ProjectModelError};
+pub use scheduler::{
+    DefaultScheduler, NodeDescriptor, Placement, PlacementError, PlacementRequest, Scheduler,
+};
+pub use source::{
+    SourceManifestError, SourcePreparation, SourceProviderKind, SourceProviderManifest,
+    SourceProviderModule, SourceTransferMode, SourceTransferPolicy,
+};
+pub use transport::{
+    BulkTransferDecision, DataPlaneObject, DataPlaneScope, DirectBulkTransferPlan,
+    NativeQuicTransport, NodeEndpoint, RendezvousRequest, Transport, TransportError, TransportKind,
+};
+pub use vfs::{
+    ReuseDecision, SyncPolicy, VfsError, VfsManifest, VfsObject, VfsOverlay, VfsPath,
+    VfsSyncDecision,
+};
+pub use wire::{
+    coordinator_authentication_metadata, coordinator_payload_operation, coordinator_wire_request,
+    COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE,
+};

crates/clusterflux-core/src/limits.rs

@@ -0,0 +1,300 @@
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::TaskInstanceId;
+
+/// Fastest supported interval for a node's signed artifact/assignment polling loop.
+/// The coordinator's bounded replay window is sized against this protocol limit.
+pub const MIN_SIGNED_NODE_POLL_INTERVAL_MS: u64 = 20;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum LimitKind {
+    ApiCall,
+    Spawn,
+    LogBytes,
+    MetadataBytes,
+    DebugReadBytes,
+    UiEvent,
+    RendezvousAttempt,
+    ArtifactDownloadBytes,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResourceLimits {
+    pub limits: BTreeMap<LimitKind, u64>,
+}
+
+impl ResourceLimits {
+    pub fn new(limits: impl IntoIterator<Item = (LimitKind, u64)>) -> Self {
+        Self {
+            limits: limits.into_iter().collect(),
+        }
+    }
+
+    pub fn unlimited() -> Self {
+        Self::new(LimitKind::ALL.into_iter().map(|kind| (kind, u64::MAX)))
+    }
+
+    pub fn limit(&self, kind: &LimitKind) -> u64 {
+        *self.limits.get(kind).unwrap_or(&0)
+    }
+}
+
+impl Default for ResourceLimits {
+    fn default() -> Self {
+        Self::unlimited()
+    }
+}
+
+impl LimitKind {
+    pub const ALL: [Self; 8] = [
+        Self::ApiCall,
+        Self::Spawn,
+        Self::LogBytes,
+        Self::MetadataBytes,
+        Self::DebugReadBytes,
+        Self::UiEvent,
+        Self::RendezvousAttempt,
+        Self::ArtifactDownloadBytes,
+    ];
+}
+
+pub const TASK_JOIN_TIMEOUT_SECONDS_ENV: &str = "CLUSTERFLUX_TASK_JOIN_TIMEOUT_SECONDS";
+pub const DEFAULT_TASK_JOIN_TIMEOUT_SECONDS: u64 = 24 * 60 * 60;
+
+pub fn task_join_timeout() -> Duration {
+    std::env::var(TASK_JOIN_TIMEOUT_SECONDS_ENV)
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        .filter(|seconds| *seconds > 0)
+        .map(Duration::from_secs)
+        .unwrap_or_else(|| Duration::from_secs(DEFAULT_TASK_JOIN_TIMEOUT_SECONDS))
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum TaskJoinError {
+    #[error(
+        "timed out after {waited_seconds} seconds waiting for task {task}; the child task was left running"
+    )]
+    Timeout {
+        task: crate::TaskInstanceId,
+        waited_seconds: u64,
+    },
+    #[error("task join for {task} was cancelled; the child task was left running")]
+    Cancelled { task: crate::TaskInstanceId },
+}
+
+impl TaskJoinError {
+    pub fn timeout(task: crate::TaskInstanceId, waited: Duration) -> Self {
+        Self::Timeout {
+            task,
+            waited_seconds: waited.as_secs(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResourceMeter {
+    used: BTreeMap<LimitKind, u64>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum LimitError {
+    #[error(
+        "resource limit exceeded for {kind:?}: requested {requested}, used {used}, limit {limit}"
+    )]
+    Exceeded {
+        kind: LimitKind,
+        requested: u64,
+        used: u64,
+        limit: u64,
+    },
+    #[error("task argument is too large: {size} bytes exceeds {limit} bytes")]
+    LargeTaskArgument { size: u64, limit: u64 },
+}
+
+impl ResourceMeter {
+    pub fn can_charge(
+        &self,
+        limits: &ResourceLimits,
+        kind: LimitKind,
+        amount: u64,
+    ) -> Result<(), LimitError> {
+        let used = self.used.get(&kind).copied().unwrap_or(0);
+        let limit = limits.limit(&kind);
+        if used.saturating_add(amount) > limit {
+            return Err(LimitError::Exceeded {
+                kind,
+                requested: amount,
+                used,
+                limit,
+            });
+        }
+        Ok(())
+    }
+
+    pub fn charge(
+        &mut self,
+        limits: &ResourceLimits,
+        kind: LimitKind,
+        amount: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(limits, kind, amount)?;
+        let used = self.used.get(&kind).copied().unwrap_or(0);
+        self.used.insert(kind, used + amount);
+        Ok(())
+    }
+
+    pub fn used(&self, kind: &LimitKind) -> u64 {
+        self.used.get(kind).copied().unwrap_or(0)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct LogRecord {
+    pub task: TaskInstanceId,
+    pub bytes: Vec<u8>,
+    pub truncated: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct LogBuffer {
+    max_bytes: usize,
+    used_bytes: usize,
+    records: Vec<LogRecord>,
+    backpressured: bool,
+}
+
+impl LogBuffer {
+    pub fn new(max_bytes: usize) -> Self {
+        Self {
+            max_bytes,
+            used_bytes: 0,
+            records: Vec::new(),
+            backpressured: false,
+        }
+    }
+
+    pub fn push(&mut self, task: TaskInstanceId, bytes: impl AsRef<[u8]>) {
+        let bytes = bytes.as_ref();
+        let remaining = self.max_bytes.saturating_sub(self.used_bytes);
+        let truncated = bytes.len() > remaining;
+        let stored = bytes[..bytes.len().min(remaining)].to_vec();
+        self.used_bytes += stored.len();
+        if truncated {
+            self.backpressured = true;
+        }
+        self.records.push(LogRecord {
+            task,
+            bytes: stored,
+            truncated,
+        });
+    }
+
+    pub fn records(&self) -> &[LogRecord] {
+        &self.records
+    }
+
+    pub fn backpressured(&self) -> bool {
+        self.backpressured
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum LargeArgumentPolicy {
+    Allow,
+    Warn,
+    Reject,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskArgumentBudget {
+    pub max_inline_bytes: u64,
+    pub policy: LargeArgumentPolicy,
+}
+
+impl TaskArgumentBudget {
+    pub fn validate(&self, size: u64) -> Result<Option<String>, LimitError> {
+        if size <= self.max_inline_bytes {
+            return Ok(None);
+        }
+
+        match self.policy {
+            LargeArgumentPolicy::Allow => Ok(None),
+            LargeArgumentPolicy::Warn => Ok(Some(format!(
+                "task argument is {size} bytes; prefer SourceSnapshot, Blob, Artifact, or VFS handles"
+            ))),
+            LargeArgumentPolicy::Reject => Err(LimitError::LargeTaskArgument {
+                size,
+                limit: self.max_inline_bytes,
+            }),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resource_meter_rejects_usage_before_work_starts() {
+        let limits = ResourceLimits {
+            limits: BTreeMap::from([(LimitKind::Spawn, 1)]),
+        };
+        let mut meter = ResourceMeter::default();
+
+        meter.charge(&limits, LimitKind::Spawn, 1).unwrap();
+        let error = meter.charge(&limits, LimitKind::Spawn, 1).unwrap_err();
+
+        assert!(matches!(error, LimitError::Exceeded { .. }));
+    }
+
+    #[test]
+    fn resource_meter_can_check_limits_without_consuming() {
+        let limits = ResourceLimits {
+            limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 4)]),
+        };
+        let mut meter = ResourceMeter::default();
+
+        meter
+            .can_charge(&limits, LimitKind::ArtifactDownloadBytes, 4)
+            .unwrap();
+        assert_eq!(meter.used(&LimitKind::ArtifactDownloadBytes), 0);
+        meter
+            .charge(&limits, LimitKind::ArtifactDownloadBytes, 3)
+            .unwrap();
+        assert!(matches!(
+            meter.can_charge(&limits, LimitKind::ArtifactDownloadBytes, 2),
+            Err(LimitError::Exceeded { .. })
+        ));
+    }
+
+    #[test]
+    fn log_buffer_caps_backpressures_and_keeps_task_association() {
+        let mut logs = LogBuffer::new(4);
+        logs.push(TaskInstanceId::from("task-a"), b"abcdef");
+
+        assert!(logs.backpressured());
+        assert_eq!(logs.records()[0].task, TaskInstanceId::from("task-a"));
+        assert_eq!(logs.records()[0].bytes, b"abcd");
+        assert!(logs.records()[0].truncated);
+    }
+
+    #[test]
+    fn large_task_arguments_are_rejected_or_warned() {
+        let reject = TaskArgumentBudget {
+            max_inline_bytes: 4,
+            policy: LargeArgumentPolicy::Reject,
+        };
+        assert!(reject.validate(5).is_err());
+
+        let warn = TaskArgumentBudget {
+            max_inline_bytes: 4,
+            policy: LargeArgumentPolicy::Warn,
+        };
+        assert!(warn.validate(5).unwrap().unwrap().contains("Artifact"));
+    }
+}

crates/clusterflux-core/src/operator_panel.rs

@@ -0,0 +1,377 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{
+    ArtifactId, DownloadAction, DownloadError, ProcessId, ProjectId, TaskInstanceId, TenantId,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum PanelWidgetKind {
+    Text {
+        value: String,
+    },
+    Progress {
+        current: u64,
+        total: u64,
+    },
+    Button {
+        action: String,
+    },
+    Toggle {
+        value: bool,
+    },
+    Select {
+        options: Vec<String>,
+        selected: String,
+    },
+    ArtifactDownload {
+        artifact: ArtifactId,
+    },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PanelWidget {
+    pub id: String,
+    pub label: String,
+    pub kind: PanelWidgetKind,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ControlPlaneAction {
+    RestartTask(TaskInstanceId),
+    CancelProcess,
+    DebugProcess,
+    DownloadArtifact(ArtifactId),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PanelState {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub widgets: BTreeMap<String, PanelWidget>,
+    pub program_ui_events_enabled: bool,
+    pub control_plane_actions: Vec<ControlPlaneAction>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum PanelEventKind {
+    ButtonClicked,
+    ToggleChanged(bool),
+    SelectChanged(String),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PanelEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub widget_id: String,
+    pub kind: PanelEventKind,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RateLimit {
+    pub max_events: u64,
+    pub used_events: u64,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum PanelError {
+    #[error("custom HTML or JavaScript is not supported in operator panels")]
+    CustomContentDenied,
+    #[error(
+        "operator panel widget `{0}` is not allowed to collect secrets or OAuth-like credentials"
+    )]
+    CredentialCollectionDenied(String),
+    #[error("panel event scope does not match tenant/project/process")]
+    ScopeMismatch,
+    #[error("program UI events are disabled while debug process is stopped")]
+    ProgramEventsDisabled,
+    #[error("panel event rate limit exceeded")]
+    RateLimited,
+    #[error("unknown panel widget `{0}`")]
+    UnknownWidget(String),
+    #[error("artifact download action is unavailable: {0}")]
+    DownloadUnavailable(String),
+}
+
+impl PanelState {
+    pub fn new(tenant: TenantId, project: ProjectId, process: ProcessId) -> Self {
+        Self {
+            tenant,
+            project,
+            process,
+            widgets: BTreeMap::new(),
+            program_ui_events_enabled: true,
+            control_plane_actions: Vec::new(),
+        }
+    }
+
+    pub fn add_widget(&mut self, widget: PanelWidget) -> Result<(), PanelError> {
+        validate_widget(&widget)?;
+        self.widgets.insert(widget.id.clone(), widget);
+        Ok(())
+    }
+
+    pub fn add_download_widget_from_action(
+        &mut self,
+        widget_id: impl Into<String>,
+        label: impl Into<String>,
+        action: Result<DownloadAction, DownloadError>,
+    ) -> Result<(), PanelError> {
+        let action = action.map_err(|err| PanelError::DownloadUnavailable(err.to_string()))?;
+        let artifact = action.artifact;
+        self.add_widget(PanelWidget {
+            id: widget_id.into(),
+            label: label.into(),
+            kind: PanelWidgetKind::ArtifactDownload {
+                artifact: artifact.clone(),
+            },
+        })?;
+        self.control_plane_actions
+            .push(ControlPlaneAction::DownloadArtifact(artifact));
+        Ok(())
+    }
+
+    pub fn reject_custom_content(_html_or_js: &str) -> Result<(), PanelError> {
+        Err(PanelError::CustomContentDenied)
+    }
+
+    pub fn freeze_program_ui_events(&mut self) {
+        self.program_ui_events_enabled = false;
+    }
+
+    pub fn set_control_plane_actions(&mut self, actions: Vec<ControlPlaneAction>) {
+        self.control_plane_actions = actions;
+    }
+
+    pub fn accept_event(
+        &self,
+        event: &PanelEvent,
+        limit: &mut RateLimit,
+    ) -> Result<(), PanelError> {
+        if !self.program_ui_events_enabled {
+            return Err(PanelError::ProgramEventsDisabled);
+        }
+        if self.tenant != event.tenant
+            || self.project != event.project
+            || self.process != event.process
+        {
+            return Err(PanelError::ScopeMismatch);
+        }
+        if !self.widgets.contains_key(&event.widget_id) {
+            return Err(PanelError::UnknownWidget(event.widget_id.clone()));
+        }
+        if limit.used_events >= limit.max_events {
+            return Err(PanelError::RateLimited);
+        }
+        limit.used_events += 1;
+        Ok(())
+    }
+
+    pub fn control_plane_actions_available(&self) -> &[ControlPlaneAction] {
+        &self.control_plane_actions
+    }
+}
+
+fn validate_widget(widget: &PanelWidget) -> Result<(), PanelError> {
+    let mut checked_text = vec![widget.id.as_str(), widget.label.as_str()];
+    match &widget.kind {
+        PanelWidgetKind::Button { action } => checked_text.push(action),
+        PanelWidgetKind::Select { options, selected } => {
+            checked_text.push(selected);
+            checked_text.extend(options.iter().map(String::as_str));
+        }
+        PanelWidgetKind::Text { .. }
+        | PanelWidgetKind::Progress { .. }
+        | PanelWidgetKind::Toggle { .. }
+        | PanelWidgetKind::ArtifactDownload { .. } => {}
+    }
+
+    let combined = checked_text.join(" ").to_ascii_lowercase();
+    if combined.contains("password")
+        || combined.contains("token")
+        || combined.contains("oauth")
+        || combined.contains("secret")
+    {
+        return Err(PanelError::CredentialCollectionDenied(widget.id.clone()));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn panel() -> PanelState {
+        PanelState::new(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            ProcessId::from("process"),
+        )
+    }
+
+    #[test]
+    fn panel_uses_typed_widgets_and_rejects_custom_content() {
+        let mut panel = panel();
+        panel
+            .add_widget(PanelWidget {
+                id: "progress".to_owned(),
+                label: "Build".to_owned(),
+                kind: PanelWidgetKind::Progress {
+                    current: 1,
+                    total: 2,
+                },
+            })
+            .unwrap();
+
+        assert!(PanelState::reject_custom_content("<script>alert(1)</script>").is_err());
+        assert!(panel.widgets.contains_key("progress"));
+    }
+
+    #[test]
+    fn panel_rejects_password_or_oauth_collection_widgets() {
+        let mut panel = panel();
+        let error = panel
+            .add_widget(PanelWidget {
+                id: "oauth_token".to_owned(),
+                label: "OAuth Token".to_owned(),
+                kind: PanelWidgetKind::Text {
+                    value: String::new(),
+                },
+            })
+            .unwrap_err();
+
+        assert!(matches!(error, PanelError::CredentialCollectionDenied(_)));
+    }
+
+    #[test]
+    fn panel_rejects_credential_collection_in_interactive_fields() {
+        let mut panel = panel();
+        let button_error = panel
+            .add_widget(PanelWidget {
+                id: "continue".to_owned(),
+                label: "Continue".to_owned(),
+                kind: PanelWidgetKind::Button {
+                    action: "collect-secret".to_owned(),
+                },
+            })
+            .unwrap_err();
+        assert!(matches!(
+            button_error,
+            PanelError::CredentialCollectionDenied(_)
+        ));
+
+        let select_error = panel
+            .add_widget(PanelWidget {
+                id: "auth-mode".to_owned(),
+                label: "Auth Mode".to_owned(),
+                kind: PanelWidgetKind::Select {
+                    options: vec!["password".to_owned(), "public key".to_owned()],
+                    selected: "public key".to_owned(),
+                },
+            })
+            .unwrap_err();
+        assert!(matches!(
+            select_error,
+            PanelError::CredentialCollectionDenied(_)
+        ));
+    }
+
+    #[test]
+    fn panel_events_are_scoped_and_rate_limited() {
+        let mut panel = panel();
+        panel
+            .add_widget(PanelWidget {
+                id: "restart".to_owned(),
+                label: "Restart".to_owned(),
+                kind: PanelWidgetKind::Button {
+                    action: "restart".to_owned(),
+                },
+            })
+            .unwrap();
+        let event = PanelEvent {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            process: ProcessId::from("process"),
+            widget_id: "restart".to_owned(),
+            kind: PanelEventKind::ButtonClicked,
+        };
+        let mut limit = RateLimit {
+            max_events: 1,
+            used_events: 0,
+        };
+
+        panel.accept_event(&event, &mut limit).unwrap();
+        assert_eq!(
+            panel.accept_event(&event, &mut limit),
+            Err(PanelError::RateLimited)
+        );
+    }
+
+    #[test]
+    fn stopped_debug_process_keeps_control_plane_actions_available() {
+        let mut panel = panel();
+        panel.freeze_program_ui_events();
+        panel.set_control_plane_actions(vec![
+            ControlPlaneAction::RestartTask(TaskInstanceId::from("task")),
+            ControlPlaneAction::DownloadArtifact(ArtifactId::from("artifact")),
+        ]);
+
+        let event = PanelEvent {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            process: ProcessId::from("process"),
+            widget_id: "missing".to_owned(),
+            kind: PanelEventKind::ButtonClicked,
+        };
+        let mut limit = RateLimit {
+            max_events: 1,
+            used_events: 0,
+        };
+
+        assert_eq!(
+            panel.accept_event(&event, &mut limit),
+            Err(PanelError::ProgramEventsDisabled)
+        );
+        assert_eq!(panel.control_plane_actions_available().len(), 2);
+    }
+
+    #[test]
+    fn download_widget_is_only_created_from_available_action() {
+        let mut panel = panel();
+        let action = Ok(DownloadAction {
+            artifact: ArtifactId::from("artifact"),
+            source: crate::StorageLocation::RetainedNode(crate::NodeId::from("node")),
+            scoped_token_subject: "tenant/project/process/artifact".to_owned(),
+        });
+
+        panel
+            .add_download_widget_from_action("download-artifact", "Download", action)
+            .unwrap();
+
+        assert!(matches!(
+            panel.widgets["download-artifact"].kind,
+            PanelWidgetKind::ArtifactDownload { .. }
+        ));
+        assert!(matches!(
+            panel.control_plane_actions_available()[0],
+            ControlPlaneAction::DownloadArtifact(_)
+        ));
+
+        let before = panel.widgets.len();
+        let error = panel
+            .add_download_widget_from_action(
+                "missing-download",
+                "Download",
+                Err(DownloadError::Unavailable),
+            )
+            .unwrap_err();
+
+        assert_eq!(panel.widgets.len(), before);
+        assert!(matches!(error, PanelError::DownloadUnavailable(_)));
+    }
+}

crates/clusterflux-core/src/policy.rs

@@ -0,0 +1,134 @@
+use std::collections::BTreeSet;
+
+use serde::{Deserialize, Serialize};
+
+use crate::{Action, AuthContext, Capability, Scope};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum PolicyReason {
+    Allowed,
+    MissingCapability(Capability),
+    HostedNativeComputeDenied,
+    HostedContainerDenied,
+    QuotaExceeded(String),
+    Unauthorized(String),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Decision {
+    pub allowed: bool,
+    pub reason: PolicyReason,
+}
+
+impl Decision {
+    pub fn allow() -> Self {
+        Self {
+            allowed: true,
+            reason: PolicyReason::Allowed,
+        }
+    }
+
+    pub fn deny(reason: PolicyReason) -> Self {
+        Self {
+            allowed: false,
+            reason,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ResourceRequest {
+    pub action: Action,
+    pub required_capabilities: BTreeSet<Capability>,
+    pub hosted_control_plane: bool,
+}
+
+pub trait CapabilityPolicy {
+    fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision;
+}
+
+pub trait ServicePolicy: CapabilityPolicy + Send + Sync {}
+
+impl<T> ServicePolicy for T where T: CapabilityPolicy + Send + Sync {}
+
+#[derive(Clone, Debug, Default)]
+pub struct LocalTrustedPolicy;
+
+impl CapabilityPolicy for LocalTrustedPolicy {
+    fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision {
+        let authz = crate::auth::same_tenant_project(context, scope);
+        if !authz.allowed {
+            return Decision::deny(PolicyReason::Unauthorized(authz.reason));
+        }
+        if request.hosted_control_plane && request.action == Action::RunNativeCommand {
+            return Decision::deny(PolicyReason::HostedNativeComputeDenied);
+        }
+        if request.hosted_control_plane && request.action == Action::RunContainer {
+            return Decision::deny(PolicyReason::HostedContainerDenied);
+        }
+        Decision::allow()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{Actor, ProjectId, TenantId, UserId};
+
+    use super::*;
+
+    #[test]
+    fn public_policy_interface_denies_hosted_native_compute() {
+        let policy = LocalTrustedPolicy;
+        let context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let scope = Scope {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            process: None,
+            task: None,
+            node: None,
+            artifact: None,
+        };
+        let request = ResourceRequest {
+            action: Action::RunNativeCommand,
+            required_capabilities: BTreeSet::new(),
+            hosted_control_plane: true,
+        };
+
+        let decision = policy.decide(&context, &scope, &request);
+
+        assert!(!decision.allowed);
+        assert_eq!(decision.reason, PolicyReason::HostedNativeComputeDenied);
+    }
+
+    #[test]
+    fn local_trusted_policy_allows_owner_controlled_native_capability_request() {
+        let policy = LocalTrustedPolicy;
+        let context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::User(UserId::from("owner")),
+        };
+        let scope = Scope {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            process: None,
+            task: None,
+            node: None,
+            artifact: None,
+        };
+        let request = ResourceRequest {
+            action: Action::RunNativeCommand,
+            required_capabilities: BTreeSet::from([Capability::Command]),
+            hosted_control_plane: false,
+        };
+
+        let decision = policy.decide(&context, &scope, &request);
+
+        assert!(decision.allowed);
+        assert_eq!(decision.reason, PolicyReason::Allowed);
+    }
+}

crates/clusterflux-core/src/project.rs

@@ -0,0 +1,303 @@
+use std::collections::BTreeMap;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use syn::{punctuated::Punctuated, Expr, Item, Lit, Meta, Token};
+use thiserror::Error;
+
+use crate::{discover_environments, environment::EnvironmentError, EnvironmentResource};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Entrypoint {
+    pub name: String,
+    pub function: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProjectModel {
+    pub root: PathBuf,
+    pub environments: Vec<EnvironmentResource>,
+    pub entrypoints: BTreeMap<String, Entrypoint>,
+    pub default_entrypoint: String,
+    pub required_config_file: Option<PathBuf>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ProjectModelError {
+    #[error("environment discovery failed: {0}")]
+    Environment(String),
+    #[error("Clusterflux entrypoint discovery failed: {0}")]
+    EntrypointDiscovery(String),
+    #[error(
+        "no Clusterflux entrypoint is declared; add `#[clusterflux::main]` to a function under src/"
+    )]
+    NoEntrypoints,
+    #[error("unknown Clusterflux entrypoint `{name}`; available entrypoints: {available:?}")]
+    UnknownEntrypoint {
+        name: String,
+        available: Vec<String>,
+    },
+}
+
+impl ProjectModel {
+    pub fn discover_without_config(root: &Path) -> Result<Self, ProjectModelError> {
+        let environments = discover_environments(root).map_err(|err| {
+            ProjectModelError::Environment(match err {
+                EnvironmentError::Read { path, source } => {
+                    format!("failed to read {}: {source}", path.display())
+                }
+            })
+        })?;
+        let entrypoints = discover_entrypoints(root)?;
+        let default_entrypoint = if entrypoints.contains_key("build") {
+            "build".to_owned()
+        } else {
+            entrypoints.keys().next().cloned().unwrap_or_default()
+        };
+        Ok(Self {
+            root: root.to_path_buf(),
+            environments,
+            entrypoints,
+            default_entrypoint,
+            required_config_file: None,
+        })
+    }
+
+    pub fn select_entrypoint(&self, name: Option<&str>) -> Result<&Entrypoint, ProjectModelError> {
+        if self.entrypoints.is_empty() {
+            return Err(ProjectModelError::NoEntrypoints);
+        }
+        let name = name.unwrap_or(&self.default_entrypoint);
+        self.entrypoints
+            .get(name)
+            .ok_or_else(|| ProjectModelError::UnknownEntrypoint {
+                name: name.to_owned(),
+                available: self.entrypoints.keys().cloned().collect(),
+            })
+    }
+}
+
+fn discover_entrypoints(root: &Path) -> Result<BTreeMap<String, Entrypoint>, ProjectModelError> {
+    let source_root = root.join("src");
+    if !source_root.is_dir() {
+        return Ok(BTreeMap::new());
+    }
+
+    let mut source_files = Vec::new();
+    collect_rust_sources(&source_root, &mut source_files)?;
+    source_files.sort();
+
+    let mut entrypoints = BTreeMap::new();
+    for path in source_files {
+        let source = fs::read_to_string(&path).map_err(|error| {
+            ProjectModelError::EntrypointDiscovery(format!(
+                "failed to read {}: {error}",
+                path.display()
+            ))
+        })?;
+        let syntax = syn::parse_file(&source).map_err(|error| {
+            ProjectModelError::EntrypointDiscovery(format!(
+                "failed to parse {}: {error}",
+                path.display()
+            ))
+        })?;
+        collect_entrypoint_items(&syntax.items, &path, &mut entrypoints)?;
+    }
+    Ok(entrypoints)
+}
+
+fn collect_rust_sources(
+    directory: &Path,
+    files: &mut Vec<PathBuf>,
+) -> Result<(), ProjectModelError> {
+    let entries = fs::read_dir(directory).map_err(|error| {
+        ProjectModelError::EntrypointDiscovery(format!(
+            "failed to read {}: {error}",
+            directory.display()
+        ))
+    })?;
+    for entry in entries {
+        let entry = entry.map_err(|error| {
+            ProjectModelError::EntrypointDiscovery(format!(
+                "failed to inspect {}: {error}",
+                directory.display()
+            ))
+        })?;
+        let path = entry.path();
+        let file_type = entry.file_type().map_err(|error| {
+            ProjectModelError::EntrypointDiscovery(format!(
+                "failed to inspect {}: {error}",
+                path.display()
+            ))
+        })?;
+        if file_type.is_dir() {
+            collect_rust_sources(&path, files)?;
+        } else if file_type.is_file() && path.extension().is_some_and(|ext| ext == "rs") {
+            files.push(path);
+        }
+    }
+    Ok(())
+}
+
+fn collect_entrypoint_items(
+    items: &[Item],
+    path: &Path,
+    entrypoints: &mut BTreeMap<String, Entrypoint>,
+) -> Result<(), ProjectModelError> {
+    for item in items {
+        match item {
+            Item::Fn(function) => {
+                let Some(attribute) = function.attrs.iter().find(|attribute| {
+                    let segments = attribute
+                        .path()
+                        .segments
+                        .iter()
+                        .map(|segment| segment.ident.to_string())
+                        .collect::<Vec<_>>();
+                    segments.as_slice() == ["clusterflux", "main"]
+                }) else {
+                    continue;
+                };
+                let function_name = function.sig.ident.to_string();
+                let default_name = function_name
+                    .strip_suffix("_main")
+                    .unwrap_or(&function_name);
+                let name = entrypoint_name(attribute, default_name);
+                let entrypoint = Entrypoint {
+                    name: name.clone(),
+                    function: function_name,
+                };
+                if let Some(existing) = entrypoints.insert(name.clone(), entrypoint) {
+                    return Err(ProjectModelError::EntrypointDiscovery(format!(
+                        "duplicate entrypoint `{name}` in {}; it was already declared by `{}`",
+                        path.display(),
+                        existing.function
+                    )));
+                }
+            }
+            Item::Mod(module) => {
+                if let Some((_, nested)) = &module.content {
+                    collect_entrypoint_items(nested, path, entrypoints)?;
+                }
+            }
+            _ => {}
+        }
+    }
+    Ok(())
+}
+
+fn entrypoint_name(attribute: &syn::Attribute, default: &str) -> String {
+    let Meta::List(_) = &attribute.meta else {
+        return default.to_owned();
+    };
+    let Ok(arguments) = attribute.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)
+    else {
+        return default.to_owned();
+    };
+    arguments
+        .into_iter()
+        .find_map(|meta| {
+            let Meta::NameValue(name_value) = meta else {
+                return None;
+            };
+            if !name_value.path.is_ident("name") {
+                return None;
+            }
+            let Expr::Lit(expression) = name_value.value else {
+                return None;
+            };
+            let Lit::Str(value) = expression.lit else {
+                return None;
+            };
+            Some(value.value())
+        })
+        .unwrap_or_else(|| default.to_owned())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs;
+
+    use super::*;
+
+    #[test]
+    fn project_works_without_hand_written_configuration_file() {
+        let temp = tempfile::tempdir().unwrap();
+        fs::create_dir_all(temp.path().join("envs/linux")).unwrap();
+        fs::create_dir_all(temp.path().join("src")).unwrap();
+        fs::write(
+            temp.path().join("envs/linux/Containerfile"),
+            "FROM alpine\n",
+        )
+        .unwrap();
+        fs::write(
+            temp.path().join("src/main.rs"),
+            "#[clusterflux::main]\npub fn build_main() {}\n",
+        )
+        .unwrap();
+
+        let model = ProjectModel::discover_without_config(temp.path()).unwrap();
+
+        assert_eq!(model.required_config_file, None);
+        assert_eq!(model.environments[0].name, "linux");
+        assert_eq!(model.select_entrypoint(None).unwrap().name, "build");
+    }
+
+    #[test]
+    fn project_can_define_multiple_default_entrypoints() {
+        let temp = tempfile::tempdir().unwrap();
+        fs::create_dir_all(temp.path().join("src/nested")).unwrap();
+        fs::write(
+            temp.path().join("src/lib.rs"),
+            "#[clusterflux::main(name = \"check\")]\npub fn test_main() {}\n",
+        )
+        .unwrap();
+        fs::write(
+            temp.path().join("src/nested/release.rs"),
+            "#[clusterflux::main]\npub fn release_main() {}\n",
+        )
+        .unwrap();
+        let model = ProjectModel::discover_without_config(temp.path()).unwrap();
+
+        assert_eq!(
+            model.select_entrypoint(Some("check")).unwrap().function,
+            "test_main"
+        );
+        assert_eq!(
+            model.select_entrypoint(Some("release")).unwrap().function,
+            "release_main"
+        );
+    }
+
+    #[test]
+    fn unknown_entrypoint_lists_available_choices() {
+        let temp = tempfile::tempdir().unwrap();
+        fs::create_dir_all(temp.path().join("src")).unwrap();
+        fs::write(
+            temp.path().join("src/main.rs"),
+            "#[clusterflux::main]\npub fn build_main() {}\n",
+        )
+        .unwrap();
+        let model = ProjectModel::discover_without_config(temp.path()).unwrap();
+        let error = model.select_entrypoint(Some("deploy")).unwrap_err();
+
+        assert!(matches!(error, ProjectModelError::UnknownEntrypoint { .. }));
+    }
+
+    #[test]
+    fn project_without_declared_entrypoint_does_not_invent_product_surfaces() {
+        let temp = tempfile::tempdir().unwrap();
+        fs::create_dir_all(temp.path().join("src")).unwrap();
+        fs::write(temp.path().join("src/main.rs"), "fn main() {}\n").unwrap();
+
+        let model = ProjectModel::discover_without_config(temp.path()).unwrap();
+
+        assert!(model.entrypoints.is_empty());
+        assert_eq!(model.default_entrypoint, "");
+        assert_eq!(
+            model.select_entrypoint(None).unwrap_err(),
+            ProjectModelError::NoEntrypoints
+        );
+    }
+}

crates/clusterflux-core/src/scheduler.rs

@@ -0,0 +1,472 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{
+    ArtifactId, Capability, Digest, EnvironmentRequirements, NodeCapabilities, NodeId, ProjectId,
+    TenantId,
+};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeDescriptor {
+    pub id: NodeId,
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub capabilities: NodeCapabilities,
+    pub cached_environments: BTreeSet<Digest>,
+    pub dependency_caches: BTreeSet<Digest>,
+    pub source_snapshots: BTreeSet<Digest>,
+    pub artifact_locations: BTreeSet<ArtifactId>,
+    pub direct_connectivity: bool,
+    pub online: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PlacementRequest {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub environment: Option<EnvironmentRequirements>,
+    pub environment_digest: Option<Digest>,
+    #[serde(default)]
+    pub environment_cache_required: bool,
+    pub required_capabilities: BTreeSet<Capability>,
+    pub dependency_cache: Option<Digest>,
+    pub source_snapshot: Option<Digest>,
+    pub required_artifacts: BTreeSet<ArtifactId>,
+    pub quota_available: bool,
+    pub policy_allowed: bool,
+    pub prefer_node: Option<NodeId>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Placement {
+    pub node: NodeId,
+    pub score: i64,
+    pub reasons: Vec<String>,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+#[error("no capable node for placement: {message}")]
+pub struct PlacementError {
+    pub message: String,
+}
+
+pub trait Scheduler {
+    fn place(
+        &self,
+        nodes: &[NodeDescriptor],
+        request: &PlacementRequest,
+    ) -> Result<Placement, PlacementError>;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct DefaultScheduler;
+
+impl Scheduler for DefaultScheduler {
+    fn place(
+        &self,
+        nodes: &[NodeDescriptor],
+        request: &PlacementRequest,
+    ) -> Result<Placement, PlacementError> {
+        let mut scored = Vec::new();
+        let mut rejection_counts = BTreeMap::<String, usize>::new();
+
+        for node in nodes {
+            match compatibility(node, request) {
+                Ok(mut placement) => {
+                    locality_score(node, request, &mut placement);
+                    scored.push(placement);
+                }
+                Err(reasons) => {
+                    for reason in reasons {
+                        *rejection_counts.entry(reason).or_default() += 1;
+                    }
+                }
+            }
+        }
+
+        scored
+            .into_iter()
+            .max_by_key(|placement| placement.score)
+            .ok_or_else(|| PlacementError {
+                message: rejection_counts
+                    .into_iter()
+                    .map(|(reason, count)| format!("{reason} ({count} node(s))"))
+                    .collect::<Vec<_>>()
+                    .join("; "),
+            })
+    }
+}
+
+fn compatibility(
+    node: &NodeDescriptor,
+    request: &PlacementRequest,
+) -> Result<Placement, Vec<String>> {
+    let mut reasons = Vec::new();
+    if !node.online {
+        reasons.push("node offline".to_owned());
+    }
+    if node.tenant != request.tenant {
+        reasons.push("tenant mismatch".to_owned());
+    }
+    if node.project != request.project {
+        reasons.push("project mismatch".to_owned());
+    }
+    if !request.quota_available {
+        reasons.push("quota unavailable for placement".to_owned());
+    }
+    if !request.policy_allowed {
+        reasons.push("policy denied placement".to_owned());
+    }
+    for capability in &request.required_capabilities {
+        if !node.capabilities.capabilities.contains(capability) {
+            reasons.push(format!("missing capability {capability:?}"));
+        }
+    }
+    if let Some(environment) = &request.environment {
+        if let Some(required_os) = &environment.os {
+            if &node.capabilities.os != required_os {
+                reasons.push(format!("environment requires os {required_os:?}"));
+            }
+        }
+        if let Some(required_arch) = &environment.arch {
+            if &node.capabilities.arch != required_arch {
+                reasons.push(format!("environment requires arch {required_arch}"));
+            }
+        }
+        for capability in &environment.capabilities {
+            if !node.capabilities.capabilities.contains(capability) {
+                reasons.push(format!("environment requires capability {capability:?}"));
+            }
+        }
+    }
+    if request.environment_cache_required {
+        match request.environment_digest.as_ref() {
+            Some(digest) if !node.cached_environments.contains(digest) => {
+                reasons.push(format!(
+                    "required named environment cache {digest} is unavailable"
+                ));
+            }
+            None => reasons.push("required named environment cache digest is missing".to_owned()),
+            Some(_) => {}
+        }
+    }
+    let source_transfer_required = request
+        .source_snapshot
+        .as_ref()
+        .is_some_and(|digest| !node.source_snapshots.contains(digest));
+    if source_transfer_required && !node.direct_connectivity {
+        reasons.push("source snapshot unavailable and direct connectivity unavailable".to_owned());
+    }
+    let missing_artifacts = request
+        .required_artifacts
+        .iter()
+        .filter(|artifact| !node.artifact_locations.contains(*artifact))
+        .count();
+    if missing_artifacts > 0 && !node.direct_connectivity {
+        reasons.push(format!(
+            "{missing_artifacts} required artifact(s) unavailable and direct connectivity unavailable"
+        ));
+    }
+
+    if reasons.is_empty() {
+        Ok(Placement {
+            node: node.id.clone(),
+            score: 0,
+            reasons: Vec::new(),
+        })
+    } else {
+        Err(reasons)
+    }
+}
+
+fn locality_score(node: &NodeDescriptor, request: &PlacementRequest, placement: &mut Placement) {
+    if request.prefer_node.as_ref() == Some(&node.id) {
+        placement.score += 100;
+        placement.reasons.push("preferred node".to_owned());
+    }
+    if request
+        .environment_digest
+        .as_ref()
+        .is_some_and(|digest| node.cached_environments.contains(digest))
+    {
+        placement.score += 50;
+        placement.reasons.push("warm environment cache".to_owned());
+    }
+    if request
+        .source_snapshot
+        .as_ref()
+        .is_some_and(|digest| node.source_snapshots.contains(digest))
+    {
+        placement.score += 40;
+        placement
+            .reasons
+            .push("source snapshot already local".to_owned());
+    }
+    if request
+        .dependency_cache
+        .as_ref()
+        .is_some_and(|digest| node.dependency_caches.contains(digest))
+    {
+        placement.score += 30;
+        placement.reasons.push("warm dependency cache".to_owned());
+    }
+    let artifact_hits = request
+        .required_artifacts
+        .iter()
+        .filter(|artifact| node.artifact_locations.contains(*artifact))
+        .count() as i64;
+    if artifact_hits > 0 {
+        placement.score += 10 * artifact_hits;
+        placement.reasons.push(format!(
+            "{artifact_hits} required artifact(s) already local"
+        ));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{EnvironmentBackend, Os};
+
+    use super::*;
+
+    fn node(id: &str, cached_source: bool) -> NodeDescriptor {
+        let source = Digest::sha256("source");
+        NodeDescriptor {
+            id: NodeId::from(id),
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            capabilities: NodeCapabilities {
+                os: Os::Linux,
+                arch: "x86_64".to_owned(),
+                capabilities: BTreeSet::from([
+                    Capability::Command,
+                    Capability::Containers,
+                    Capability::RootlessPodman,
+                ]),
+                environment_backends: BTreeSet::from([EnvironmentBackend::Container]),
+                source_providers: BTreeSet::from(["filesystem".to_owned()]),
+            },
+            cached_environments: BTreeSet::from([Digest::sha256("env")]),
+            dependency_caches: if cached_source {
+                BTreeSet::from([Digest::sha256("deps")])
+            } else {
+                BTreeSet::new()
+            },
+            source_snapshots: if cached_source {
+                BTreeSet::from([source])
+            } else {
+                BTreeSet::new()
+            },
+            artifact_locations: BTreeSet::new(),
+            direct_connectivity: true,
+            online: true,
+        }
+    }
+
+    #[test]
+    fn scheduler_prefers_warm_source_and_environment() {
+        let request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: Some(EnvironmentRequirements::linux_container()),
+            environment_digest: Some(Digest::sha256("env")),
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::from([Capability::Command]),
+            dependency_cache: Some(Digest::sha256("deps")),
+            source_snapshot: Some(Digest::sha256("source")),
+            required_artifacts: BTreeSet::new(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let placement = DefaultScheduler
+            .place(&[node("cold", false), node("warm", true)], &request)
+            .unwrap();
+
+        assert_eq!(placement.node, NodeId::from("warm"));
+        assert!(placement
+            .reasons
+            .iter()
+            .any(|reason| reason.contains("source")));
+        assert!(placement
+            .reasons
+            .iter()
+            .any(|reason| reason.contains("dependency")));
+    }
+
+    #[test]
+    fn scheduler_requires_requested_named_environment_cache() {
+        let request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: None,
+            environment_digest: Some(Digest::sha256("missing-environment")),
+            environment_cache_required: true,
+            required_capabilities: BTreeSet::new(),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: BTreeSet::new(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let error = DefaultScheduler
+            .place(&[node("uncached", false)], &request)
+            .unwrap_err();
+
+        assert!(error.message.contains("named environment cache"));
+    }
+
+    #[test]
+    fn scheduler_failure_names_missing_constraint() {
+        let mut request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: BTreeSet::new(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+        request.required_capabilities.insert(Capability::Command);
+
+        let error = DefaultScheduler
+            .place(&[node("linux", false)], &request)
+            .unwrap_err();
+
+        assert!(error.message.contains("WindowsCommandDev"));
+    }
+
+    #[test]
+    fn scheduler_failure_names_environment_constraint() {
+        let request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: Some(EnvironmentRequirements::windows_command_dev()),
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::new(),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: BTreeSet::new(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let error = DefaultScheduler
+            .place(&[node("linux", false)], &request)
+            .unwrap_err();
+
+        assert!(error.message.contains("environment requires os Windows"));
+        assert!(error
+            .message
+            .contains("environment requires capability WindowsCommandDev"));
+    }
+
+    #[test]
+    fn scheduler_requires_direct_connectivity_when_transfer_is_needed() {
+        let mut disconnected = node("disconnected", false);
+        disconnected.direct_connectivity = false;
+        let mut local = node("local", true);
+        local.direct_connectivity = false;
+        let request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::from([Capability::Command]),
+            dependency_cache: None,
+            source_snapshot: Some(Digest::sha256("source")),
+            required_artifacts: BTreeSet::new(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let placement = DefaultScheduler
+            .place(&[disconnected, local], &request)
+            .unwrap();
+        assert_eq!(placement.node, NodeId::from("local"));
+
+        let mut disconnected = node("disconnected", false);
+        disconnected.direct_connectivity = false;
+        let error = DefaultScheduler
+            .place(&[disconnected], &request)
+            .unwrap_err();
+
+        assert!(error
+            .message
+            .contains("source snapshot unavailable and direct connectivity unavailable"));
+    }
+
+    #[test]
+    fn scheduler_failure_names_required_artifact_transfer_constraint() {
+        let mut disconnected = node("disconnected", true);
+        disconnected.direct_connectivity = false;
+        let request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::from([Capability::Command]),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: BTreeSet::from([ArtifactId::from("cache")]),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let error = DefaultScheduler
+            .place(&[disconnected], &request)
+            .unwrap_err();
+
+        assert!(error
+            .message
+            .contains("1 required artifact(s) unavailable and direct connectivity unavailable"));
+    }
+
+    #[test]
+    fn scheduler_failure_names_quota_and_policy_constraints() {
+        let mut request = PlacementRequest {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: BTreeSet::from([Capability::Command]),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: BTreeSet::new(),
+            quota_available: false,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+
+        let error = DefaultScheduler
+            .place(&[node("linux", false)], &request)
+            .unwrap_err();
+
+        assert!(error.message.contains("quota unavailable for placement"));
+
+        request.quota_available = true;
+        request.policy_allowed = false;
+        let error = DefaultScheduler
+            .place(&[node("linux", false)], &request)
+            .unwrap_err();
+
+        assert!(error.message.contains("policy denied placement"));
+    }
+}

crates/clusterflux-core/src/source.rs

@@ -0,0 +1,324 @@
+use std::collections::BTreeSet;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{Capability, Digest, ProjectId, TenantId};
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum SourceProviderKind {
+    Filesystem,
+    Git,
+    Custom(String),
+}
+
+impl SourceProviderKind {
+    pub fn provider_id(&self) -> &str {
+        match self {
+            SourceProviderKind::Filesystem => "filesystem",
+            SourceProviderKind::Git => "git",
+            SourceProviderKind::Custom(provider) => provider,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub enum SourceTransferMode {
+    RequiredContent,
+    ExplicitSnapshotChunks,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourceTransferPolicy {
+    pub local_source_bytes_remain_node_local: bool,
+    pub coordinator_receives_source_bytes_by_default: bool,
+    pub default_full_repo_tarball: bool,
+    pub allowed_remote_transfer: BTreeSet<SourceTransferMode>,
+}
+
+impl SourceTransferPolicy {
+    pub fn local_first_snapshot_chunks() -> Self {
+        Self {
+            local_source_bytes_remain_node_local: true,
+            coordinator_receives_source_bytes_by_default: false,
+            default_full_repo_tarball: false,
+            allowed_remote_transfer: BTreeSet::from([
+                SourceTransferMode::RequiredContent,
+                SourceTransferMode::ExplicitSnapshotChunks,
+            ]),
+        }
+    }
+}
+
+impl Default for SourceTransferPolicy {
+    fn default() -> Self {
+        Self::local_first_snapshot_chunks()
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourceProviderManifest {
+    pub kind: SourceProviderKind,
+    pub digest: Digest,
+    pub description: String,
+    #[serde(default)]
+    pub coordinator_requires_checkout_access: bool,
+    #[serde(default)]
+    pub transfer_policy: SourceTransferPolicy,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum SourceManifestError {
+    #[error("source provider manifest digest is not a valid sha256 digest: {0}")]
+    InvalidDigest(String),
+    #[error("custom source provider id `{0}` is invalid")]
+    InvalidProviderId(String),
+    #[error("source provider manifest description must be non-empty")]
+    EmptyDescription,
+    #[error("source provider manifest description is too long")]
+    DescriptionTooLong,
+    #[error("source provider manifest description contains control characters")]
+    DescriptionControlCharacter,
+    #[error("source provider manifest would require coordinator checkout access")]
+    CoordinatorCheckoutAccess,
+    #[error("source provider manifest would send source bytes to the coordinator by default")]
+    CoordinatorReceivesSourceBytes,
+    #[error("source provider manifest would default to a full-repo tarball")]
+    DefaultFullRepoTarball,
+    #[error("source provider manifest has no allowed remote transfer mode")]
+    MissingRemoteTransferMode,
+}
+
+pub trait SourceProviderModule {
+    fn kind(&self) -> SourceProviderKind;
+    fn manifest(&self) -> SourceProviderManifest;
+}
+
+impl SourceProviderManifest {
+    pub fn local_first(kind: SourceProviderKind, description: impl Into<String>) -> Self {
+        let transfer_policy = SourceTransferPolicy::local_first_snapshot_chunks();
+        let digest = Self::digest_for(&kind, false, &transfer_policy);
+        Self {
+            kind,
+            digest,
+            description: description.into(),
+            coordinator_requires_checkout_access: false,
+            transfer_policy,
+        }
+    }
+
+    pub fn validate_public_mvp(&self) -> Result<(), SourceManifestError> {
+        self.validate_shape()?;
+        if self.coordinator_requires_checkout_access {
+            return Err(SourceManifestError::CoordinatorCheckoutAccess);
+        }
+        if self
+            .transfer_policy
+            .coordinator_receives_source_bytes_by_default
+        {
+            return Err(SourceManifestError::CoordinatorReceivesSourceBytes);
+        }
+        if self.transfer_policy.default_full_repo_tarball {
+            return Err(SourceManifestError::DefaultFullRepoTarball);
+        }
+        if self.transfer_policy.allowed_remote_transfer.is_empty() {
+            return Err(SourceManifestError::MissingRemoteTransferMode);
+        }
+        Ok(())
+    }
+
+    fn validate_shape(&self) -> Result<(), SourceManifestError> {
+        if !self.digest.is_valid_sha256() {
+            return Err(SourceManifestError::InvalidDigest(
+                self.digest.as_str().to_owned(),
+            ));
+        }
+        if let SourceProviderKind::Custom(provider) = &self.kind {
+            if !valid_provider_id(provider) {
+                return Err(SourceManifestError::InvalidProviderId(provider.clone()));
+            }
+        }
+        if self.description.trim().is_empty() {
+            return Err(SourceManifestError::EmptyDescription);
+        }
+        if self.description.len() > 256 {
+            return Err(SourceManifestError::DescriptionTooLong);
+        }
+        if self.description.chars().any(char::is_control) {
+            return Err(SourceManifestError::DescriptionControlCharacter);
+        }
+        Ok(())
+    }
+
+    fn digest_for(
+        kind: &SourceProviderKind,
+        coordinator_requires_checkout_access: bool,
+        transfer_policy: &SourceTransferPolicy,
+    ) -> Digest {
+        let mut modes = transfer_policy
+            .allowed_remote_transfer
+            .iter()
+            .map(|mode| format!("{mode:?}"))
+            .collect::<Vec<_>>();
+        modes.sort();
+        let mut parts = vec![
+            b"source-provider-manifest:v2".to_vec(),
+            kind.provider_id().as_bytes().to_vec(),
+            coordinator_requires_checkout_access
+                .to_string()
+                .into_bytes(),
+            transfer_policy
+                .local_source_bytes_remain_node_local
+                .to_string()
+                .into_bytes(),
+            transfer_policy
+                .coordinator_receives_source_bytes_by_default
+                .to_string()
+                .into_bytes(),
+            transfer_policy
+                .default_full_repo_tarball
+                .to_string()
+                .into_bytes(),
+        ];
+        parts.extend(modes.into_iter().map(String::into_bytes));
+        Digest::from_parts(parts)
+    }
+}
+
+fn valid_provider_id(provider: &str) -> bool {
+    !provider.is_empty()
+        && provider.len() <= 64
+        && provider
+            .bytes()
+            .all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourcePreparation {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub provider: SourceProviderKind,
+    pub required_capabilities: BTreeSet<Capability>,
+    pub coordinator_requires_checkout_access: bool,
+}
+
+impl SourcePreparation {
+    pub fn node_task(tenant: TenantId, project: ProjectId, provider: SourceProviderKind) -> Self {
+        let capability = match provider {
+            SourceProviderKind::Filesystem => Capability::SourceFilesystem,
+            SourceProviderKind::Git => Capability::SourceGit,
+            SourceProviderKind::Custom(_) => Capability::SourceFilesystem,
+        };
+
+        Self {
+            tenant,
+            project,
+            provider,
+            required_capabilities: BTreeSet::from([capability]),
+            coordinator_requires_checkout_access: false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn source_preparation_can_be_scheduled_as_node_task() {
+        let prep = SourcePreparation::node_task(
+            TenantId::from("tenant"),
+            ProjectId::from("project"),
+            SourceProviderKind::Git,
+        );
+
+        assert!(!prep.coordinator_requires_checkout_access);
+        assert!(prep.required_capabilities.contains(&Capability::SourceGit));
+    }
+
+    #[test]
+    fn local_first_source_manifest_rejects_bulk_coordinator_paths() {
+        let manifest = SourceProviderManifest::local_first(
+            SourceProviderKind::Git,
+            "node-side Git snapshot provider",
+        );
+
+        assert!(manifest.validate_public_mvp().is_ok());
+        assert!(!manifest.coordinator_requires_checkout_access);
+        assert!(
+            !manifest
+                .transfer_policy
+                .coordinator_receives_source_bytes_by_default
+        );
+        assert!(!manifest.transfer_policy.default_full_repo_tarball);
+        assert!(manifest
+            .transfer_policy
+            .allowed_remote_transfer
+            .contains(&SourceTransferMode::ExplicitSnapshotChunks));
+    }
+
+    #[test]
+    fn source_manifest_validation_treats_manifest_as_hostile_input() {
+        let mut manifest = SourceProviderManifest::local_first(
+            SourceProviderKind::Custom("gitlab-lfs".to_owned()),
+            "custom provider",
+        );
+        assert!(manifest.validate_public_mvp().is_ok());
+
+        manifest.kind = SourceProviderKind::Custom("../checkout".to_owned());
+        assert_eq!(
+            manifest.validate_public_mvp(),
+            Err(SourceManifestError::InvalidProviderId(
+                "../checkout".to_owned()
+            ))
+        );
+
+        manifest.kind = SourceProviderKind::Git;
+        manifest.digest = Digest::sha256("valid");
+        manifest.coordinator_requires_checkout_access = true;
+        assert_eq!(
+            manifest.validate_public_mvp(),
+            Err(SourceManifestError::CoordinatorCheckoutAccess)
+        );
+
+        manifest.coordinator_requires_checkout_access = false;
+        manifest
+            .transfer_policy
+            .coordinator_receives_source_bytes_by_default = true;
+        assert_eq!(
+            manifest.validate_public_mvp(),
+            Err(SourceManifestError::CoordinatorReceivesSourceBytes)
+        );
+
+        manifest
+            .transfer_policy
+            .coordinator_receives_source_bytes_by_default = false;
+        manifest.transfer_policy.default_full_repo_tarball = true;
+        assert_eq!(
+            manifest.validate_public_mvp(),
+            Err(SourceManifestError::DefaultFullRepoTarball)
+        );
+    }
+
+    #[test]
+    fn source_manifest_rejects_malformed_digest_from_json() {
+        let mut manifest = SourceProviderManifest::local_first(
+            SourceProviderKind::Filesystem,
+            "filesystem provider",
+        );
+        let value = serde_json::to_value(&manifest).unwrap();
+        let mut object = value.as_object().unwrap().clone();
+        object.insert(
+            "digest".to_owned(),
+            serde_json::Value::String("sha256:not-a-real-digest".to_owned()),
+        );
+        manifest = serde_json::from_value(serde_json::Value::Object(object)).unwrap();
+
+        assert_eq!(
+            manifest.validate_public_mvp(),
+            Err(SourceManifestError::InvalidDigest(
+                "sha256:not-a-real-digest".to_owned()
+            ))
+        );
+    }
+}

crates/clusterflux-core/src/transport.rs

@@ -0,0 +1,243 @@
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{ArtifactId, Digest, NodeId, ProcessId, ProjectId, TenantId};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum TransportKind {
+    NativeQuic,
+}
+
+pub trait Transport {
+    fn kind(&self) -> TransportKind;
+    fn authenticated_direct_connections(&self) -> bool;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct NativeQuicTransport;
+
+impl Transport for NativeQuicTransport {
+    fn kind(&self) -> TransportKind {
+        TransportKind::NativeQuic
+    }
+
+    fn authenticated_direct_connections(&self) -> bool {
+        true
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DataPlaneScope {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub object: DataPlaneObject,
+    pub authorization_subject: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DataPlaneObject {
+    Artifact(ArtifactId),
+    Blob(Digest),
+    SourceSnapshot(Digest),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NodeEndpoint {
+    pub node: NodeId,
+    pub advertised_addr: String,
+    pub public_key_fingerprint: Digest,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RendezvousRequest {
+    pub scope: DataPlaneScope,
+    pub source: NodeEndpoint,
+    pub destination: NodeEndpoint,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DirectBulkTransferPlan {
+    pub transport: TransportKind,
+    pub scope: DataPlaneScope,
+    pub source: NodeEndpoint,
+    pub destination: NodeEndpoint,
+    pub authorization_digest: Digest,
+    pub coordinator_assisted_rendezvous: bool,
+    pub coordinator_bulk_relay_allowed: bool,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum TransportError {
+    #[error(
+        "direct node-to-node connectivity is unavailable for scoped data-plane transfer: {reason}; coordinator bulk relay is disabled"
+    )]
+    DirectConnectivityUnavailable { reason: String },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum BulkTransferDecision {
+    DirectAuthenticated { scope: DataPlaneScope },
+    FailClear { message: String },
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+#[error("bulk relay through coordinator is not allowed by default")]
+pub struct BulkRelayDenied;
+
+impl NativeQuicTransport {
+    pub fn plan_authenticated_direct_bulk_transfer(
+        &self,
+        request: RendezvousRequest,
+        direct_connectivity: bool,
+        failure_reason: impl Into<String>,
+    ) -> Result<DirectBulkTransferPlan, TransportError> {
+        if !direct_connectivity {
+            return Err(TransportError::DirectConnectivityUnavailable {
+                reason: failure_reason.into(),
+            });
+        }
+
+        let authorization_digest = data_plane_authorization_digest(&request);
+        Ok(DirectBulkTransferPlan {
+            transport: self.kind(),
+            scope: request.scope,
+            source: request.source,
+            destination: request.destination,
+            authorization_digest,
+            coordinator_assisted_rendezvous: true,
+            coordinator_bulk_relay_allowed: false,
+        })
+    }
+}
+
+pub fn direct_bulk_transfer_or_error(
+    scope: DataPlaneScope,
+    direct_connectivity: bool,
+) -> BulkTransferDecision {
+    if direct_connectivity {
+        BulkTransferDecision::DirectAuthenticated { scope }
+    } else {
+        BulkTransferDecision::FailClear {
+            message:
+                "direct node-to-node connectivity is unavailable; coordinator bulk relay is disabled"
+                    .to_owned(),
+        }
+    }
+}
+
+fn data_plane_authorization_digest(request: &RendezvousRequest) -> Digest {
+    let object = match &request.scope.object {
+        DataPlaneObject::Artifact(artifact) => format!("artifact:{artifact}"),
+        DataPlaneObject::Blob(digest) => format!("blob:{}", digest.as_str()),
+        DataPlaneObject::SourceSnapshot(digest) => format!("source:{}", digest.as_str()),
+    };
+
+    Digest::from_parts([
+        b"dataplane-auth:v1".as_slice(),
+        request.scope.tenant.as_str().as_bytes(),
+        request.scope.project.as_str().as_bytes(),
+        request.scope.process.as_str().as_bytes(),
+        object.as_bytes(),
+        request.scope.authorization_subject.as_bytes(),
+        request.source.node.as_str().as_bytes(),
+        request.source.public_key_fingerprint.as_str().as_bytes(),
+        request.destination.node.as_str().as_bytes(),
+        request
+            .destination
+            .public_key_fingerprint
+            .as_str()
+            .as_bytes(),
+    ])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn endpoint(name: &str) -> NodeEndpoint {
+        NodeEndpoint {
+            node: NodeId::from(name),
+            advertised_addr: format!("{name}.mesh.invalid:4433"),
+            public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
+        }
+    }
+
+    fn scope(project: &str) -> DataPlaneScope {
+        DataPlaneScope {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from(project),
+            process: ProcessId::from("process"),
+            object: DataPlaneObject::Artifact(ArtifactId::from("artifact")),
+            authorization_subject: "node-a-to-node-b".to_owned(),
+        }
+    }
+
+    #[test]
+    fn failed_direct_transfer_does_not_silently_relay() {
+        let decision = direct_bulk_transfer_or_error(scope("project"), false);
+
+        assert!(matches!(decision, BulkTransferDecision::FailClear { .. }));
+    }
+
+    #[test]
+    fn native_quic_rendezvous_plan_is_scoped_and_disallows_coordinator_bulk_relay() {
+        let transport = NativeQuicTransport;
+        let request = RendezvousRequest {
+            scope: scope("project"),
+            source: endpoint("node-a"),
+            destination: endpoint("node-b"),
+        };
+
+        let plan = transport
+            .plan_authenticated_direct_bulk_transfer(request.clone(), true, "")
+            .unwrap();
+        let changed_scope_plan = transport
+            .plan_authenticated_direct_bulk_transfer(
+                RendezvousRequest {
+                    scope: scope("other-project"),
+                    ..request
+                },
+                true,
+                "",
+            )
+            .unwrap();
+
+        assert_eq!(plan.transport, TransportKind::NativeQuic);
+        assert_eq!(plan.scope.tenant, TenantId::from("tenant"));
+        assert_eq!(plan.scope.project, ProjectId::from("project"));
+        assert_eq!(plan.scope.process, ProcessId::from("process"));
+        assert_eq!(
+            plan.scope.object,
+            DataPlaneObject::Artifact(ArtifactId::from("artifact"))
+        );
+        assert_eq!(plan.source.node, NodeId::from("node-a"));
+        assert_eq!(plan.destination.node, NodeId::from("node-b"));
+        assert!(plan.coordinator_assisted_rendezvous);
+        assert!(!plan.coordinator_bulk_relay_allowed);
+        assert_ne!(
+            plan.authorization_digest,
+            changed_scope_plan.authorization_digest
+        );
+    }
+
+    #[test]
+    fn failed_direct_rendezvous_reports_clear_error_instead_of_relaying() {
+        let error = NativeQuicTransport
+            .plan_authenticated_direct_bulk_transfer(
+                RendezvousRequest {
+                    scope: scope("project"),
+                    source: endpoint("node-a"),
+                    destination: endpoint("node-b"),
+                },
+                false,
+                "nat traversal failed",
+            )
+            .unwrap_err();
+
+        assert!(error.to_string().contains("nat traversal failed"));
+        assert!(error
+            .to_string()
+            .contains("coordinator bulk relay is disabled"));
+    }
+}

crates/clusterflux-core/src/vfs.rs

@@ -0,0 +1,241 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{Digest, NodeId, TaskInstanceId};
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct VfsPath(String);
+
+impl VfsPath {
+    pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
+        let path = path.into();
+        if !path.starts_with("/vfs/") {
+            return Err(VfsError::InvalidPath(path));
+        }
+        Ok(Self(path))
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct VfsObject {
+    pub path: VfsPath,
+    pub digest: Digest,
+    pub size: u64,
+    pub producer: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct VfsManifest {
+    pub epoch: u64,
+    pub producer: TaskInstanceId,
+    pub node: NodeId,
+    pub objects: BTreeMap<VfsPath, VfsObject>,
+    pub large_bytes_uploaded: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SyncPolicy {
+    MetadataOnly,
+    ExplicitNode(NodeId),
+    ExplicitStore(String),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum VfsSyncDecision {
+    NoBytesMoved,
+    MoveBytesToNode(NodeId),
+    MoveBytesToStore(String),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ReuseDecision {
+    SameNodeZeroCopy,
+    NeedsTransfer { from: NodeId, to: NodeId },
+    Unavailable,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum VfsError {
+    #[error("VFS path must start with /vfs/: {0}")]
+    InvalidPath(String),
+    #[error("path is not visible in the published VFS manifest: {0}")]
+    NotVisible(String),
+}
+
+#[derive(Clone, Debug)]
+pub struct VfsOverlay {
+    task: TaskInstanceId,
+    node: NodeId,
+    epoch: u64,
+    pending: BTreeMap<VfsPath, VfsObject>,
+    published: BTreeMap<VfsPath, VfsObject>,
+}
+
+impl VfsOverlay {
+    pub fn new(task: TaskInstanceId, node: NodeId) -> Self {
+        Self {
+            task,
+            node,
+            epoch: 0,
+            pending: BTreeMap::new(),
+            published: BTreeMap::new(),
+        }
+    }
+
+    pub fn write(&mut self, path: VfsPath, digest: Digest, size: u64) -> VfsObject {
+        let object = VfsObject {
+            path: path.clone(),
+            digest,
+            size,
+            producer: self.task.clone(),
+            node: self.node.clone(),
+        };
+        self.pending.insert(path, object.clone());
+        object
+    }
+
+    pub fn flush(&mut self) -> VfsManifest {
+        self.epoch += 1;
+        self.published.append(&mut self.pending);
+        VfsManifest {
+            epoch: self.epoch,
+            producer: self.task.clone(),
+            node: self.node.clone(),
+            objects: self.published.clone(),
+            large_bytes_uploaded: false,
+        }
+    }
+
+    pub fn sync(&self, policy: SyncPolicy) -> VfsSyncDecision {
+        match policy {
+            SyncPolicy::MetadataOnly => VfsSyncDecision::NoBytesMoved,
+            SyncPolicy::ExplicitNode(node) => VfsSyncDecision::MoveBytesToNode(node),
+            SyncPolicy::ExplicitStore(store) => VfsSyncDecision::MoveBytesToStore(store),
+        }
+    }
+
+    pub fn read_published<'a>(
+        manifest: &'a VfsManifest,
+        path: &VfsPath,
+    ) -> Result<&'a VfsObject, VfsError> {
+        manifest
+            .objects
+            .get(path)
+            .ok_or_else(|| VfsError::NotVisible(path.as_str().to_owned()))
+    }
+
+    pub fn reuse_for_consumer(
+        manifest: &VfsManifest,
+        path: &VfsPath,
+        consumer_node: &NodeId,
+    ) -> ReuseDecision {
+        let Some(object) = manifest.objects.get(path) else {
+            return ReuseDecision::Unavailable;
+        };
+        if &object.node == consumer_node {
+            ReuseDecision::SameNodeZeroCopy
+        } else {
+            ReuseDecision::NeedsTransfer {
+                from: object.node.clone(),
+                to: consumer_node.clone(),
+            }
+        }
+    }
+
+    pub fn discard_unflushed(&mut self) {
+        self.pending.clear();
+    }
+
+    pub fn pending_len(&self) -> usize {
+        self.pending.len()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn path() -> VfsPath {
+        VfsPath::new("/vfs/artifacts/app").unwrap()
+    }
+
+    #[test]
+    fn flush_publishes_manifest_without_large_byte_upload() {
+        let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
+        overlay.write(path(), Digest::sha256("binary"), 6);
+
+        let manifest = overlay.flush();
+
+        assert_eq!(manifest.epoch, 1);
+        assert!(!manifest.large_bytes_uploaded);
+        assert!(manifest.objects.contains_key(&path()));
+    }
+
+    #[test]
+    fn downstream_task_can_read_after_flush_but_not_before() {
+        let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
+        overlay.write(path(), Digest::sha256("binary"), 6);
+
+        let empty = VfsManifest {
+            epoch: 0,
+            producer: TaskInstanceId::from("task"),
+            node: NodeId::from("node-a"),
+            objects: BTreeMap::new(),
+            large_bytes_uploaded: false,
+        };
+        assert!(VfsOverlay::read_published(&empty, &path()).is_err());
+
+        let manifest = overlay.flush();
+        assert!(VfsOverlay::read_published(&manifest, &path()).is_ok());
+    }
+
+    #[test]
+    fn sync_is_explicit_and_policy_driven() {
+        let overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
+
+        assert_eq!(
+            overlay.sync(SyncPolicy::MetadataOnly),
+            VfsSyncDecision::NoBytesMoved
+        );
+        assert_eq!(
+            overlay.sync(SyncPolicy::ExplicitStore("s3://bucket/app".to_owned())),
+            VfsSyncDecision::MoveBytesToStore("s3://bucket/app".to_owned())
+        );
+    }
+
+    #[test]
+    fn same_node_reuse_avoids_transfer() {
+        let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
+        overlay.write(path(), Digest::sha256("binary"), 6);
+        let manifest = overlay.flush();
+
+        assert_eq!(
+            VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-a")),
+            ReuseDecision::SameNodeZeroCopy
+        );
+        assert_eq!(
+            VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-b")),
+            ReuseDecision::NeedsTransfer {
+                from: NodeId::from("node-a"),
+                to: NodeId::from("node-b")
+            }
+        );
+    }
+
+    #[test]
+    fn unflushed_task_local_changes_can_be_discarded() {
+        let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
+        overlay.write(path(), Digest::sha256("binary"), 6);
+
+        overlay.discard_unflushed();
+
+        assert_eq!(overlay.pending_len(), 0);
+    }
+}

crates/clusterflux-core/src/wire.rs

@@ -0,0 +1,114 @@
+use serde_json::{json, Value};
+
+pub const COORDINATOR_PROTOCOL_VERSION: u64 = 1;
+pub const COORDINATOR_WIRE_REQUEST_TYPE: &str = "coordinator_request";
+
+pub fn coordinator_wire_request(request_id: impl Into<String>, payload: Value) -> Value {
+    let operation = coordinator_payload_operation(&payload);
+    let authentication = coordinator_authentication_metadata(&payload);
+    json!({
+        "type": COORDINATOR_WIRE_REQUEST_TYPE,
+        "protocol_version": COORDINATOR_PROTOCOL_VERSION,
+        "request_id": request_id.into(),
+        "operation": operation,
+        "authentication": authentication,
+        "payload": payload,
+    })
+}
+
+pub fn coordinator_payload_operation(payload: &Value) -> String {
+    payload
+        .get("type")
+        .and_then(Value::as_str)
+        .unwrap_or("unknown")
+        .to_owned()
+}
+
+pub fn coordinator_authentication_metadata(payload: &Value) -> Value {
+    let operation = coordinator_payload_operation(payload);
+    match operation.as_str() {
+        "authenticated" => json!({
+            "kind": "cli_session",
+            "session": true,
+            "request_operation": payload
+                .get("request")
+                .map(coordinator_payload_operation)
+                .unwrap_or_else(|| "unknown".to_owned()),
+        }),
+        "signed_node" => json!({
+            "kind": "node_signature",
+            "node": payload.get("node").and_then(Value::as_str),
+        }),
+        "node_heartbeat" if payload.get("node_signature").is_some() => json!({
+            "kind": "node_signature",
+            "node": payload.get("node").and_then(Value::as_str),
+        }),
+        "start_process" | "launch_task" if payload.get("agent_signature").is_some() => json!({
+            "kind": "agent_signature",
+            "agent": payload.get("actor_agent").and_then(Value::as_str),
+            "fingerprint": payload.get("agent_public_key_fingerprint").and_then(Value::as_str),
+        }),
+        "admin_status" | "suspend_tenant" if payload.get("admin_proof").is_some() => json!({
+            "kind": "admin_proof",
+            "actor": payload.get("actor_user").and_then(Value::as_str),
+            "nonce": payload.get("admin_nonce").and_then(Value::as_str),
+            "issued_at_epoch_seconds": payload.get("issued_at_epoch_seconds").and_then(Value::as_u64),
+        }),
+        "exchange_node_enrollment_grant" => json!({
+            "kind": "node_enrollment_grant",
+            "node": payload.get("node").and_then(Value::as_str),
+        }),
+        _ => json!({
+            "kind": "none",
+        }),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn coordinator_wire_request_wraps_payload_without_exposing_secret_metadata() {
+        let envelope = coordinator_wire_request(
+            "cli-1",
+            json!({
+                "type": "authenticated",
+                "session_secret": "secret-value",
+                "request": { "type": "list_projects" },
+            }),
+        );
+
+        assert_eq!(envelope["type"], COORDINATOR_WIRE_REQUEST_TYPE);
+        assert_eq!(envelope["protocol_version"], COORDINATOR_PROTOCOL_VERSION);
+        assert_eq!(envelope["request_id"], "cli-1");
+        assert_eq!(envelope["operation"], "authenticated");
+        assert_eq!(envelope["authentication"]["kind"], "cli_session");
+        assert_eq!(
+            envelope["authentication"]["request_operation"],
+            "list_projects"
+        );
+        assert_eq!(envelope["authentication"].get("session_secret"), None);
+        assert_eq!(envelope["payload"]["session_secret"], "secret-value");
+    }
+
+    #[test]
+    fn coordinator_wire_request_describes_signature_metadata() {
+        let envelope = coordinator_wire_request(
+            "node-1",
+            json!({
+                "type": "signed_node",
+                "node": "node-a",
+                "node_signature": {
+                    "nonce": "nonce",
+                    "issued_at_epoch_seconds": 1,
+                    "signature": "ed25519:sig"
+                },
+                "request": { "type": "poll_task_assignment", "node": "node-a" },
+            }),
+        );
+
+        assert_eq!(envelope["authentication"]["kind"], "node_signature");
+        assert_eq!(envelope["authentication"]["node"], "node-a");
+    }
+}

crates/clusterflux-dap/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "clusterflux-dap"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[[bin]]
+name = "clusterflux-debug-dap"
+path = "src/main.rs"
+
+[dependencies]
+anyhow.workspace = true
+base64.workspace = true
+clusterflux-core = { path = "../clusterflux-core" }
+clusterflux-control = { path = "../clusterflux-control" }
+clusterflux-node = { path = "../clusterflux-node" }
+serde_json.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true

crates/clusterflux-dap/src/breakpoints.rs

@@ -0,0 +1,375 @@
+use std::fs;
+
+use clusterflux_core::{
+    discover_source_debug_probes, BundleDebugProbe, DebugRuntimeState, TaskInstanceId,
+};
+use serde_json::{json, Value};
+
+use crate::demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
+use crate::virtual_model::{AdapterState, VirtualThread};
+
+pub(crate) fn request_thread<'a>(request: &Value, state: &'a AdapterState) -> &'a VirtualThread {
+    let thread_id = request_thread_id(request).unwrap_or(MAIN_THREAD);
+    state
+        .threads
+        .get(&thread_id)
+        .unwrap_or_else(|| &state.threads[&MAIN_THREAD])
+}
+
+pub(crate) fn request_thread_id(request: &Value) -> Option<i64> {
+    request
+        .get("arguments")
+        .and_then(|value| value.get("threadId"))
+        .and_then(Value::as_i64)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct ResolvedBreakpoint {
+    pub(crate) id: usize,
+    pub(crate) line: i64,
+    pub(crate) verified: bool,
+    pub(crate) message: String,
+}
+
+impl ResolvedBreakpoint {
+    pub(crate) fn to_dap(&self) -> Value {
+        json!({
+            "id": self.id,
+            "verified": self.verified,
+            "line": self.line,
+            "message": self.message,
+        })
+    }
+}
+
+pub(crate) fn load_bundle_debug_probes(project: &str, source_path: &str) -> Vec<BundleDebugProbe> {
+    let source = fs::read_to_string(crate::source::resolve_source_path(project, source_path));
+    source
+        .map(|source| discover_source_debug_probes(source_path, &source))
+        .unwrap_or_default()
+}
+
+pub(crate) fn resolve_breakpoints(
+    state: &mut AdapterState,
+    requested_lines: Vec<i64>,
+) -> Vec<ResolvedBreakpoint> {
+    let resolved = requested_lines
+        .into_iter()
+        .enumerate()
+        .map(|(index, line)| {
+            let probe = debug_probe_for_line(state, line);
+            let verified = probe.is_some()
+                || (state.debug_probes.is_empty()
+                    && source_function_name_at_line(state, line).is_some());
+            let message = match probe {
+                Some(probe) => format!(
+                    "Mapped to Clusterflux debug probe {} for task {}",
+                    probe.id, probe.task
+                ),
+                None if verified => "Mapped to Clusterflux virtual source location".to_owned(),
+                None => "No Clusterflux debug probe metadata covers this source line".to_owned(),
+            };
+            ResolvedBreakpoint {
+                id: index + 1,
+                line,
+                verified,
+                message,
+            }
+        })
+        .collect::<Vec<_>>();
+    state.breakpoints = resolved
+        .iter()
+        .filter(|breakpoint| breakpoint.verified)
+        .map(|breakpoint| breakpoint.line)
+        .collect();
+    resolved
+}
+
+pub(crate) fn resolve_breakpoints_for_source(
+    state: &mut AdapterState,
+    requested_source_path: Option<&str>,
+    requested_lines: Vec<i64>,
+) -> Vec<ResolvedBreakpoint> {
+    if requested_source_path.is_none_or(|requested_source_path| {
+        crate::source::source_paths_match(&state.project, &state.source_path, requested_source_path)
+    }) {
+        return resolve_breakpoints(state, requested_lines);
+    }
+
+    requested_lines
+        .into_iter()
+        .enumerate()
+        .map(|(index, line)| ResolvedBreakpoint {
+            id: index + 1,
+            line,
+            verified: false,
+            message: format!(
+                "This debug session is configured for `{}`; breakpoints from another source file are not applied",
+                state.source_path
+            ),
+        })
+        .collect()
+}
+
+pub(crate) fn restart_requires_whole_process(request: &Value) -> bool {
+    let Some(arguments) = request.get("arguments") else {
+        return false;
+    };
+
+    arguments
+        .get("requiresWholeProcessRestart")
+        .and_then(Value::as_bool)
+        .unwrap_or(false)
+        || [
+            "compatibility",
+            "sourceCompatibility",
+            "sourceEditCompatibility",
+            "taskCompatibility",
+        ]
+        .iter()
+        .filter_map(|field| arguments.get(field).and_then(Value::as_str))
+        .any(is_incompatible_restart)
+        || arguments
+            .get("sourceEdit")
+            .and_then(|value| value.get("compatibility"))
+            .and_then(Value::as_str)
+            .is_some_and(is_incompatible_restart)
+}
+
+fn is_incompatible_restart(value: &str) -> bool {
+    value.eq_ignore_ascii_case("incompatible")
+        || value.eq_ignore_ascii_case("whole-process")
+        || value.eq_ignore_ascii_case("whole_process")
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct FreezeFailure {
+    pub(crate) thread_id: i64,
+    pub(crate) task: TaskInstanceId,
+}
+
+impl FreezeFailure {
+    pub(crate) fn message(&self) -> String {
+        format!(
+            "debug all-stop failed: participant `{}` on thread {} could not freeze",
+            self.task, self.thread_id
+        )
+    }
+}
+
+pub(crate) fn simulated_freeze_failure_thread(request: &Value) -> Option<i64> {
+    request
+        .get("arguments")
+        .and_then(|arguments| arguments.get("simulateFreezeFailure"))
+        .and_then(Value::as_bool)
+        .unwrap_or(false)
+        .then_some(PACKAGE_THREAD)
+}
+
+pub(crate) fn freeze_all(
+    state: &mut AdapterState,
+    stopped_thread: i64,
+    forced_failure_thread: Option<i64>,
+) -> Result<(), FreezeFailure> {
+    let line = state
+        .breakpoints
+        .iter()
+        .copied()
+        .find(|line| thread_for_source_line(state, *line) == stopped_thread)
+        .or_else(|| state.breakpoints.first().copied())
+        .unwrap_or_else(|| state.threads[&stopped_thread].line);
+    freeze_all_at_line(state, stopped_thread, line, forced_failure_thread)
+}
+
+pub(crate) fn freeze_all_at_line(
+    state: &mut AdapterState,
+    stopped_thread: i64,
+    line: i64,
+    forced_failure_thread: Option<i64>,
+) -> Result<(), FreezeFailure> {
+    if let Some(failure) = state.threads.values().find(|thread| {
+        thread.state == DebugRuntimeState::Running
+            && (!thread.freeze_supported || forced_failure_thread == Some(thread.id))
+    }) {
+        return Err(FreezeFailure {
+            thread_id: failure.id,
+            task: failure.task.clone(),
+        });
+    }
+
+    state.epoch += 1;
+    for thread in state.threads.values_mut() {
+        if thread.state == DebugRuntimeState::Running {
+            thread.state = DebugRuntimeState::Frozen;
+        }
+    }
+    if let Some(thread) = state.threads.get_mut(&stopped_thread) {
+        thread.line = line;
+        thread
+            .recent_output
+            .push(format!("debug epoch {} all-stop", state.epoch));
+    }
+    Ok(())
+}
+
+pub(crate) fn stopped_thread_for_breakpoint(state: &AdapterState) -> i64 {
+    if let Some(stopped_task) = state.stopped_task.as_ref() {
+        if let Some(thread) = state
+            .threads
+            .values()
+            .find(|thread| &thread.task == stopped_task)
+        {
+            return thread.id;
+        }
+    }
+    state
+        .breakpoints
+        .first()
+        .map(|line| thread_for_source_line(state, *line))
+        .unwrap_or(MAIN_THREAD)
+}
+
+pub(crate) fn position_confirmed_breakpoint_stop(state: &mut AdapterState, stopped_thread: i64) {
+    let line = state
+        .stopped_probe_symbol
+        .as_deref()
+        .and_then(|symbol| symbol.strip_prefix("clusterflux.probe."))
+        .and_then(|function| {
+            state
+                .debug_probes
+                .iter()
+                .find(|probe| probe.function == function)
+                .and_then(|probe| {
+                    state.breakpoints.iter().copied().find(|line| {
+                        *line >= i64::from(probe.line_start) && *line <= i64::from(probe.line_end)
+                    })
+                })
+        })
+        .or_else(|| {
+            state
+                .breakpoints
+                .iter()
+                .copied()
+                .find(|line| thread_for_source_line(state, *line) == stopped_thread)
+        })
+        .or_else(|| state.breakpoints.first().copied());
+    if let (Some(line), Some(thread)) = (line, state.threads.get_mut(&stopped_thread)) {
+        thread.line = line;
+        thread.recent_output.push(format!(
+            "debug epoch {} confirmed frozen by all participants",
+            state.epoch
+        ));
+    }
+}
+
+pub(crate) fn next_breakpoint_after(state: &AdapterState, thread_id: i64) -> Option<(i64, i64)> {
+    let current_line = state.threads.get(&thread_id)?.line;
+    state
+        .breakpoints
+        .iter()
+        .copied()
+        .filter(|line| *line > current_line)
+        .min()
+        .map(|line| (thread_for_source_line(state, line), line))
+}
+
+fn thread_for_source_line(state: &AdapterState, line: i64) -> i64 {
+    if let Some(probe) = debug_probe_for_line(state, line) {
+        if state.runtime_backend == crate::virtual_model::RuntimeBackend::Simulated {
+            let function = probe.function.to_ascii_lowercase();
+            if function.contains("linux") {
+                return LINUX_THREAD;
+            }
+            if function.contains("windows") {
+                return WINDOWS_THREAD;
+            }
+            if function.contains("package") {
+                return PACKAGE_THREAD;
+            }
+            return MAIN_THREAD;
+        }
+        return state
+            .stopped_task
+            .as_ref()
+            .and_then(|task| {
+                state
+                    .threads
+                    .values()
+                    .find(|thread| &thread.task == task)
+                    .map(|thread| thread.id)
+            })
+            .unwrap_or(MAIN_THREAD);
+    }
+
+    if let Some(function_name) = source_function_name_at_line(state, line) {
+        let lower = function_name.to_ascii_lowercase();
+        if lower.contains("linux") {
+            return LINUX_THREAD;
+        }
+        if lower.contains("windows") {
+            return WINDOWS_THREAD;
+        }
+        if lower.contains("package") {
+            return PACKAGE_THREAD;
+        }
+        return MAIN_THREAD;
+    }
+
+    state
+        .threads
+        .values()
+        .find(|thread| thread.line == line)
+        .map(|thread| thread.id)
+        .unwrap_or(MAIN_THREAD)
+}
+
+fn debug_probe_for_line(state: &AdapterState, line: i64) -> Option<&BundleDebugProbe> {
+    let line = u32::try_from(line).ok()?;
+    state
+        .debug_probes
+        .iter()
+        .find(|probe| probe.line_start <= line && line <= probe.line_end)
+}
+
+pub(crate) fn source_function_name_at_line(state: &AdapterState, line: i64) -> Option<String> {
+    let source = fs::read_to_string(crate::source::resolve_source_path(
+        &state.project,
+        &state.source_path,
+    ))
+    .ok()?;
+    let lines = source.lines().collect::<Vec<_>>();
+    let mut index = usize::try_from(line).ok()?.saturating_sub(1);
+    if index >= lines.len() {
+        index = lines.len().saturating_sub(1);
+    }
+    lines[..=index]
+        .iter()
+        .rev()
+        .find_map(|line| parse_rust_function_name(line))
+}
+
+pub(crate) fn parse_rust_function_name(line: &str) -> Option<String> {
+    let start = line.find("fn ")? + 3;
+    let rest = &line[start..];
+    let name = rest
+        .chars()
+        .take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
+        .collect::<String>();
+    (!name.is_empty()).then_some(name)
+}
+
+pub(crate) fn step_thread(state: &mut AdapterState, thread_id: i64, description: &str) {
+    state.epoch += 1;
+    let resolved_thread = if state.threads.contains_key(&thread_id) {
+        thread_id
+    } else {
+        LINUX_THREAD
+    };
+    if let Some(thread) = state.threads.get_mut(&resolved_thread) {
+        thread.state = DebugRuntimeState::Frozen;
+        thread.line += 1;
+        thread
+            .recent_output
+            .push(format!("debug epoch {} {description}", state.epoch));
+    }
+}

crates/clusterflux-dap/src/dap_protocol.rs

@@ -0,0 +1,129 @@
+use std::io::{self, BufRead, Write};
+
+use anyhow::{anyhow, Result};
+use serde_json::{json, Value};
+
+#[derive(Clone)]
+pub(crate) struct DapWriter {
+    seq: i64,
+}
+
+impl DapWriter {
+    pub(crate) fn new() -> Self {
+        Self { seq: 1 }
+    }
+
+    pub(crate) fn response(
+        &mut self,
+        request: &Value,
+        success: bool,
+        body: Value,
+    ) -> io::Result<()> {
+        let command = request
+            .get("command")
+            .and_then(Value::as_str)
+            .unwrap_or("<unknown>");
+        let request_seq = request.get("seq").and_then(Value::as_i64).unwrap_or(0);
+        let seq = self.next_seq();
+        self.write(json!({
+            "seq": seq,
+            "type": "response",
+            "request_seq": request_seq,
+            "success": success,
+            "command": command,
+            "body": body,
+        }))
+    }
+
+    pub(crate) fn error_response(
+        &mut self,
+        request: &Value,
+        message: impl Into<String>,
+    ) -> io::Result<()> {
+        let command = request
+            .get("command")
+            .and_then(Value::as_str)
+            .unwrap_or("<unknown>");
+        let request_seq = request.get("seq").and_then(Value::as_i64).unwrap_or(0);
+        let seq = self.next_seq();
+        self.write(json!({
+            "seq": seq,
+            "type": "response",
+            "request_seq": request_seq,
+            "success": false,
+            "command": command,
+            "message": message.into(),
+        }))
+    }
+
+    pub(crate) fn event(&mut self, event: &str, body: Value) -> io::Result<()> {
+        let seq = self.next_seq();
+        self.write(json!({
+            "seq": seq,
+            "type": "event",
+            "event": event,
+            "body": body,
+        }))
+    }
+
+    pub(crate) fn output(&mut self, category: &str, output: impl Into<String>) -> io::Result<()> {
+        self.event(
+            "output",
+            json!({
+                "category": category,
+                "output": output.into(),
+            }),
+        )
+    }
+
+    fn next_seq(&mut self) -> i64 {
+        let seq = self.seq;
+        self.seq += 1;
+        seq
+    }
+
+    fn write(&mut self, message: Value) -> io::Result<()> {
+        let payload = serde_json::to_vec(&message)?;
+        let mut out = io::stdout().lock();
+        write!(out, "Content-Length: {}\r\n\r\n", payload.len())?;
+        out.write_all(&payload)?;
+        out.flush()
+    }
+}
+
+pub(crate) fn initialize_capabilities() -> Value {
+    json!({
+        "supportsConfigurationDoneRequest": true,
+        "supportsTerminateRequest": true,
+        "supportsRestartRequest": true,
+        "supportsRestartFrame": true,
+        "supportsEvaluateForHovers": true,
+        "supportsStepBack": false,
+    })
+}
+
+pub(crate) fn read_message<R: BufRead>(reader: &mut R) -> Result<Option<Value>> {
+    let mut content_length = None;
+
+    loop {
+        let mut line = String::new();
+        let bytes = reader.read_line(&mut line)?;
+        if bytes == 0 {
+            return Ok(None);
+        }
+
+        let line = line.trim_end_matches(['\r', '\n']);
+        if line.is_empty() {
+            break;
+        }
+
+        if let Some(value) = line.strip_prefix("Content-Length:") {
+            content_length = Some(value.trim().parse::<usize>()?);
+        }
+    }
+
+    let length = content_length.ok_or_else(|| anyhow!("DAP message missing Content-Length"))?;
+    let mut payload = vec![0; length];
+    reader.read_exact(&mut payload)?;
+    Ok(Some(serde_json::from_slice(&payload)?))
+}

crates/clusterflux-dap/src/demo_backend.rs

@@ -0,0 +1,68 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{DebugRuntimeState, TaskInstanceId};
+
+use crate::virtual_model::{AdapterState, VirtualThread};
+
+pub(crate) const MAIN_THREAD: i64 = 1;
+pub(crate) const LINUX_THREAD: i64 = 2;
+pub(crate) const WINDOWS_THREAD: i64 = 3;
+pub(crate) const PACKAGE_THREAD: i64 = 4;
+
+pub(crate) fn is_explicit_demo_backend(value: &str) -> bool {
+    value == "simulated" || value == "demo"
+}
+
+pub(crate) fn launch_threads(entry: &str) -> BTreeMap<i64, VirtualThread> {
+    [
+        thread(MAIN_THREAD, "main", &format!("{entry} virtual process"), 12),
+        thread(LINUX_THREAD, "compile-linux", "compile linux", 42),
+        thread(WINDOWS_THREAD, "compile-windows", "compile windows", 52),
+        thread(PACKAGE_THREAD, "package-release", "package artifacts", 64),
+    ]
+    .into_iter()
+    .map(|thread| (thread.id, thread))
+    .collect()
+}
+
+pub(crate) fn start_simulated_backend(state: &mut AdapterState) {
+    state.command_status = "running under simulated debug adapter state".to_owned();
+}
+
+fn thread(id: i64, task: &str, name: &str, line: i64) -> VirtualThread {
+    VirtualThread {
+        id,
+        frame_id: 1000 + id,
+        locals_ref: 5000 + id,
+        wasm_locals_ref: 9000 + id,
+        args_ref: 2000 + id,
+        runtime_ref: 3000 + id,
+        output_ref: 4000 + id,
+        target_ref: 6000 + id,
+        vfs_ref: 7000 + id,
+        command_ref: 8000 + id,
+        task: TaskInstanceId::from(task),
+        attempt_id: format!("demo-attempt-{id}"),
+        task_definition: clusterflux_core::TaskDefinitionId::from(task),
+        name: name.to_owned(),
+        line,
+        state: DebugRuntimeState::Running,
+        freeze_supported: true,
+        recent_output: vec![format!("{name}: waiting")],
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        runtime_stack_frames: Vec::new(),
+        wasm_local_values: Vec::new(),
+        runtime_task_args: Vec::new(),
+        runtime_handles: Vec::new(),
+        runtime_command_status: None,
+        runtime_node: Some("demo-node".to_owned()),
+        runtime_environment: Some("demo".to_owned()),
+        runtime_vfs_checkpoint: Some("demo-vfs".to_owned()),
+        restart_compatible: Some(true),
+    }
+}

crates/clusterflux-dap/src/main.rs

@@ -0,0 +1,43 @@
+use anyhow::Result;
+#[cfg(test)]
+use std::path::Path;
+
+mod adapter;
+mod breakpoints;
+mod dap_protocol;
+mod demo_backend;
+mod runtime_client;
+mod source;
+mod variables;
+mod view_state;
+mod virtual_model;
+
+#[cfg(test)]
+use adapter::runtime_backend_from_launch_arg;
+#[cfg(test)]
+use breakpoints::{
+    freeze_all, resolve_breakpoints, resolve_breakpoints_for_source,
+    restart_requires_whole_process, stopped_thread_for_breakpoint,
+};
+#[cfg(test)]
+use dap_protocol::{initialize_capabilities, read_message};
+#[cfg(test)]
+use runtime_client::{client_user_request, parse_task_restart_response};
+#[cfg(test)]
+use variables::variables_response;
+#[cfg(test)]
+use virtual_model::{AdapterState, RuntimeLaunchRecord};
+
+#[cfg(test)]
+use clusterflux_core::{BundleDebugProbe, DebugRuntimeState, TaskInstanceId};
+#[cfg(test)]
+use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
+#[cfg(test)]
+use virtual_model::{process_id, RuntimeBackend};
+
+fn main() -> Result<()> {
+    adapter::run_adapter()
+}
+
+#[cfg(test)]
+mod tests;

crates/clusterflux-dap/src/runtime_client/debug_protocol.rs

@@ -0,0 +1,243 @@
+use std::time::{Duration, Instant};
+
+use anyhow::{anyhow, Result};
+use clusterflux_core::TaskInstanceId;
+use serde_json::{json, Value};
+
+use crate::virtual_model::AdapterState;
+
+use super::{
+    client_user_request, coordinator_request, DebugEpochRecord, DebugEpochStatusRecord,
+    TaskRestartRecord,
+};
+
+pub(super) fn coordinator_debug_epoch_request(
+    state: &AdapterState,
+    payload: Value,
+) -> Result<Value> {
+    let coordinator =
+        crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
+    coordinator_request(&coordinator, payload)
+}
+
+pub(super) fn parse_debug_epoch_response(response: Value) -> Result<DebugEpochRecord> {
+    let epoch = response
+        .get("epoch")
+        .and_then(Value::as_u64)
+        .ok_or_else(|| anyhow!("coordinator debug epoch response did not include an epoch"))?;
+    let command = response
+        .get("command")
+        .and_then(Value::as_str)
+        .ok_or_else(|| anyhow!("coordinator debug epoch response did not include a command"))?
+        .to_owned();
+    let affected_tasks = response
+        .get("affected_tasks")
+        .and_then(Value::as_array)
+        .map(Vec::len)
+        .unwrap_or(0);
+    Ok(DebugEpochRecord {
+        epoch,
+        command,
+        affected_tasks,
+    })
+}
+
+pub(super) fn wait_for_debug_epoch_state(
+    state: &AdapterState,
+    epoch: u64,
+    frozen: bool,
+) -> Result<DebugEpochStatusRecord> {
+    let deadline = Instant::now() + Duration::from_secs(60);
+    loop {
+        let response = match coordinator_debug_epoch_request(
+            state,
+            client_user_request(
+                state,
+                json!({
+                    "type": "inspect_debug_epoch",
+                    "tenant": state.tenant,
+                    "project": state.project_id,
+                    "actor_user": state.actor_user,
+                    "process": state.process.as_str(),
+                    "epoch": epoch,
+                }),
+            ),
+        ) {
+            Ok(response) => response,
+            Err(error) if !frozen && debug_epoch_was_released(&error, state, epoch) => {
+                return Ok(DebugEpochStatusRecord {
+                    epoch,
+                    command: "resume".to_owned(),
+                    expected_tasks: 0,
+                    acknowledgements: Vec::new(),
+                    fully_frozen: false,
+                    partially_frozen: false,
+                    fully_resumed: true,
+                    failed: false,
+                    failure_messages: Vec::new(),
+                });
+            }
+            Err(error) => return Err(error),
+        };
+        let status = parse_debug_epoch_status(response)?;
+        if frozen && (status.fully_frozen || status.partially_frozen) {
+            return Ok(status);
+        }
+        if status.failed {
+            return Err(anyhow!(
+                "debug epoch {epoch} participant failed: {}",
+                status.failure_messages.join("; ")
+            ));
+        }
+        if !frozen && status.fully_resumed {
+            return Ok(status);
+        }
+        if Instant::now() >= deadline {
+            return Err(anyhow!(
+                "debug epoch {epoch} did not receive {}/{} signed participant acknowledgements for {} within 60 seconds",
+                status.acknowledgements.len(),
+                status.expected_tasks,
+                if frozen { "frozen state" } else { "resumed state" }
+            ));
+        }
+        std::thread::sleep(Duration::from_millis(100));
+    }
+}
+
+fn debug_epoch_was_released(error: &anyhow::Error, state: &AdapterState, epoch: u64) -> bool {
+    format!("{error:#}").contains(&format!(
+        "debug epoch {epoch} is not active for {}",
+        state.process
+    ))
+}
+
+fn parse_debug_epoch_status(response: Value) -> Result<DebugEpochStatusRecord> {
+    let epoch = response
+        .get("epoch")
+        .and_then(Value::as_u64)
+        .ok_or_else(|| anyhow!("coordinator debug epoch status omitted epoch"))?;
+    let command = response
+        .get("command")
+        .and_then(Value::as_str)
+        .ok_or_else(|| anyhow!("coordinator debug epoch status omitted command"))?
+        .to_owned();
+    let expected_tasks = response
+        .get("expected_tasks")
+        .and_then(Value::as_array)
+        .map(Vec::len)
+        .unwrap_or(0);
+    let acknowledgements = response
+        .get("acknowledgements")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    let failure_messages = response
+        .get("failure_messages")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+        .filter_map(Value::as_str)
+        .map(str::to_owned)
+        .collect();
+    Ok(DebugEpochStatusRecord {
+        epoch,
+        command,
+        expected_tasks,
+        acknowledgements,
+        fully_frozen: response
+            .get("fully_frozen")
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        partially_frozen: response
+            .get("partially_frozen")
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        fully_resumed: response
+            .get("fully_resumed")
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        failed: response
+            .get("failed")
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        failure_messages,
+    })
+}
+
+pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestartRecord> {
+    Ok(TaskRestartRecord {
+        accepted: response
+            .get("accepted")
+            .and_then(Value::as_bool)
+            .ok_or_else(|| anyhow!("coordinator task restart response did not include accepted"))?,
+        restarted_task_instance: response
+            .get("restarted_task_instance")
+            .and_then(Value::as_str)
+            .map(TaskInstanceId::new),
+        restarted_attempt_id: response
+            .get("restarted_attempt_id")
+            .and_then(Value::as_str)
+            .map(str::to_owned),
+        clean_boundary_available: response
+            .get("clean_boundary_available")
+            .and_then(Value::as_bool)
+            .ok_or_else(|| {
+                anyhow!("coordinator task restart response did not include clean_boundary_available")
+            })?,
+        requires_whole_process_restart: response
+            .get("requires_whole_process_restart")
+            .and_then(Value::as_bool)
+            .ok_or_else(|| {
+                anyhow!(
+                    "coordinator task restart response did not include requires_whole_process_restart"
+                )
+            })?,
+        active_task: response
+            .get("active_task")
+            .and_then(Value::as_bool)
+            .ok_or_else(|| anyhow!("coordinator task restart response did not include active_task"))?,
+        completed_event_observed: response
+            .get("completed_event_observed")
+            .and_then(Value::as_bool)
+            .ok_or_else(|| {
+                anyhow!("coordinator task restart response did not include completed_event_observed")
+            })?,
+        message: response
+            .get("message")
+            .and_then(Value::as_str)
+            .ok_or_else(|| anyhow!("coordinator task restart response did not include message"))?
+            .to_owned(),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use anyhow::anyhow;
+
+    use super::debug_epoch_was_released;
+    use crate::virtual_model::AdapterState;
+
+    #[test]
+    fn accepts_epoch_release_after_an_accepted_resume() {
+        let state = AdapterState {
+            process: "vp-completed".into(),
+            ..AdapterState::default()
+        };
+
+        assert!(debug_epoch_was_released(
+            &anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-completed"),
+            &state,
+            3,
+        ));
+        assert!(!debug_epoch_was_released(
+            &anyhow!("coordinator protocol error: debug epoch 4 is not active for vp-completed"),
+            &state,
+            3,
+        ));
+        assert!(!debug_epoch_was_released(
+            &anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-other"),
+            &state,
+            3,
+        ));
+    }
+}

crates/clusterflux-dap/src/runtime_client/local_tools.rs

@@ -0,0 +1,63 @@
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command};
+
+pub(super) fn child_stderr_suffix(child: &mut Child) -> String {
+    let mut stderr = String::new();
+    if let Some(mut stream) = child.stderr.take() {
+        let _ = stream.read_to_string(&mut stderr);
+    }
+    let stderr = stderr.trim();
+    if stderr.is_empty() {
+        String::new()
+    } else {
+        format!("; worker stderr: {stderr}")
+    }
+}
+
+pub(super) fn local_tool_command(
+    env_var: &str,
+    binary: &str,
+    package: &str,
+    repo: &Path,
+) -> Command {
+    if let Some(path) = std::env::var_os(env_var) {
+        return Command::new(path);
+    }
+    let current_exe = std::env::current_exe().ok();
+    if !workspace_development_executable(current_exe.as_deref(), repo) {
+        if let Some(path) = sibling_tool_path(current_exe.as_deref(), binary) {
+            return Command::new(path);
+        }
+    }
+
+    // A workspace-built adapter must ask Cargo for the matching service binary:
+    // a sibling executable can be older than the adapter even though both happen
+    // to live in target/debug. Installed releases take the sibling path above.
+    let mut command = Command::new("cargo");
+    command.args(["run", "-q", "-p", package, "--bin", binary, "--"]);
+    command
+}
+
+fn sibling_tool_path(current_exe: Option<&Path>, binary: &str) -> Option<PathBuf> {
+    let mut sibling = current_exe?.to_path_buf();
+    sibling.set_file_name(format!("{binary}{}", std::env::consts::EXE_SUFFIX));
+    sibling.is_file().then_some(sibling)
+}
+
+fn workspace_development_executable(current_exe: Option<&Path>, repo: &Path) -> bool {
+    let Some(current_exe) = current_exe else {
+        return false;
+    };
+    let target = std::env::var_os("CARGO_TARGET_DIR")
+        .map(PathBuf::from)
+        .map(|path| {
+            if path.is_absolute() {
+                path
+            } else {
+                repo.join(path)
+            }
+        })
+        .unwrap_or_else(|| repo.join("target"));
+    current_exe.starts_with(target)
+}

crates/clusterflux-dap/src/runtime_client/transport.rs

@@ -0,0 +1,53 @@
+use anyhow::{anyhow, Result};
+use clusterflux_control::ControlSession;
+use clusterflux_core::coordinator_wire_request;
+use serde_json::{json, Value};
+
+use crate::virtual_model::AdapterState;
+
+pub(crate) fn client_user_request(state: &AdapterState, mut request: Value) -> Value {
+    let Some(session_secret) = state.client_session_secret.as_deref() else {
+        return request;
+    };
+    if let Value::Object(fields) = &mut request {
+        fields.remove("tenant");
+        fields.remove("project");
+        fields.remove("actor_user");
+    }
+    json!({
+        "type": "authenticated",
+        "session_secret": session_secret,
+        "request": request,
+    })
+}
+
+pub(super) struct CoordinatorSession {
+    session: ControlSession,
+}
+
+impl CoordinatorSession {
+    pub(super) fn connect(addr: &str) -> Result<Self> {
+        Ok(Self {
+            session: ControlSession::connect(addr)?,
+        })
+    }
+
+    pub(super) fn request(&mut self, request: Value) -> Result<Value> {
+        let wire_request = coordinator_wire_request("dap-1", request);
+        let response = self.session.request(&wire_request)?;
+        if response.get("type").and_then(Value::as_str) == Some("error") {
+            return Err(anyhow!(
+                "{}",
+                response
+                    .get("message")
+                    .and_then(Value::as_str)
+                    .unwrap_or("coordinator returned an error")
+            ));
+        }
+        Ok(response)
+    }
+}
+
+pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
+    CoordinatorSession::connect(addr)?.request(request)
+}

crates/clusterflux-dap/src/source.rs

@@ -0,0 +1,71 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use anyhow::Result;
+use serde_json::{json, Value};
+
+use crate::virtual_model::AdapterState;
+
+pub(crate) fn infer_source_path(project: &str) -> String {
+    if Path::new(project).join("src/lib.rs").is_file() {
+        "src/lib.rs".to_owned()
+    } else if Path::new(project).join("src/main.rs").is_file() {
+        "src/main.rs".to_owned()
+    } else if Path::new(project).join("src/build.rs").is_file() {
+        "src/build.rs".to_owned()
+    } else {
+        "src/main.rs".to_owned()
+    }
+}
+
+pub(crate) fn source_name(source_path: &str) -> &str {
+    Path::new(source_path)
+        .file_name()
+        .and_then(|name| name.to_str())
+        .unwrap_or(source_path)
+}
+
+pub(crate) fn stack_source_path(state: &AdapterState) -> String {
+    normalized_source_path(&state.project, &state.source_path)
+        .to_string_lossy()
+        .into_owned()
+}
+
+pub(crate) fn source_paths_match(project: &str, configured: &str, requested: &str) -> bool {
+    normalized_source_path(project, configured) == normalized_source_path(project, requested)
+}
+
+pub(crate) fn source_response(state: &AdapterState, request: &Value) -> Result<Value> {
+    let source_path = request
+        .get("arguments")
+        .and_then(|arguments| arguments.get("source"))
+        .and_then(|source| source.get("path"))
+        .and_then(Value::as_str)
+        .unwrap_or(&state.source_path);
+    let content = fs::read_to_string(resolve_source_path(&state.project, source_path))?;
+    Ok(json!({
+        "content": content,
+        "mimeType": "text/x-rustsrc",
+    }))
+}
+
+pub(crate) fn resolve_source_path(project: &str, source_path: &str) -> PathBuf {
+    let path = Path::new(source_path);
+    if path.is_absolute() {
+        path.to_path_buf()
+    } else {
+        Path::new(project).join(path)
+    }
+}
+
+fn normalized_source_path(project: &str, source_path: &str) -> PathBuf {
+    let resolved = resolve_source_path(project, source_path);
+    let absolute = if resolved.is_absolute() {
+        resolved
+    } else {
+        std::env::current_dir()
+            .unwrap_or_else(|_| Path::new(".").to_path_buf())
+            .join(resolved)
+    };
+    absolute.canonicalize().unwrap_or(absolute)
+}

crates/clusterflux-dap/src/variables.rs

@@ -0,0 +1,764 @@
+use std::collections::BTreeMap;
+use std::fs;
+
+use clusterflux_core::{Digest, TaskInstanceId};
+use serde_json::{json, Value};
+
+use crate::breakpoints::parse_rust_function_name;
+use crate::demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
+use crate::virtual_model::{AdapterState, VirtualThread};
+
+pub(crate) fn variables_response(state: &AdapterState, reference: i64) -> Value {
+    let Some(thread) = state.threads.values().find(|thread| {
+        thread.locals_ref == reference
+            || thread.wasm_locals_ref == reference
+            || thread.args_ref == reference
+            || thread.runtime_ref == reference
+            || thread.output_ref == reference
+            || thread.target_ref == reference
+            || thread.vfs_ref == reference
+            || thread.command_ref == reference
+    }) else {
+        return json!({ "variables": [] });
+    };
+
+    if thread.locals_ref == reference {
+        return json!({ "variables": source_locals_variables(state, thread) });
+    }
+
+    if thread.wasm_locals_ref == reference {
+        return json!({ "variables": wasm_frame_local_variables(state, thread) });
+    }
+
+    if thread.args_ref == reference {
+        let mut variables = thread
+            .runtime_task_args
+            .iter()
+            .map(|(name, value)| {
+                json!({
+                    "name": name,
+                    "value": value,
+                    "type": "task-argument",
+                    "variablesReference": 0,
+                })
+            })
+            .chain(thread.runtime_handles.iter().map(|(name, value)| {
+                json!({
+                    "name": name,
+                    "value": value,
+                    "type": "runtime-handle",
+                    "variablesReference": 0,
+                })
+            }))
+            .collect::<Vec<_>>();
+        if variables.is_empty() {
+            variables.push(json!({
+                "name": "runtime-boundary-diagnostic",
+                "value": "the frozen runtime participant reported no task arguments or handles at this probe",
+                "type": "diagnostic",
+                "variablesReference": 0,
+            }));
+        }
+        return json!({ "variables": variables });
+    }
+
+    if thread.target_ref == reference {
+        let runtime_backed =
+            state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated;
+        return json!({
+            "variables": [
+                {
+                    "name": "task",
+                    "value": thread.task.to_string(),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "attempt",
+                    "value": thread.attempt_id,
+                    "variablesReference": 0
+                },
+                {
+                    "name": "environment",
+                    "value": if runtime_backed { thread.runtime_environment.as_deref().unwrap_or("unknown") } else { task_environment(thread) },
+                    "variablesReference": 0
+                },
+                {
+                    "name": "kind",
+                    "value": if thread.id == MAIN_THREAD { "wasm entrypoint" } else { "wasm task" },
+                    "variablesReference": 0
+                },
+                {
+                    "name": "required_capability",
+                    "value": if runtime_backed { "not reported by the frozen node participant" } else if thread.id == MAIN_THREAD { "none" } else { "Command" },
+                    "variablesReference": 0
+                }
+            ]
+        });
+    }
+
+    if thread.vfs_ref == reference {
+        if state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated {
+            return json!({
+                "variables": [
+                {
+                    "name": "runtime-vfs-diagnostic",
+                    "value": thread.runtime_vfs_checkpoint.as_deref().unwrap_or("the coordinator has no current VFS checkpoint identity"),
+                        "variablesReference": 0
+                    },
+                    {
+                        "name": "reported_artifact_path",
+                        "value": state.runtime_artifact_path.as_deref().unwrap_or("unavailable"),
+                        "variablesReference": 0
+                    }
+                ]
+            });
+        }
+        return json!({
+            "variables": [
+                {
+                    "name": "/vfs/artifacts",
+                    "value": artifact_handle_value(state, thread),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "/vfs/sources",
+                    "value": format!("source://local-checkout/{}", project_snapshot_suffix(&state.project)),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "/vfs/blobs",
+                    "value": blob_digest(state, thread),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "durability",
+                    "value": "best-effort until explicitly exported or stored",
+                    "variablesReference": 0
+                }
+            ]
+        });
+    }
+
+    if thread.runtime_ref == reference {
+        return json!({
+            "variables": [
+                {
+                    "name": "virtual_process_id",
+                    "value": state.process.to_string(),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "virtual_thread",
+                    "value": thread.task.to_string(),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "task_attempt_id",
+                    "value": thread.attempt_id,
+                    "variablesReference": 0
+                },
+                {
+                    "name": "node",
+                    "value": thread.runtime_node.as_deref().unwrap_or("unknown"),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "restart_compatible",
+                    "value": thread.restart_compatible.map_or("unknown".to_owned(), |value| value.to_string()),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "debug_epoch",
+                    "value": state.epoch,
+                    "variablesReference": 0
+                },
+                {
+                    "name": "state",
+                    "value": format!("{:?}", thread.state),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "runtime_backend",
+                    "value": format!("{:?}", state.runtime_backend),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "coordinator_task_events",
+                    "value": state.runtime_event_count,
+                    "variablesReference": 0
+                },
+                {
+                    "name": "command_status",
+                    "value": thread.runtime_command_status.as_deref().unwrap_or(&state.command_status),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "command_spec",
+                    "value": command_spec_summary(state, thread),
+                    "variablesReference": if thread.runtime_command_status.is_some() { thread.command_ref } else { 0 }
+                },
+                {
+                    "name": "stdout_tail",
+                    "value": output_tail_display(&thread.stdout_tail, thread.stdout_bytes, thread.stdout_truncated, "stdout"),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "stderr_tail",
+                    "value": output_tail_display(&thread.stderr_tail, thread.stderr_bytes, thread.stderr_truncated, "stderr"),
+                    "variablesReference": 0
+                }
+            ]
+        });
+    }
+
+    if thread.command_ref == reference {
+        if state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated {
+            return json!({
+                "variables": [
+                    {
+                        "name": "status",
+                        "value": thread.runtime_command_status.as_deref().unwrap_or("no native command state was reported by this participant"),
+                        "variablesReference": 0
+                    },
+                    {
+                        "name": "command-spec-diagnostic",
+                        "value": "the node debug snapshot does not yet report native command program, arguments, or capability requirements",
+                        "variablesReference": 0
+                    }
+                ]
+            });
+        }
+        return json!({
+            "variables": [
+                {
+                    "name": "program",
+                    "value": command_program(thread),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "args",
+                    "value": command_args_summary(state, thread),
+                    "variablesReference": 0
+                },
+                {
+                    "name": "required_capability",
+                    "value": if thread.id == MAIN_THREAD { "none" } else { "Command" },
+                    "variablesReference": 0
+                },
+                {
+                    "name": "status",
+                    "value": state.command_status,
+                    "variablesReference": 0
+                }
+            ]
+        });
+    }
+
+    let mut variables = vec![
+        json!({
+            "name": "stdout_tail",
+            "value": output_tail_display(&thread.stdout_tail, thread.stdout_bytes, thread.stdout_truncated, "stdout"),
+            "variablesReference": 0
+        }),
+        json!({
+            "name": "stderr_tail",
+            "value": output_tail_display(&thread.stderr_tail, thread.stderr_bytes, thread.stderr_truncated, "stderr"),
+            "variablesReference": 0
+        }),
+    ];
+    variables.extend(
+        thread
+            .recent_output
+            .iter()
+            .enumerate()
+            .map(|(index, output)| {
+                json!({
+                    "name": format!("line_{index}"),
+                    "value": output,
+                    "variablesReference": 0
+                })
+            }),
+    );
+
+    json!({ "variables": variables })
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct SourceLocal {
+    name: String,
+    line: i64,
+    value: Option<String>,
+}
+
+fn source_locals_variables(state: &AdapterState, thread: &VirtualThread) -> Vec<Value> {
+    let (locals, diagnostic) = source_locals_for_thread(state, thread);
+    let mut variables = locals
+        .into_iter()
+        .map(|local| {
+            let has_value = local.value.is_some();
+            let value = local.value.unwrap_or_else(|| {
+                format!(
+                    "cannot be inspected: source-level local from line {} is known, but DWARF/runtime value snapshot is unavailable",
+                    local.line
+                )
+            });
+            json!({
+                "name": local.name,
+                "value": value,
+                "type": if has_value { "clusterflux-source-local" } else { "unavailable-local" },
+                "variablesReference": 0
+            })
+        })
+        .collect::<Vec<_>>();
+    variables.push(json!({
+        "name": "unavailable-local-diagnostic",
+        "value": diagnostic,
+        "type": "unavailable-local",
+        "variablesReference": 0
+    }));
+    variables
+}
+
+fn wasm_frame_local_variables(_state: &AdapterState, thread: &VirtualThread) -> Vec<Value> {
+    if thread.wasm_local_values.is_empty() {
+        return vec![json!({
+            "name": "wasm-local-diagnostic",
+            "value": "the frozen node participant did not report inspectable Wasm frame locals at this safepoint",
+            "type": "unavailable-local",
+            "variablesReference": 0
+        })];
+    }
+    thread
+        .wasm_local_values
+        .iter()
+        .map(|(name, value)| {
+            json!({
+                "name": name,
+                "value": value,
+                "type": "wasm-frame-local",
+                "variablesReference": 0
+            })
+        })
+        .collect()
+}
+
+fn source_locals_for_thread(
+    state: &AdapterState,
+    thread: &VirtualThread,
+) -> (Vec<SourceLocal>, String) {
+    if state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated && thread.line <= 0
+    {
+        return (
+            Vec::new(),
+            "cannot be inspected: the coordinator has no current source location for this attempt"
+                .to_owned(),
+        );
+    }
+    let source_path = crate::source::resolve_source_path(&state.project, &state.source_path);
+    let Ok(source) = fs::read_to_string(&source_path) else {
+        return (
+            Vec::new(),
+            format!(
+                "cannot be inspected: source file `{}` is unavailable; task args and handles remain visible",
+                source_path.display()
+            ),
+        );
+    };
+    let lines = source.lines().collect::<Vec<_>>();
+    if lines.is_empty() {
+        return (
+            Vec::new(),
+            "cannot be inspected: source file is empty; task args and handles remain visible"
+                .to_owned(),
+        );
+    }
+    let current = usize::try_from(thread.line)
+        .ok()
+        .and_then(|line| line.checked_sub(1))
+        .unwrap_or(0)
+        .min(lines.len() - 1);
+    let function_start = lines[..=current]
+        .iter()
+        .rposition(|line| parse_rust_function_name(line).is_some())
+        .unwrap_or(0);
+    let mut locals = Vec::new();
+    let mut runtime_values = BTreeMap::new();
+    let mut index = function_start;
+    while index <= current {
+        let Some(name) = parse_let_binding_name(lines[index]) else {
+            index += 1;
+            continue;
+        };
+
+        let line = (index + 1) as i64;
+        let mut statement = lines[index].to_owned();
+        while !statement.contains(';') && index < current {
+            index += 1;
+            statement.push('\n');
+            statement.push_str(lines[index]);
+        }
+
+        let runtime_value =
+            infer_clusterflux_source_local_value(state, &statement, &runtime_values);
+        if let Some(value) = runtime_value.clone() {
+            runtime_values.insert(name.clone(), value);
+        }
+        locals.push(SourceLocal {
+            name,
+            line,
+            value: runtime_value.map(|value| value.display(state)),
+        });
+        index += 1;
+    }
+    let diagnostic = if locals.is_empty() {
+        "cannot be inspected: no source-level `let` locals are in scope for this frame; task args and handles remain visible"
+            .to_owned()
+    } else if locals.iter().any(|local| local.value.is_some()) {
+        "selected source-level Clusterflux API locals are populated from virtual-thread runtime state; non-Clusterflux locals remain best-effort until DWARF value snapshots are available"
+            .to_owned()
+    } else {
+        "cannot be inspected: selected source-level local names are best-effort until DWARF/runtime value snapshots are available"
+            .to_owned()
+    };
+    (locals, diagnostic)
+}
+
+fn parse_let_binding_name(line: &str) -> Option<String> {
+    let rest = line.trim_start().strip_prefix("let ")?;
+    let rest = rest
+        .trim_start()
+        .strip_prefix("mut ")
+        .unwrap_or(rest)
+        .trim_start();
+    let name = rest
+        .chars()
+        .take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
+        .collect::<String>();
+    (!name.is_empty()).then_some(name)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+enum SourceLocalRuntimeValue {
+    SourceSnapshot(String),
+    TaskHandle {
+        task: TaskInstanceId,
+        thread_id: i64,
+        env: String,
+        state: String,
+    },
+    ThreadId(i64),
+    Artifact(String),
+}
+
+impl SourceLocalRuntimeValue {
+    fn display(&self, _state: &AdapterState) -> String {
+        match self {
+            SourceLocalRuntimeValue::SourceSnapshot(digest) => {
+                format!("SourceSnapshot {{ digest = \"{digest}\" }}")
+            }
+            SourceLocalRuntimeValue::TaskHandle {
+                task,
+                thread_id,
+                env,
+                state,
+            } => format!(
+                "TaskHandle {{ task = \"{task}\", virtual_thread_id = {thread_id}, env = \"{env}\", state = {state} }}"
+            ),
+            SourceLocalRuntimeValue::ThreadId(thread_id) => thread_id.to_string(),
+            SourceLocalRuntimeValue::Artifact(artifact) => {
+                format!("Artifact {{ id = \"{artifact}\" }}")
+            }
+        }
+    }
+}
+
+fn infer_clusterflux_source_local_value(
+    state: &AdapterState,
+    statement: &str,
+    runtime_values: &BTreeMap<String, SourceLocalRuntimeValue>,
+) -> Option<SourceLocalRuntimeValue> {
+    if statement.contains("prepare_source_snapshot()") {
+        return Some(SourceLocalRuntimeValue::SourceSnapshot(
+            source_snapshot_digest_from_project(state).unwrap_or_else(|| {
+                format!(
+                    "source://local-checkout/{}",
+                    project_snapshot_suffix(&state.project)
+                )
+            }),
+        ));
+    }
+
+    if let Some(task_function) = extract_spawn_task_function(statement) {
+        let task_name = extract_string_argument(statement, ".name(");
+        let spawned_thread =
+            thread_for_spawn_statement(state, &task_function, task_name.as_deref())?;
+        return Some(SourceLocalRuntimeValue::TaskHandle {
+            task: spawned_thread.task.clone(),
+            thread_id: spawned_thread.id,
+            env: extract_env_name(statement)
+                .unwrap_or_else(|| task_environment(spawned_thread).to_owned()),
+            state: format!("{:?}", spawned_thread.state),
+        });
+    }
+
+    if let Some(receiver) = receiver_before_method(statement, ".virtual_thread_id()") {
+        if let Some(SourceLocalRuntimeValue::TaskHandle { thread_id, .. }) =
+            runtime_values.get(&receiver)
+        {
+            return Some(SourceLocalRuntimeValue::ThreadId(*thread_id));
+        }
+    }
+
+    if let Some(receiver) = receiver_before_method(statement, ".join()") {
+        if let Some(SourceLocalRuntimeValue::TaskHandle { thread_id, .. }) =
+            runtime_values.get(&receiver)
+        {
+            if let Some(thread) = state.threads.get(thread_id) {
+                return Some(SourceLocalRuntimeValue::Artifact(artifact_handle_value(
+                    state, thread,
+                )));
+            }
+        }
+    }
+
+    None
+}
+
+fn extract_spawn_task_function(statement: &str) -> Option<String> {
+    for marker in [
+        "clusterflux::spawn::async_task(",
+        "clusterflux::spawn::task(",
+    ] {
+        if let Some(args) = extract_call_arguments(statement, marker) {
+            return args.first().cloned();
+        }
+    }
+    for marker in [
+        "clusterflux::spawn::async_task_with_arg(",
+        "clusterflux::spawn::task_with_arg(",
+    ] {
+        if let Some(args) = extract_call_arguments(statement, marker) {
+            return args.get(1).cloned();
+        }
+    }
+    None
+}
+
+fn source_snapshot_digest_from_project(state: &AdapterState) -> Option<String> {
+    let source = fs::read_to_string(crate::source::resolve_source_path(
+        &state.project,
+        &state.source_path,
+    ))
+    .ok()?;
+    let digest_marker = source.find("digest:")?;
+    let after_digest = &source[digest_marker..];
+    let first_quote = after_digest.find('"')? + 1;
+    let rest = &after_digest[first_quote..];
+    let end_quote = rest.find('"')?;
+    Some(rest[..end_quote].to_owned())
+}
+
+fn extract_call_arguments(statement: &str, marker: &str) -> Option<Vec<String>> {
+    let start = statement.find(marker)? + marker.len();
+    let rest = &statement[start..];
+    let mut args = Vec::new();
+    let mut current = String::new();
+    let mut depth = 0_i32;
+    let mut in_string = false;
+    let mut escaped = false;
+
+    for ch in rest.chars() {
+        if in_string {
+            current.push(ch);
+            if escaped {
+                escaped = false;
+            } else if ch == '\\' {
+                escaped = true;
+            } else if ch == '"' {
+                in_string = false;
+            }
+            continue;
+        }
+
+        match ch {
+            '"' => {
+                in_string = true;
+                current.push(ch);
+            }
+            '(' | '[' | '{' => {
+                depth += 1;
+                current.push(ch);
+            }
+            ')' => {
+                if depth == 0 {
+                    let value = current.trim();
+                    if !value.is_empty() {
+                        args.push(value.to_owned());
+                    }
+                    return (!args.is_empty()).then_some(args);
+                }
+                depth -= 1;
+                current.push(ch);
+            }
+            ']' | '}' => {
+                depth -= 1;
+                current.push(ch);
+            }
+            ',' if depth == 0 => {
+                let value = current.trim();
+                if !value.is_empty() {
+                    args.push(value.to_owned());
+                }
+                current.clear();
+            }
+            _ => current.push(ch),
+        }
+    }
+
+    None
+}
+
+fn extract_string_argument(statement: &str, marker: &str) -> Option<String> {
+    let start = statement.find(marker)? + marker.len();
+    let rest = &statement[start..];
+    let quoted = rest.strip_prefix('"')?;
+    let end = quoted.find('"')?;
+    Some(quoted[..end].to_owned())
+}
+
+fn extract_env_name(statement: &str) -> Option<String> {
+    if statement.contains("windows_env") || statement.contains("env!(\"windows\")") {
+        return Some("windows".to_owned());
+    }
+    if statement.contains("linux_command_env") || statement.contains("env!(\"linux-command\")") {
+        return Some("linux-command".to_owned());
+    }
+    if statement.contains("linux_env") || statement.contains("env!(\"linux\")") {
+        return Some("linux".to_owned());
+    }
+    if statement.contains("coordinator_env") || statement.contains("env!(\"coordinator\")") {
+        return Some("coordinator".to_owned());
+    }
+    None
+}
+
+fn receiver_before_method(statement: &str, method: &str) -> Option<String> {
+    let before = statement.split(method).next()?.trim_end();
+    let receiver = before
+        .chars()
+        .rev()
+        .take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
+        .collect::<String>()
+        .chars()
+        .rev()
+        .collect::<String>();
+    (!receiver.is_empty()).then_some(receiver)
+}
+
+fn thread_for_spawn_statement<'a>(
+    state: &'a AdapterState,
+    task_function: &str,
+    task_name: Option<&str>,
+) -> Option<&'a VirtualThread> {
+    if let Some(task_name) = task_name {
+        if let Some(thread) = state
+            .threads
+            .values()
+            .find(|thread| thread.name == task_name)
+        {
+            return Some(thread);
+        }
+    }
+    let normalized = task_function.replace('_', "-");
+    state.threads.values().find(|thread| {
+        thread.task.as_str() == normalized
+            || (thread.id == PACKAGE_THREAD && normalized == "package-release")
+    })
+}
+
+fn task_environment(thread: &VirtualThread) -> &'static str {
+    match thread.id {
+        WINDOWS_THREAD => "windows",
+        MAIN_THREAD => "coordinator",
+        PACKAGE_THREAD => "coordinator",
+        LINUX_THREAD => "linux",
+        _ => "unconstrained",
+    }
+}
+
+fn artifact_handle_value(state: &AdapterState, thread: &VirtualThread) -> String {
+    if state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated {
+        return state.runtime_artifact_path.clone().unwrap_or_else(|| {
+            "unavailable: runtime participant did not report an artifact handle".to_owned()
+        });
+    }
+    if thread.id == LINUX_THREAD {
+        return state.runtime_artifact_path.clone().unwrap_or_else(|| {
+            format!("artifact://{}/{}/app.tar.zst", state.process, thread.task)
+        });
+    }
+    if thread.id == PACKAGE_THREAD {
+        return "artifact://package/release.tar.zst".to_owned();
+    }
+    format!("artifact://{}/{}/app.tar.zst", state.process, thread.task)
+}
+
+fn blob_digest(state: &AdapterState, thread: &VirtualThread) -> String {
+    Digest::from_parts([
+        b"blob:v1".as_slice(),
+        state.project.as_bytes(),
+        thread.task.as_str().as_bytes(),
+    ])
+    .as_str()
+    .to_owned()
+}
+
+fn command_spec_summary(state: &AdapterState, thread: &VirtualThread) -> String {
+    if state.runtime_backend != crate::virtual_model::RuntimeBackend::Simulated {
+        return thread
+            .runtime_command_status
+            .clone()
+            .unwrap_or_else(|| "no native command state reported".to_owned());
+    }
+    format!(
+        "{} {} ({})",
+        command_program(thread),
+        command_args_summary(state, thread),
+        state.command_status
+    )
+}
+
+fn command_program(thread: &VirtualThread) -> &'static str {
+    if thread.id == MAIN_THREAD {
+        "coordinator-side wasm"
+    } else {
+        "cargo"
+    }
+}
+
+fn command_args_summary(state: &AdapterState, thread: &VirtualThread) -> String {
+    if thread.id == MAIN_THREAD {
+        return format!("entry={}", state.entry);
+    }
+    format!("test --quiet --manifest-path {}/Cargo.toml", state.project)
+}
+
+fn output_tail_display(tail: &str, bytes: u64, truncated: bool, stream: &str) -> String {
+    if tail.is_empty() {
+        return format!("no {stream} captured ({bytes} bytes)");
+    }
+    if truncated {
+        format!("{tail}\n... truncated")
+    } else {
+        tail.to_owned()
+    }
+}
+
+fn project_snapshot_suffix(project: &str) -> String {
+    Digest::from_parts([b"source-snapshot:v1".as_slice(), project.as_bytes()])
+        .as_str()
+        .trim_start_matches("sha256:")
+        .chars()
+        .take(12)
+        .collect()
+}

crates/clusterflux-dap/src/view_state.rs

@@ -0,0 +1,89 @@
+use std::fs;
+use std::path::Path;
+
+use anyhow::Result;
+use serde_json::json;
+
+use crate::virtual_model::{AdapterState, RuntimeLaunchRecord};
+
+pub(crate) fn normalize_coordinator_endpoint(endpoint: &str) -> String {
+    endpoint.trim().trim_end_matches('/').to_owned()
+}
+
+pub(crate) fn write_view_state(state: &AdapterState, record: &RuntimeLaunchRecord) -> Result<()> {
+    let dir = Path::new(&state.project).join(".clusterflux");
+    fs::create_dir_all(&dir)?;
+    let node_status = if record.placed_task_launched {
+        if record.status_code == Some(0) {
+            "completed"
+        } else {
+            "failed"
+        }
+    } else {
+        "not-required-yet"
+    };
+    let process_status = if record.placed_task_launched {
+        if record.status_code == Some(0) {
+            "completed"
+        } else {
+            "failed"
+        }
+    } else {
+        "running"
+    };
+    let log_message = if record.placed_task_launched {
+        "node task output was staged as a VFS artifact"
+    } else {
+        "coordinator-side virtual process started; capability task not launched"
+    };
+    let artifact_status = if record.placed_task_launched {
+        "best-effort retained on the attached node"
+    } else {
+        "not produced until a capability task is placed"
+    };
+    let view_state = json!({
+        "nodes": [{
+            "id": record.node,
+            "status": node_status,
+            "capabilities": format!("connected to {}", record.coordinator),
+        }],
+        "processes": [{
+            "id": state.process.as_str(),
+            "status": process_status,
+            "entry": state.entry,
+        }],
+        "logs": [{
+            "task": "compile-linux",
+            "message": log_message,
+            "bytes": format!("stdout={} stderr={}", record.stdout_bytes, record.stderr_bytes),
+        }],
+        "artifacts": [{
+            "path": record.artifact_path.clone().unwrap_or_else(|| "/vfs/artifacts/dap-output.txt".to_owned()),
+            "status": artifact_status,
+            "size": "reported by node",
+        }],
+        "inspector": [
+            {
+                "label": "Runtime backend",
+                "value": state.runtime_backend.description(),
+            },
+            {
+                "label": "Coordinator",
+                "value": record.coordinator,
+            },
+            {
+                "label": "Node report",
+                "value": record.node_report.to_string(),
+            },
+            {
+                "label": "Task events",
+                "value": record.task_events.to_string(),
+            }
+        ]
+    });
+    fs::write(
+        dir.join("views.json"),
+        serde_json::to_string_pretty(&view_state)?,
+    )?;
+    Ok(())
+}