Public release release-5792a1a4e1cb

Source commit: 5792a1a4e1cb24ff71b1535d4b582ef980880752

Public tree identity: sha256:eea1f740446827ef10e4892923c13f13077c9e03438f2bf53d7f90536fb4cf95

crates/clusterflux-coordinator/src/service/admin.rs

@@ -0,0 +1,147 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    admin_request_proof_from_token_digest, CredentialKind, Digest, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+const ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS: u64 = 300;
+
+impl CoordinatorService {
+    pub(super) fn handle_admin_status(
+        &mut self,
+        tenant: String,
+        actor_user: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.verify_admin_request(
+            "admin_status",
+            &tenant,
+            &actor_user,
+            &tenant,
+            &admin_proof,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        )?;
+        let tenant = TenantId::new(tenant);
+        let actor = UserId::new(actor_user);
+        Ok(CoordinatorResponse::AdminStatus {
+            suspended: self.coordinator.tenant_suspended(&tenant),
+            tenant,
+            actor,
+            safe_default: "read_only".to_owned(),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_suspend_tenant(
+        &mut self,
+        tenant: String,
+        actor_user: String,
+        target_tenant: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.verify_admin_request(
+            "suspend_tenant",
+            &tenant,
+            &actor_user,
+            &target_tenant,
+            &admin_proof,
+            &admin_nonce,
+            issued_at_epoch_seconds,
+        )?;
+        let actor_tenant = TenantId::new(tenant);
+        let actor = UserId::new(actor_user);
+        let target_tenant = TenantId::new(target_tenant);
+        self.coordinator.upsert_tenant(actor_tenant.clone());
+        self.coordinator.upsert_user(
+            actor_tenant,
+            actor.clone(),
+            CredentialKind::CliDeviceSession,
+        );
+        let policy = self
+            .coordinator
+            .suspend_tenant(target_tenant.clone(), actor.clone());
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::TenantSuspended {
+            tenant: target_tenant,
+            actor,
+            policy,
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn verify_admin_request(
+        &mut self,
+        operation: &str,
+        tenant: &str,
+        actor_user: &str,
+        target_tenant: &str,
+        admin_proof: &Digest,
+        admin_nonce: &str,
+        issued_at_epoch_seconds: u64,
+    ) -> Result<(), CoordinatorServiceError> {
+        let expected = self.admin_token_digest.as_ref().ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "self-hosted admin credential is not configured".to_owned(),
+            )
+        })?;
+        if admin_nonce.trim().is_empty() || admin_nonce.len() > 256 {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request nonce is missing or invalid".to_owned(),
+            )
+            .into());
+        }
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|duration| duration.as_secs())
+            .unwrap_or_default();
+        if now.abs_diff(issued_at_epoch_seconds) > ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request timestamp is outside the allowed 300-second window".to_owned(),
+            )
+            .into());
+        }
+        let expected_proof = admin_request_proof_from_token_digest(
+            expected,
+            operation,
+            tenant,
+            actor_user,
+            target_tenant,
+            admin_nonce,
+            issued_at_epoch_seconds,
+        );
+        if admin_proof != &expected_proof {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request proof is invalid".to_owned(),
+            )
+            .into());
+        }
+        self.admin_replay_nonces.retain(|_, issued_at| {
+            now <= issued_at.saturating_add(ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS)
+        });
+        if self.admin_replay_nonces.contains_key(admin_nonce) {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request nonce was already used".to_owned(),
+            )
+            .into());
+        }
+        if self.admin_replay_nonces.len() >= super::MAX_REPLAY_NONCES_PER_AUTHORITY {
+            return Err(CoordinatorError::Unauthorized(
+                "admin request replay window is full; retry after the bounded signature window advances"
+                    .to_owned(),
+            )
+            .into());
+        }
+        self.admin_replay_nonces
+            .insert(admin_nonce.to_owned(), issued_at_epoch_seconds);
+        Ok(())
+    }
+}

crates/clusterflux-coordinator/src/service/artifacts.rs

@@ -0,0 +1,799 @@
+use std::io::{Read, Seek, SeekFrom, Write};
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    generate_opaque_token, Actor, ArtifactId, AuthContext, DataPlaneObject, DataPlaneScope, Digest,
+    DownloadPolicy, NodeEndpoint, NodeId, ProjectId, RendezvousRequest, ResourceLimits,
+    ResourceMeter, StorageLocation, TenantId, UserId,
+};
+use sha2::{Digest as _, Sha256};
+
+use crate::CoordinatorError;
+
+use super::relay::RelayFinishReason;
+use super::{
+    bounded_ttl, ArtifactTransferAssignment, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+pub(super) const MAX_ARTIFACT_REVERSE_CHUNK_BYTES: u64 = 256 * 1024;
+
+#[derive(Debug)]
+pub(super) struct ArtifactReverseTransfer {
+    transfer_id: String,
+    token_digest: Digest,
+    tenant: TenantId,
+    project: ProjectId,
+    source_node: NodeId,
+    artifact: ArtifactId,
+    expected_digest: Digest,
+    expected_size_bytes: u64,
+    expires_at_epoch_seconds: u64,
+    spool: tempfile::NamedTempFile,
+    received_bytes: u64,
+    content_hasher: Sha256,
+    delivered_offset: u64,
+    error: Option<String>,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_create_artifact_download_link(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        ttl_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let policy = DownloadPolicy { max_bytes };
+        let action = self
+            .artifact_registry
+            .download_action(&context, &artifact, &policy)?;
+        self.ensure_download_source_connectivity(&action.source)?;
+        let downloadable_size = self
+            .artifact_registry
+            .downloadable_size(&context, &artifact, &policy)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.mutate_artifact_relay(|ledger| {
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })?;
+        self.quota.can_charge_download(
+            &context.tenant,
+            &context.project,
+            downloadable_size,
+            now_epoch_seconds,
+        )?;
+        let token_nonce = generate_opaque_token("artifact_download")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let ttl_seconds = bounded_ttl(
+            ttl_seconds,
+            self.admission.max_artifact_download_ttl_seconds,
+        );
+        let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
+        let account = match &context.actor {
+            Actor::User(user) => user.clone(),
+            Actor::Agent(_) | Actor::Node(_) | Actor::Task(_) => {
+                return Err(CoordinatorServiceError::Protocol(
+                    "artifact relay download requires a user account".to_owned(),
+                ))
+            }
+        };
+        let mut relay_candidate = self.artifact_relay.clone();
+        relay_candidate
+            .reserve(
+                token_nonce.clone(),
+                context.tenant.clone(),
+                context.project.clone(),
+                account,
+                downloadable_size,
+                MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
+                expires_at_epoch_seconds,
+                now_epoch_seconds,
+            )
+            .map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
+        let link = match self.artifact_registry.create_download_link(
+            &context,
+            &artifact,
+            &policy,
+            &token_nonce,
+            now_epoch_seconds,
+            ttl_seconds,
+        ) {
+            Ok(link) => link,
+            Err(error) => return Err(error.into()),
+        };
+        if let Err(error) =
+            relay_candidate.rekey(&token_nonce, link.scoped_token_digest.as_str().to_owned())
+        {
+            let _ = self.artifact_registry.revoke_download_link(
+                &context,
+                &artifact,
+                &link.scoped_token_digest,
+            );
+            return Err(CoordinatorServiceError::Protocol(error.to_string()));
+        }
+        if let Err(error) = self.commit_artifact_relay(relay_candidate) {
+            let _ = self.artifact_registry.revoke_download_link(
+                &context,
+                &artifact,
+                &link.scoped_token_digest,
+            );
+            return Err(error);
+        }
+        Ok(CoordinatorResponse::ArtifactDownloadLink { link })
+    }
+
+    pub(super) fn handle_open_artifact_download_stream(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let policy = DownloadPolicy { max_bytes };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.mutate_artifact_relay(|ledger| {
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let downloadable_size = self
+            .artifact_registry
+            .downloadable_size(&context, &artifact, &policy)?;
+        let validation_limits = ResourceLimits::unlimited();
+        let mut validation_meter = ResourceMeter::default();
+        let mut stream = self.artifact_registry.open_download_stream(
+            clusterflux_core::DownloadStreamRequest {
+                context: &context,
+                artifact: &artifact,
+                policy: &policy,
+                presented_token_digest: &token_digest,
+                now_epoch_seconds,
+                limits: &validation_limits,
+            },
+            &mut validation_meter,
+        )?;
+        self.ensure_download_source_connectivity(&stream.link.source)?;
+        self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
+
+        if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
+            if let Some(message) = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .and_then(|transfer| transfer.error.clone())
+            {
+                self.artifact_reverse_transfers.remove(&transfer_id);
+                self.artifact_transfer_by_token.remove(&token_digest);
+                self.mutate_artifact_relay(|ledger| {
+                    ledger.finish(token_digest.as_str(), RelayFinishReason::Failed);
+                    Ok(())
+                })?;
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "retaining node could not stream artifact: {message}"
+                )));
+            }
+            let (content_offset, content, end, complete) = {
+                let transfer = self
+                    .artifact_reverse_transfers
+                    .get_mut(&transfer_id)
+                    .ok_or_else(|| {
+                        CoordinatorServiceError::Protocol(
+                            "artifact reverse transfer index is inconsistent".to_owned(),
+                        )
+                    })?;
+                if transfer.tenant != context.tenant
+                    || transfer.project != context.project
+                    || transfer.artifact != artifact
+                    || transfer.token_digest != token_digest
+                {
+                    return Err(clusterflux_core::DownloadError::InvalidToken.into());
+                }
+                if transfer.received_bytes != transfer.expected_size_bytes {
+                    let overhead = self.artifact_relay.framing_overhead_bytes();
+                    self.mutate_artifact_relay(|ledger| {
+                        ledger.charge_egress(token_digest.as_str(), overhead, now_epoch_seconds)
+                    })?;
+                    return Ok(CoordinatorResponse::ArtifactDownloadStream {
+                        link: stream.link,
+                        streamed_bytes: 0,
+                        charged_download_bytes: self.quota.used_download_bytes(
+                            &context.tenant,
+                            &context.project,
+                            now_epoch_seconds,
+                        ),
+                        content_bytes_available: false,
+                        content_offset: None,
+                        content_eof: false,
+                        content_base64: None,
+                        content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
+                    });
+                }
+                if chunk_bytes == 0 && downloadable_size != 0 {
+                    return Err(CoordinatorServiceError::Protocol(
+                        "artifact download chunk_bytes must be greater than zero".to_owned(),
+                    ));
+                }
+                let requested = chunk_bytes
+                    .min(downloadable_size)
+                    .min(MAX_ARTIFACT_REVERSE_CHUNK_BYTES);
+                let start = transfer.delivered_offset;
+                let end = start.saturating_add(requested).min(transfer.received_bytes);
+                let length = usize::try_from(end.saturating_sub(start)).map_err(|_| {
+                    CoordinatorServiceError::Protocol(
+                        "artifact download chunk length does not fit memory bounds".to_owned(),
+                    )
+                })?;
+                let mut content = vec![0_u8; length];
+                let mut spool = transfer.spool.reopen().map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "open bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                spool.seek(SeekFrom::Start(start)).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "seek bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                spool.read_exact(&mut content).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "read bounded artifact transfer spool: {error}"
+                    ))
+                })?;
+                (start, content, end, end == transfer.received_bytes)
+            };
+            let streamed_bytes = content.len() as u64;
+            self.artifact_registry.stream_download_chunk(
+                &mut stream,
+                &validation_limits,
+                &mut validation_meter,
+                streamed_bytes,
+            )?;
+            let charged_download_bytes = self.quota.charge_download(
+                &context.tenant,
+                &context.project,
+                streamed_bytes,
+                now_epoch_seconds,
+            )?;
+            let content_base64 = BASE64_STANDARD.encode(content);
+            let egress_wire_bytes = (content_base64.len() as u64)
+                .saturating_add(self.artifact_relay.framing_overhead_bytes());
+            self.mutate_artifact_relay(|ledger| {
+                ledger.charge_egress(
+                    token_digest.as_str(),
+                    egress_wire_bytes,
+                    now_epoch_seconds,
+                )?;
+                if complete {
+                    ledger.finish(token_digest.as_str(), RelayFinishReason::Completed);
+                }
+                Ok(())
+            })?;
+            if let Some(transfer) = self.artifact_reverse_transfers.get_mut(&transfer_id) {
+                transfer.delivered_offset = end;
+            }
+            if complete {
+                self.artifact_reverse_transfers.remove(&transfer_id);
+                self.artifact_transfer_by_token.remove(&token_digest);
+            }
+            return Ok(CoordinatorResponse::ArtifactDownloadStream {
+                link: stream.link,
+                streamed_bytes,
+                charged_download_bytes,
+                content_bytes_available: true,
+                content_offset: Some(content_offset),
+                content_eof: complete,
+                content_base64: Some(content_base64),
+                content_source: Some("retaining_node_reverse_stream".to_owned()),
+            });
+        }
+
+        let StorageLocation::RetainedNode(source_node) = &stream.link.source else {
+            return Err(clusterflux_core::DownloadError::Unavailable.into());
+        };
+        let metadata = self
+            .artifact_registry
+            .metadata(&artifact)
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        let transfer_id = generate_opaque_token("artifact_transfer")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let spool = tempfile::Builder::new()
+            .prefix("clusterflux-artifact-transfer-")
+            .tempfile()
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "create bounded artifact transfer spool: {error}"
+                ))
+            })?;
+        let transfer = ArtifactReverseTransfer {
+            transfer_id: transfer_id.clone(),
+            token_digest: token_digest.clone(),
+            tenant: context.tenant.clone(),
+            project: context.project.clone(),
+            source_node: source_node.clone(),
+            artifact,
+            expected_digest: metadata.digest.clone(),
+            expected_size_bytes: metadata.size,
+            expires_at_epoch_seconds: stream.link.expires_at_epoch_seconds,
+            spool,
+            received_bytes: 0,
+            content_hasher: Sha256::new(),
+            delivered_offset: 0,
+            error: None,
+        };
+        let overhead = self.artifact_relay.framing_overhead_bytes();
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_egress(
+                stream.link.scoped_token_digest.as_str(),
+                overhead,
+                now_epoch_seconds,
+            )
+        })?;
+        self.artifact_reverse_transfers
+            .insert(transfer_id.clone(), transfer);
+        self.artifact_transfer_by_token
+            .insert(token_digest, transfer_id);
+        Ok(CoordinatorResponse::ArtifactDownloadStream {
+            link: stream.link,
+            streamed_bytes: 0,
+            charged_download_bytes: self.quota.used_download_bytes(
+                &context.tenant,
+                &context.project,
+                now_epoch_seconds,
+            ),
+            content_bytes_available: false,
+            content_offset: None,
+            content_eof: false,
+            content_base64: None,
+            content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
+        })
+    }
+
+    pub(super) fn handle_revoke_artifact_download_link(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        token_digest: Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let link = self.artifact_registry.revoke_download_link(
+            &context,
+            &ArtifactId::new(artifact),
+            &token_digest,
+        )?;
+        if let Some(transfer_id) = self.artifact_transfer_by_token.remove(&token_digest) {
+            self.artifact_reverse_transfers.remove(&transfer_id);
+        }
+        self.mutate_artifact_relay(|ledger| {
+            ledger.finish(token_digest.as_str(), RelayFinishReason::Cancelled);
+            Ok(())
+        })?;
+        Ok(CoordinatorResponse::ArtifactDownloadLinkRevoked { link })
+    }
+
+    pub(super) fn handle_export_artifact_to_node(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = user_context(tenant, project, actor_user);
+        let artifact = ArtifactId::new(artifact);
+        let receiver_node = NodeId::new(receiver_node);
+        let action = self.artifact_registry.download_action(
+            &context,
+            &artifact,
+            &DownloadPolicy {
+                max_bytes: u64::MAX,
+            },
+        )?;
+        let StorageLocation::RetainedNode(source_node) = action.source else {
+            return Err(clusterflux_core::DownloadError::Unavailable.into());
+        };
+        let metadata = self
+            .artifact_registry
+            .metadata(&artifact)
+            .ok_or(clusterflux_core::DownloadError::NotFound)?;
+        let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
+        let destination =
+            self.export_endpoint(&receiver_node, &context.tenant, &context.project)?;
+        let plan = self.transport.plan_authenticated_direct_bulk_transfer(
+            RendezvousRequest {
+                scope: DataPlaneScope {
+                    tenant: context.tenant.clone(),
+                    project: context.project.clone(),
+                    process: metadata.process.clone(),
+                    object: DataPlaneObject::Artifact(artifact.clone()),
+                    authorization_subject: format!(
+                        "artifact-export:{}-to-{}",
+                        source_node, receiver_node
+                    ),
+                },
+                source,
+                destination,
+            },
+            direct_connectivity,
+            failure_reason,
+        )?;
+        Ok(CoordinatorResponse::ArtifactExportPlan {
+            plan,
+            source_node,
+            receiver_node,
+            artifact_size_bytes: metadata.size,
+        })
+    }
+
+    pub(super) fn handle_poll_artifact_transfer(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let transfer = self.artifact_reverse_transfers.values().find(|transfer| {
+            transfer.tenant == tenant
+                && transfer.project == project
+                && transfer.source_node == node
+                && transfer.error.is_none()
+                && transfer.received_bytes < transfer.expected_size_bytes
+        });
+        Ok(CoordinatorResponse::ArtifactTransferAssignment {
+            transfer: transfer.map(|transfer| ArtifactTransferAssignment {
+                transfer_id: transfer.transfer_id.clone(),
+                artifact: transfer.artifact.clone(),
+                expected_digest: transfer.expected_digest.clone(),
+                expected_size_bytes: transfer.expected_size_bytes,
+                offset: transfer.received_bytes,
+                max_chunk_bytes: MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
+            }),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_upload_artifact_transfer_chunk(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        offset: u64,
+        content_base64: String,
+        chunk_digest: Digest,
+        eof: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let token_digest = {
+            let transfer = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(
+                        "unknown artifact reverse transfer".to_owned(),
+                    )
+                })?;
+            if transfer.tenant != tenant
+                || transfer.project != project
+                || transfer.source_node != node
+                || transfer.artifact.as_str() != artifact
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "artifact reverse transfer is outside the signed node scope".to_owned(),
+                )
+                .into());
+            }
+            transfer.token_digest.clone()
+        };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let ingress_wire_bytes = (content_base64.len() as u64)
+            .saturating_add(self.artifact_relay.framing_overhead_bytes());
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_ingress(token_digest.as_str(), ingress_wire_bytes, now_epoch_seconds)
+        })?;
+        let transfer = self
+            .artifact_reverse_transfers
+            .get_mut(&transfer_id)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
+            })?;
+        if transfer.tenant != tenant
+            || transfer.project != project
+            || transfer.source_node != node
+            || transfer.artifact.as_str() != artifact
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer is outside the signed node scope".to_owned(),
+            )
+            .into());
+        }
+        if transfer.received_bytes != offset {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "artifact reverse transfer expected offset {}, received {offset}",
+                transfer.received_bytes
+            )));
+        }
+        let content = BASE64_STANDARD.decode(content_base64).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "artifact reverse transfer chunk is not valid base64: {error}"
+            ))
+        })?;
+        if content.len() as u64 > MAX_ARTIFACT_REVERSE_CHUNK_BYTES {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer chunk exceeds the control-plane limit".to_owned(),
+            ));
+        }
+        if Digest::sha256(&content) != chunk_digest {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer chunk digest mismatch".to_owned(),
+            ));
+        }
+        let next_offset = offset.saturating_add(content.len() as u64);
+        if next_offset > transfer.expected_size_bytes {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer exceeds retained metadata size".to_owned(),
+            ));
+        }
+        let complete = next_offset == transfer.expected_size_bytes;
+        if eof != complete {
+            return Err(CoordinatorServiceError::Protocol(
+                "artifact reverse transfer EOF does not match retained metadata size".to_owned(),
+            ));
+        }
+        transfer
+            .spool
+            .as_file_mut()
+            .seek(SeekFrom::Start(offset))
+            .and_then(|_| transfer.spool.as_file_mut().write_all(&content))
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "write bounded artifact transfer spool: {error}"
+                ))
+            })?;
+        transfer.content_hasher.update(&content);
+        transfer.received_bytes = next_offset;
+        if complete {
+            transfer.spool.as_file().sync_all().map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "sync bounded artifact transfer spool: {error}"
+                ))
+            })?;
+            let digest_hex = format!("{:x}", transfer.content_hasher.clone().finalize());
+            let digest =
+                Digest::from_sha256_hex(&digest_hex).map_err(CoordinatorServiceError::Protocol)?;
+            if digest != transfer.expected_digest {
+                transfer.spool.as_file_mut().set_len(0).map_err(|error| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "reset invalid artifact transfer spool: {error}"
+                    ))
+                })?;
+                transfer.received_bytes = 0;
+                transfer.content_hasher = Sha256::new();
+                return Err(CoordinatorServiceError::Protocol(
+                    "artifact reverse transfer content digest mismatch".to_owned(),
+                ));
+            }
+        }
+        Ok(CoordinatorResponse::ArtifactTransferChunkAccepted {
+            transfer_id,
+            next_offset,
+            complete,
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_fail_artifact_transfer(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        message: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
+        let token_digest = {
+            let transfer = self
+                .artifact_reverse_transfers
+                .get(&transfer_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(
+                        "unknown artifact reverse transfer".to_owned(),
+                    )
+                })?;
+            if transfer.tenant != tenant
+                || transfer.project != project
+                || transfer.source_node != node
+                || transfer.artifact.as_str() != artifact
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "artifact reverse transfer failure is outside the signed node scope".to_owned(),
+                )
+                .into());
+            }
+            transfer.token_digest.clone()
+        };
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let failure_wire_bytes =
+            (message.len() as u64).saturating_add(self.artifact_relay.framing_overhead_bytes());
+        self.mutate_artifact_relay(|ledger| {
+            ledger.charge_ingress(token_digest.as_str(), failure_wire_bytes, now_epoch_seconds)
+        })?;
+        let transfer = self
+            .artifact_reverse_transfers
+            .get_mut(&transfer_id)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
+            })?;
+        if transfer.tenant != tenant
+            || transfer.project != project
+            || transfer.source_node != node
+            || transfer.artifact.as_str() != artifact
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer failure is outside the signed node scope".to_owned(),
+            )
+            .into());
+        }
+        let message = message.trim();
+        transfer.error = Some(if message.is_empty() {
+            "retained artifact bytes are unavailable".to_owned()
+        } else {
+            message.chars().take(1024).collect()
+        });
+        self.mutate_artifact_relay(|ledger| {
+            ledger.finish(token_digest.as_str(), RelayFinishReason::Failed);
+            Ok(())
+        })?;
+        Ok(CoordinatorResponse::ArtifactTransferFailed { transfer_id })
+    }
+
+    fn authorize_artifact_transfer_node(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        node: &NodeId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact reverse transfer node is outside its enrolled tenant/project scope"
+                    .to_owned(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    fn expire_artifact_reverse_transfers(
+        &mut self,
+        now_epoch_seconds: u64,
+    ) -> Result<(), CoordinatorServiceError> {
+        let expired = self
+            .artifact_reverse_transfers
+            .iter()
+            .filter(|(_, transfer)| transfer.expires_at_epoch_seconds < now_epoch_seconds)
+            .map(|(id, transfer)| (id.clone(), transfer.token_digest.clone()))
+            .collect::<Vec<_>>();
+        for (id, token) in &expired {
+            self.artifact_reverse_transfers.remove(id);
+            self.artifact_transfer_by_token.remove(token);
+        }
+        self.mutate_artifact_relay(|ledger| {
+            for (_, token) in &expired {
+                ledger.finish(token.as_str(), RelayFinishReason::Expired);
+            }
+            ledger.expire(now_epoch_seconds);
+            Ok(())
+        })
+    }
+
+    fn export_endpoint(
+        &self,
+        node: &NodeId,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> Result<NodeEndpoint, CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if &identity.tenant != tenant || &identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact export node is outside the tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+            clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                "node {node} has not reported export connectivity"
+            ))
+        })?;
+        if descriptor.tenant != *tenant || descriptor.project != *project {
+            return Err(CoordinatorError::Unauthorized(
+                "artifact export node descriptor is outside the tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        if !self.node_is_live(node) {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "node {node} is offline for artifact export"
+                ))
+                .into(),
+            );
+        }
+        if !descriptor.direct_connectivity {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "direct connectivity unavailable to node {node} for artifact export"
+                ))
+                .into(),
+            );
+        }
+        Ok(NodeEndpoint {
+            node: node.clone(),
+            advertised_addr: format!("{node}.mesh.invalid:4433"),
+            public_key_fingerprint: Digest::sha256(&identity.public_key),
+        })
+    }
+
+    fn ensure_download_source_connectivity(
+        &self,
+        source: &StorageLocation,
+    ) -> Result<(), clusterflux_core::DownloadError> {
+        let StorageLocation::RetainedNode(node) = source else {
+            return Ok(());
+        };
+        let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
+            clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                "retaining node {node} has not reported online status for artifact download"
+            ))
+        })?;
+        if !self.node_is_live(node) {
+            return Err(
+                clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
+                    "retaining node {node} is offline for artifact download"
+                )),
+            );
+        }
+        Ok(())
+    }
+}
+
+fn user_context(tenant: String, project: String, actor_user: String) -> AuthContext {
+    AuthContext {
+        tenant: TenantId::new(tenant),
+        project: ProjectId::new(project),
+        actor: Actor::User(UserId::new(actor_user)),
+    }
+}

crates/clusterflux-coordinator/src/service/authenticated.rs

@@ -0,0 +1,139 @@
+use clusterflux_core::{AuthContext, UserId};
+
+use super::{
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_authenticated_agent_key_request(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let request = match request {
+            AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { agent, public_key } => {
+                CoordinatorRequest::RegisterAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                    public_key,
+                }
+            }
+            AuthenticatedCoordinatorRequest::ListAgentPublicKeys => {
+                CoordinatorRequest::ListAgentPublicKeys {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                }
+            }
+            AuthenticatedCoordinatorRequest::RotateAgentPublicKey { agent, public_key } => {
+                CoordinatorRequest::RotateAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                    public_key,
+                }
+            }
+            AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => {
+                CoordinatorRequest::RevokeAgentPublicKey {
+                    tenant: context.tenant.as_str().to_owned(),
+                    project: context.project.as_str().to_owned(),
+                    user: actor.as_str().to_owned(),
+                    agent,
+                }
+            }
+            _ => unreachable!("caller filters authenticated agent key operations"),
+        };
+        self.handle_request(request)
+    }
+
+    pub(super) fn handle_authenticated_launch_task(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::LaunchTask {
+            task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        } = request
+        else {
+            unreachable!("caller filters authenticated task launches");
+        };
+        self.handle_launch_task(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            Some(actor.as_str().to_owned()),
+            None,
+            None,
+            None,
+            None,
+            *task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        )
+    }
+
+    pub(super) fn handle_authenticated_schedule_task(
+        &mut self,
+        context: &AuthContext,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::ScheduleTask {
+            environment,
+            environment_digest,
+            required_capabilities,
+            dependency_cache,
+            source_snapshot,
+            required_artifacts,
+            prefer_node,
+        } = request
+        else {
+            unreachable!("caller filters authenticated task scheduling");
+        };
+        self.handle_schedule_task(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            environment,
+            environment_digest,
+            required_capabilities,
+            dependency_cache,
+            source_snapshot,
+            required_artifacts,
+            prefer_node,
+        )
+    }
+
+    pub(super) fn handle_authenticated_artifact_export(
+        &mut self,
+        context: &AuthContext,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let AuthenticatedCoordinatorRequest::ExportArtifactToNode {
+            artifact,
+            receiver_node,
+            direct_connectivity,
+            failure_reason,
+        } = request
+        else {
+            unreachable!("caller filters authenticated artifact exports");
+        };
+        self.handle_export_artifact_to_node(
+            context.tenant.as_str().to_owned(),
+            context.project.as_str().to_owned(),
+            actor.as_str().to_owned(),
+            artifact,
+            receiver_node,
+            direct_connectivity,
+            failure_reason,
+        )
+    }
+}

crates/clusterflux-coordinator/src/service/authorization.rs

@@ -0,0 +1,204 @@
+use clusterflux_core::{Actor, AuthContext, UserId};
+
+use crate::CoordinatorError;
+
+use super::{AuthenticatedCoordinatorRequest, CoordinatorServiceError};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum PublicUserOperation {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject,
+    SelectProject,
+    ListProjects,
+    RegisterAgentPublicKey,
+    ListAgentPublicKeys,
+    RotateAgentPublicKey,
+    RevokeAgentPublicKey,
+    CreateNodeEnrollmentGrant,
+    ListNodeDescriptors,
+    RevokeNodeCredential,
+    StartProcess,
+    ScheduleTask,
+    LaunchTask,
+    CancelProcess,
+    AbortProcess,
+    ListProcesses,
+    QuotaStatus,
+    RestartTask,
+    ResolveTaskFailure,
+    DebugAttach,
+    SetDebugBreakpoints,
+    InspectDebugBreakpoints,
+    CreateDebugEpoch,
+    ResumeDebugEpoch,
+    InspectDebugEpoch,
+    ListTaskEvents,
+    ListTaskSnapshots,
+    JoinTask,
+    CreateArtifactDownloadLink,
+    OpenArtifactDownloadStream,
+    RevokeArtifactDownloadLink,
+    ExportArtifactToNode,
+}
+
+impl PublicUserOperation {
+    pub(super) fn as_str(self) -> &'static str {
+        match self {
+            Self::AuthStatus => "auth_status",
+            Self::RevokeCliSession => "revoke_cli_session",
+            Self::CreateProject => "create_project",
+            Self::SelectProject => "select_project",
+            Self::ListProjects => "list_projects",
+            Self::RegisterAgentPublicKey => "register_agent_public_key",
+            Self::ListAgentPublicKeys => "list_agent_public_keys",
+            Self::RotateAgentPublicKey => "rotate_agent_public_key",
+            Self::RevokeAgentPublicKey => "revoke_agent_public_key",
+            Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
+            Self::ListNodeDescriptors => "list_node_descriptors",
+            Self::RevokeNodeCredential => "revoke_node_credential",
+            Self::StartProcess => "start_process",
+            Self::ScheduleTask => "schedule_task",
+            Self::LaunchTask => "launch_task",
+            Self::CancelProcess => "cancel_process",
+            Self::AbortProcess => "abort_process",
+            Self::ListProcesses => "list_processes",
+            Self::QuotaStatus => "quota_status",
+            Self::RestartTask => "restart_task",
+            Self::ResolveTaskFailure => "resolve_task_failure",
+            Self::DebugAttach => "debug_attach",
+            Self::SetDebugBreakpoints => "set_debug_breakpoints",
+            Self::InspectDebugBreakpoints => "inspect_debug_breakpoints",
+            Self::CreateDebugEpoch => "create_debug_epoch",
+            Self::ResumeDebugEpoch => "resume_debug_epoch",
+            Self::InspectDebugEpoch => "inspect_debug_epoch",
+            Self::ListTaskEvents => "list_task_events",
+            Self::ListTaskSnapshots => "list_task_snapshots",
+            Self::JoinTask => "join_task",
+            Self::CreateArtifactDownloadLink => "create_artifact_download_link",
+            Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
+            Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
+            Self::ExportArtifactToNode => "export_artifact_to_node",
+        }
+    }
+}
+
+impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
+    fn from(request: &AuthenticatedCoordinatorRequest) -> Self {
+        match request {
+            AuthenticatedCoordinatorRequest::AuthStatus => Self::AuthStatus,
+            AuthenticatedCoordinatorRequest::RevokeCliSession => Self::RevokeCliSession,
+            AuthenticatedCoordinatorRequest::CreateProject { .. } => Self::CreateProject,
+            AuthenticatedCoordinatorRequest::SelectProject { .. } => Self::SelectProject,
+            AuthenticatedCoordinatorRequest::ListProjects => Self::ListProjects,
+            AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. } => {
+                Self::RegisterAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Self::ListAgentPublicKeys,
+            AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. } => {
+                Self::RotateAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. } => {
+                Self::RevokeAgentPublicKey
+            }
+            AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } => {
+                Self::CreateNodeEnrollmentGrant
+            }
+            AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
+            AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
+                Self::RevokeNodeCredential
+            }
+            AuthenticatedCoordinatorRequest::StartProcess { .. } => Self::StartProcess,
+            AuthenticatedCoordinatorRequest::ScheduleTask { .. } => Self::ScheduleTask,
+            AuthenticatedCoordinatorRequest::LaunchTask { .. } => Self::LaunchTask,
+            AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
+            AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
+            AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
+            AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
+            AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
+            AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
+            AuthenticatedCoordinatorRequest::DebugAttach { .. } => Self::DebugAttach,
+            AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } => {
+                Self::SetDebugBreakpoints
+            }
+            AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. } => {
+                Self::InspectDebugBreakpoints
+            }
+            AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. } => Self::CreateDebugEpoch,
+            AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. } => Self::ResumeDebugEpoch,
+            AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
+            AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
+            AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
+            AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
+            AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
+                Self::CreateArtifactDownloadLink
+            }
+            AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { .. } => {
+                Self::OpenArtifactDownloadStream
+            }
+            AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { .. } => {
+                Self::RevokeArtifactDownloadLink
+            }
+            AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
+                Self::ExportArtifactToNode
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct AuthorizedPublicUser {
+    pub(super) actor: UserId,
+    pub(super) operation: PublicUserOperation,
+}
+
+pub(super) fn authorize_authenticated_user_operation(
+    context: &AuthContext,
+    request: &AuthenticatedCoordinatorRequest,
+) -> Result<AuthorizedPublicUser, CoordinatorServiceError> {
+    let operation = PublicUserOperation::from(request);
+    let actor = match &context.actor {
+        Actor::User(user) => user.clone(),
+        _ => {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "authenticated {} request requires a user CLI session",
+                operation.as_str()
+            ))
+            .into());
+        }
+    };
+    Ok(AuthorizedPublicUser { actor, operation })
+}
+
+#[cfg(test)]
+mod tests {
+    use clusterflux_core::{AgentId, ProjectId, TenantId};
+
+    use super::*;
+
+    #[test]
+    fn authenticated_public_authorization_requires_user_context_and_names_operation() {
+        let request = AuthenticatedCoordinatorRequest::DebugAttach {
+            process: "vp".to_owned(),
+        };
+        let agent_context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::Agent(AgentId::from("agent-ci")),
+        };
+
+        let denied = authorize_authenticated_user_operation(&agent_context, &request).unwrap_err();
+        assert!(denied
+            .to_string()
+            .contains("authenticated debug_attach request requires a user CLI session"));
+
+        let user_context = AuthContext {
+            tenant: TenantId::from("tenant"),
+            project: ProjectId::from("project"),
+            actor: Actor::User(UserId::from("user")),
+        };
+        let authorized = authorize_authenticated_user_operation(&user_context, &request).unwrap();
+        assert_eq!(authorized.actor, UserId::from("user"));
+        assert_eq!(authorized.operation, PublicUserOperation::DebugAttach);
+    }
+}

crates/clusterflux-coordinator/src/service/debug/validation.rs

@@ -0,0 +1,92 @@
+use std::collections::BTreeSet;
+
+use crate::CoordinatorServiceError;
+
+use super::{DebugAcknowledgementState, DebugEpochRuntime};
+
+pub(super) fn runtime_all_in_state(
+    runtime: &DebugEpochRuntime,
+    state: DebugAcknowledgementState,
+) -> bool {
+    !runtime.expected.is_empty()
+        && runtime.expected.iter().all(|key| {
+            runtime
+                .acknowledgements
+                .get(key)
+                .is_some_and(|ack| ack.state == state)
+        })
+}
+
+pub(super) fn validate_probe_symbols(
+    probe_symbols: Vec<String>,
+) -> Result<Vec<String>, CoordinatorServiceError> {
+    if probe_symbols.len() > 128 {
+        return Err(CoordinatorServiceError::Protocol(
+            "debug breakpoint request exceeds 128 probe symbols".to_owned(),
+        ));
+    }
+    let mut unique = BTreeSet::new();
+    for symbol in probe_symbols {
+        if symbol.trim().is_empty()
+            || symbol.len() > 256
+            || !symbol
+                .chars()
+                .all(|character| character.is_ascii_alphanumeric() || "._:-/".contains(character))
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "debug breakpoint probe symbol is invalid".to_owned(),
+            ));
+        }
+        unique.insert(symbol);
+    }
+    Ok(unique.into_iter().collect())
+}
+
+pub(super) fn validate_debug_snapshot(
+    stack_frames: &[String],
+    local_values: &[(String, String)],
+    task_args: &[(String, String)],
+    handles: &[(String, String)],
+    command_status: Option<&str>,
+    recent_output: &[String],
+    message: Option<&str>,
+) -> Result<(), CoordinatorServiceError> {
+    const MAX_ITEMS: usize = 128;
+    const MAX_TEXT_BYTES: usize = 16 * 1024;
+    if [
+        stack_frames.len(),
+        local_values.len(),
+        task_args.len(),
+        handles.len(),
+        recent_output.len(),
+    ]
+    .into_iter()
+    .any(|count| count > MAX_ITEMS)
+    {
+        return Err(CoordinatorServiceError::Protocol(
+            "debug participant snapshot exceeds the 128-item field limit".to_owned(),
+        ));
+    }
+    let text_bytes = stack_frames.iter().map(String::len).sum::<usize>()
+        + local_values
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + task_args
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + handles
+            .iter()
+            .map(|(name, value)| name.len() + value.len())
+            .sum::<usize>()
+        + recent_output.iter().map(String::len).sum::<usize>()
+        + command_status.map(str::len).unwrap_or(0)
+        + message.map(str::len).unwrap_or(0);
+    if text_bytes > MAX_TEXT_BYTES {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "debug participant snapshot is {text_bytes} bytes; maximum is {MAX_TEXT_BYTES}"
+        )));
+    }
+    Ok(())
+}

crates/clusterflux-coordinator/src/service/debug_requests.rs

@@ -0,0 +1,575 @@
+use std::collections::BTreeSet;
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    Actor, Capability, CredentialKind, Digest, EnvironmentResource, ProcessId, ProjectId,
+    RestartDecision, RestartPolicy, RestartRequest, TaskDispatch, TaskInstanceId, TenantId, UserId,
+    WasmExportAbi,
+};
+
+use crate::{CoordinatorError, CoordinatorServiceError};
+
+use super::keys::task_restart_key;
+use super::protocol::{TaskAttemptState, TaskFailureResolution};
+use super::{
+    AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
+    TaskReplacementBundle, TaskTerminalState, WorkflowActor,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_debug_request(
+        &mut self,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        match request {
+            CoordinatorRequest::DebugAttach {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_debug_attach(tenant, project, actor_user, process),
+            CoordinatorRequest::SetDebugBreakpoints {
+                tenant,
+                project,
+                actor_user,
+                process,
+                probe_symbols,
+            } => self.handle_set_debug_breakpoints(
+                tenant,
+                project,
+                actor_user,
+                process,
+                probe_symbols,
+            ),
+            CoordinatorRequest::InspectDebugBreakpoints {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_inspect_debug_breakpoints(tenant, project, actor_user, process),
+            CoordinatorRequest::CreateDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                stopped_task,
+                reason,
+            } => self.handle_create_debug_epoch(
+                tenant,
+                project,
+                actor_user,
+                process,
+                stopped_task,
+                reason,
+            ),
+            CoordinatorRequest::ResumeDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                epoch,
+            } => self.handle_resume_debug_epoch(tenant, project, actor_user, process, epoch),
+            CoordinatorRequest::InspectDebugEpoch {
+                tenant,
+                project,
+                actor_user,
+                process,
+                epoch,
+            } => self.handle_inspect_debug_epoch(tenant, project, actor_user, process, epoch),
+            _ => unreachable!("handle_debug_request only accepts debug coordinator requests"),
+        }
+    }
+
+    pub(super) fn handle_authenticated_debug_request(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        actor: &UserId,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        match request {
+            AuthenticatedCoordinatorRequest::DebugAttach { process } => self.handle_debug_attach(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+            ),
+            AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
+                process,
+                probe_symbols,
+            } => self.handle_set_debug_breakpoints(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                probe_symbols,
+            ),
+            AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
+                .handle_inspect_debug_breakpoints(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::CreateDebugEpoch {
+                process,
+                stopped_task,
+                reason,
+            } => self.handle_create_debug_epoch(
+                tenant.as_str().to_owned(),
+                project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                stopped_task,
+                reason,
+            ),
+            AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, epoch } => self
+                .handle_resume_debug_epoch(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                    epoch,
+                ),
+            AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, epoch } => self
+                .handle_inspect_debug_epoch(
+                    tenant.as_str().to_owned(),
+                    project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                    epoch,
+                ),
+            _ => unreachable!(
+                "handle_authenticated_debug_request only accepts debug coordinator requests"
+            ),
+        }
+    }
+
+    pub(super) fn handle_restart_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        replacement_bundle: Option<TaskReplacementBundle>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor.clone()),
+        };
+        let authorization = self.coordinator.authorize_debug_attach(&context, &process);
+        if !authorization.allowed {
+            let _ = self.record_debug_audit_event(
+                tenant,
+                project,
+                process,
+                Some(task),
+                actor,
+                "restart_task",
+                false,
+                authorization.reason.clone(),
+            )?;
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task restart denied: {}",
+                authorization.reason
+            ))
+            .into());
+        }
+        let active_key = self
+            .active_tasks
+            .iter()
+            .find(|(task_tenant, task_project, task_process, _, task_id)| {
+                task_tenant == &tenant
+                    && task_project == &project
+                    && task_process == &process
+                    && task_id == &task
+            })
+            .cloned();
+        let process_key = super::keys::process_control_key(&tenant, &project, &process);
+        let active_main = self
+            .main_runtime
+            .controls
+            .get(&process_key)
+            .is_some_and(|control| {
+                control.task_instance == task
+                    && matches!(control.state.as_str(), "running" | "stopping")
+            });
+        let active_task = active_key.is_some() || active_main;
+        let completed_event_observed = self.task_events.iter().any(|event| {
+            event.tenant == tenant
+                && event.project == project
+                && event.process == process
+                && event.task == task
+        });
+        let checkpoint_key = task_restart_key(&tenant, &project, &process, &task);
+        let checkpoint = self.task_restart_checkpoints.get(&checkpoint_key).cloned();
+        let mut accepted = false;
+        let mut restarted_task_instance = None;
+        let mut restarted_attempt_id = None;
+        let mut clean_boundary_available = false;
+        let mut requires_whole_process_restart = true;
+        let message = if active_main {
+            "selected coordinator main is still active; restart the whole virtual process to rerun its capless entry boundary".to_owned()
+        } else if active_task {
+            "selected task is still active; wait for its terminal event or abort the whole virtual process before restarting from its clean entry boundary".to_owned()
+        } else if !completed_event_observed {
+            "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
+        } else if let Some(checkpoint) = checkpoint {
+            let vfs_available =
+                checkpoint
+                    .checkpoint
+                    .vfs_manifest
+                    .objects
+                    .iter()
+                    .all(|(path, object)| {
+                        let artifact = super::keys::artifact_id_from_path(path);
+                        self.artifact_registry
+                            .metadata(&artifact)
+                            .is_some_and(|metadata| {
+                                metadata.tenant == tenant
+                                    && metadata.project == project
+                                    && metadata.digest == object.digest
+                                    && metadata.size == object.size
+                                    && !metadata.retaining_nodes.is_empty()
+                            })
+                    });
+            if !vfs_available {
+                "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
+            } else {
+                let replacement = replacement_bundle
+                    .as_ref()
+                    .map(|bundle| {
+                        validate_task_replacement(
+                            bundle,
+                            &checkpoint.assignment.task_spec.task_definition,
+                            checkpoint.assignment.task_spec.environment_id.as_deref(),
+                        )
+                    })
+                    .transpose()?;
+                let request = RestartRequest {
+                    task: task.clone(),
+                    entrypoint: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.task_entrypoint.clone(),
+                        |replacement| replacement.export.clone(),
+                    ),
+                    serialized_args: checkpoint.checkpoint.boundary.serialized_args.clone(),
+                    environment_digest: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.environment_digest.clone(),
+                        |replacement| replacement.environment_digest.clone(),
+                    ),
+                    task_abi: replacement.as_ref().map_or_else(
+                        || checkpoint.checkpoint.boundary.task_abi.clone(),
+                        |replacement| replacement.restart_compatibility.clone(),
+                    ),
+                    source_edited: replacement.is_some(),
+                };
+                match RestartPolicy.decide(&checkpoint.checkpoint, &request) {
+                    RestartDecision::RestartTask { from_vfs_epoch, .. } => {
+                        clean_boundary_available = true;
+                        let mut assignment = checkpoint.assignment;
+                        assignment.task = task.clone();
+                        assignment.task_spec.task_instance = task.clone();
+                        let next_attempt = self
+                            .task_attempts
+                            .get(&checkpoint_key)
+                            .map_or(1, |attempts| attempts.len() + 1);
+                        assignment.artifact_path = format!(
+                            "/vfs/artifacts/{}-attempt-{next_attempt}-result.json",
+                            task.as_str()
+                        );
+                        if let Some(replacement) = replacement {
+                            assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
+                                export: Some(replacement.export),
+                                abi: WasmExportAbi::TaskV1,
+                            };
+                            assignment.task_spec.environment = replacement
+                                .environment
+                                .map(|environment| environment.requirements);
+                            assignment.task_spec.environment_digest =
+                                Some(replacement.environment_digest);
+                            assignment.task_spec.required_capabilities =
+                                replacement.required_capabilities;
+                            assignment.task_spec.bundle_digest =
+                                Some(replacement.bundle_digest.clone());
+                            assignment.wasm_module_base64 = replacement.wasm_module_base64;
+                        }
+                        let restart_task_spec = assignment.task_spec.clone();
+                        let restart_artifact_path = assignment.artifact_path.clone();
+                        self.restart_launches.insert(checkpoint_key.clone());
+                        let launch = self.handle_launch_task_with_actor(
+                            tenant.clone(),
+                            project.clone(),
+                            WorkflowActor {
+                                kind: "user".to_owned(),
+                                user: Some(actor.clone()),
+                                agent: None,
+                                credential_kind: CredentialKind::CliDeviceSession,
+                                public_key_fingerprint: None,
+                                authenticated_without_browser: false,
+                                scopes: vec!["process:restart-task".to_owned()],
+                            },
+                            assignment.task_spec,
+                            true,
+                            assignment.artifact_path,
+                            assignment.wasm_module_base64,
+                        );
+                        self.restart_launches.remove(&checkpoint_key);
+                        let launch = launch?;
+                        let queued = matches!(launch, CoordinatorResponse::TaskQueued { .. });
+                        accepted = matches!(
+                            &launch,
+                            CoordinatorResponse::TaskLaunched { .. }
+                                | CoordinatorResponse::TaskQueued { .. }
+                        );
+                        if accepted {
+                            restarted_task_instance = Some(task.clone());
+                            restarted_attempt_id = self
+                                .task_attempts
+                                .get(&checkpoint_key)
+                                .and_then(|attempts| attempts.last())
+                                .map(|attempt| attempt.attempt_id.clone());
+                            if restarted_attempt_id.is_none() {
+                                restarted_attempt_id = Some(self.begin_task_attempt(
+                                    &restart_task_spec,
+                                    None,
+                                    Some(&restart_artifact_path),
+                                    queued,
+                                )?);
+                            }
+                        }
+                        requires_whole_process_restart = !accepted;
+                        format!(
+                            "selected logical task {task} restarted as a new attempt from clean VFS entry boundary epoch {from_vfs_epoch}; unflushed task changes were discarded"
+                        )
+                    }
+                    RestartDecision::RestartWholeVirtualProcess { message } => message,
+                }
+            }
+        } else {
+            "selected task has terminal metadata but no captured clean VFS entry boundary; restart the whole virtual process".to_owned()
+        };
+        let audit_event = self.record_debug_audit_event(
+            tenant,
+            project,
+            process.clone(),
+            Some(task.clone()),
+            actor.clone(),
+            "restart_task",
+            true,
+            &message,
+        )?;
+        Ok(CoordinatorResponse::TaskRestart {
+            process,
+            task,
+            restarted_task_instance,
+            restarted_attempt_id,
+            actor,
+            accepted,
+            clean_boundary_available,
+            active_task,
+            completed_event_observed,
+            requires_whole_process_restart,
+            message,
+            charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
+            used_debug_read_bytes: audit_event.used_debug_read_bytes,
+            audit_event,
+        })
+    }
+
+    pub(super) fn handle_resolve_task_failure(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor),
+        };
+        let authorization = self.coordinator.authorize_debug_attach(&context, &process);
+        if !authorization.allowed {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task failure resolution denied: {}",
+                authorization.reason
+            ))
+            .into());
+        }
+        let key = task_restart_key(&tenant, &project, &process, &task);
+        let attempt = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+            .filter(|attempt| attempt.state == TaskAttemptState::FailedAwaitingAction)
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(
+                    "task is not failed awaiting operator action".to_owned(),
+                )
+            })?;
+        attempt.state = match resolution {
+            TaskFailureResolution::AcceptFailure => TaskAttemptState::Failed,
+            TaskFailureResolution::Cancel => TaskAttemptState::Cancelled,
+        };
+        attempt.command_state = Some(
+            match resolution {
+                TaskFailureResolution::AcceptFailure => "failure_accepted",
+                TaskFailureResolution::Cancel => "cancelled",
+            }
+            .to_owned(),
+        );
+        let attempt_id = attempt.attempt_id.clone();
+        let mut event = self
+            .task_events
+            .iter()
+            .rev()
+            .find(|event| {
+                event.tenant == tenant
+                    && event.project == project
+                    && event.process == process
+                    && event.task == task
+                    && event.attempt_id.as_deref() == Some(attempt_id.as_str())
+            })
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(
+                    "failed attempt terminal event is unavailable".to_owned(),
+                )
+            })?;
+        if resolution == TaskFailureResolution::Cancel {
+            event.terminal_state = TaskTerminalState::Cancelled;
+            event.stderr_tail = "operator cancelled task after failure".to_owned();
+        }
+        self.record_task_completion_event(event.clone());
+        self.notify_coordinator_main_waiters(&event);
+        Ok(CoordinatorResponse::TaskFailureResolved {
+            process,
+            task,
+            attempt_id,
+            resolution,
+        })
+    }
+}
+
+struct ValidatedTaskReplacement {
+    bundle_digest: Digest,
+    wasm_module_base64: String,
+    export: String,
+    restart_compatibility: Digest,
+    environment: Option<EnvironmentResource>,
+    environment_digest: Digest,
+    required_capabilities: BTreeSet<Capability>,
+}
+
+fn validate_task_replacement(
+    replacement: &TaskReplacementBundle,
+    task_definition: &clusterflux_core::TaskDefinitionId,
+    environment_id: Option<&str>,
+) -> Result<ValidatedTaskReplacement, CoordinatorServiceError> {
+    let module = BASE64_STANDARD
+        .decode(&replacement.wasm_module_base64)
+        .map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "replacement task bundle is not valid base64: {error}"
+            ))
+        })?;
+    let actual_digest = Digest::sha256(&module);
+    if actual_digest != replacement.bundle_digest {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "replacement task bundle digest mismatch: expected {}, actual {actual_digest}",
+            replacement.bundle_digest
+        )));
+    }
+    let descriptors = super::main_runtime::task_descriptors(&module)?;
+    let descriptor = descriptors.get(task_definition.as_str()).ok_or_else(|| {
+        CoordinatorServiceError::Protocol(format!(
+            "replacement bundle has no task definition `{task_definition}`"
+        ))
+    })?;
+    if descriptor
+        .get("abi_version")
+        .and_then(serde_json::Value::as_u64)
+        != Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
+    {
+        return Err(CoordinatorServiceError::Protocol(format!(
+            "replacement task `{task_definition}` uses an unsupported task ABI"
+        )));
+    }
+    let export = descriptor
+        .get("export")
+        .and_then(serde_json::Value::as_str)
+        .filter(|export| !export.is_empty())
+        .ok_or_else(|| {
+            CoordinatorServiceError::Protocol(format!(
+                "replacement task `{task_definition}` omitted its export"
+            ))
+        })?
+        .to_owned();
+    let restart_compatibility = serde_json::from_value(
+        descriptor
+            .get("restart_compatibility_hash")
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorServiceError::Protocol(format!(
+                    "replacement task `{task_definition}` omitted restart compatibility metadata"
+                ))
+            })?,
+    )?;
+    let mut required_capabilities = descriptor
+        .get("required_capabilities")
+        .and_then(serde_json::Value::as_array)
+        .into_iter()
+        .flatten()
+        .map(|capability| {
+            super::main_runtime::capability_from_descriptor(capability.as_str().ok_or_else(
+                || {
+                    CoordinatorServiceError::Protocol(
+                        "replacement task capability is not a string".to_owned(),
+                    )
+                },
+            )?)
+            .map_err(CoordinatorServiceError::Protocol)
+        })
+        .collect::<Result<BTreeSet<_>, _>>()?;
+    let environment = environment_id
+        .map(|environment_id| {
+            super::main_runtime::bundle_environments(&module)?
+                .remove(environment_id)
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "replacement bundle has no environment `{environment_id}`"
+                    ))
+                })
+        })
+        .transpose()?;
+    if let Some(environment) = &environment {
+        required_capabilities.extend(environment.requirements.capabilities.iter().cloned());
+    }
+    let environment_digest = environment.as_ref().map_or_else(
+        || Digest::sha256("clusterflux.environment.unconstrained.v1"),
+        |environment| environment.digest.clone(),
+    );
+    Ok(ValidatedTaskReplacement {
+        bundle_digest: replacement.bundle_digest.clone(),
+        wasm_module_base64: replacement.wasm_module_base64.clone(),
+        export,
+        restart_compatibility,
+        environment,
+        environment_digest,
+        required_capabilities,
+    })
+}

crates/clusterflux-coordinator/src/service/durable_runtime.rs

@@ -0,0 +1,47 @@
+use crate::{
+    DurableState, DurableStore, FallibleDurableStore, InMemoryDurableStore, PostgresDurableStore,
+};
+
+pub(super) enum RuntimeDurableStore {
+    InMemory(InMemoryDurableStore),
+    Postgres(PostgresDurableStore),
+}
+
+impl RuntimeDurableStore {
+    pub(super) fn from_database_url(database_url: Option<&str>) -> Result<Self, String> {
+        match database_url.map(str::trim).filter(|url| !url.is_empty()) {
+            Some(url) => PostgresDurableStore::connect(url)
+                .map(Self::Postgres)
+                .map_err(|error| error.to_string()),
+            None => Ok(Self::InMemory(InMemoryDurableStore::default())),
+        }
+    }
+
+    pub(super) fn kind(&self) -> &'static str {
+        match self {
+            Self::InMemory(_) => "in_memory",
+            Self::Postgres(_) => "postgres",
+        }
+    }
+}
+
+impl FallibleDurableStore for RuntimeDurableStore {
+    type Error = String;
+
+    fn load_state(&mut self) -> Result<DurableState, Self::Error> {
+        match self {
+            Self::InMemory(store) => Ok(store.load()),
+            Self::Postgres(store) => store.load_state().map_err(|error| error.to_string()),
+        }
+    }
+
+    fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
+        match self {
+            Self::InMemory(store) => {
+                store.save(state.clone());
+                Ok(())
+            }
+            Self::Postgres(store) => store.save_state(state).map_err(|error| error.to_string()),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/keys.rs

@@ -0,0 +1,73 @@
+use clusterflux_core::{
+    ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
+};
+
+pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
+pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
+pub(super) type TaskAssignmentKey = (TenantId, ProjectId, NodeId);
+pub(super) type PanelStopKey = (TenantId, ProjectId, ProcessId);
+pub(super) type EnrollmentGrantKey = (TenantId, ProjectId, String);
+pub(super) type ProcessControlKey = (TenantId, ProjectId, ProcessId);
+
+pub(super) fn task_control_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+    node: &NodeId,
+    task: &TaskInstanceId,
+) -> TaskControlKey {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        node.clone(),
+        task.clone(),
+    )
+}
+
+pub(super) fn task_restart_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+    task: &TaskInstanceId,
+) -> TaskRestartKey {
+    (
+        tenant.clone(),
+        project.clone(),
+        process.clone(),
+        task.clone(),
+    )
+}
+
+pub(super) fn process_control_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+) -> ProcessControlKey {
+    (tenant.clone(), project.clone(), process.clone())
+}
+
+pub(super) fn panel_stop_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    process: &ProcessId,
+) -> PanelStopKey {
+    (tenant.clone(), project.clone(), process.clone())
+}
+
+pub(super) fn enrollment_grant_key(
+    tenant: &TenantId,
+    project: &ProjectId,
+    grant: &str,
+) -> EnrollmentGrantKey {
+    (tenant.clone(), project.clone(), grant.to_owned())
+}
+
+pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
+    let value = path
+        .as_str()
+        .strip_prefix("/vfs/artifacts/")
+        .unwrap_or(path.as_str())
+        .replace('/', ":");
+    ArtifactId::new(value)
+}

crates/clusterflux-coordinator/src/service/logs.rs

@@ -0,0 +1,661 @@
+use clusterflux_core::{
+    ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
+    TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key, task_restart_key};
+use super::protocol::TaskAttemptState;
+use super::{
+    artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
+    TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
+};
+
+impl CoordinatorService {
+    pub(super) fn handle_report_task_log(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        backpressured: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        validate_task_log_tail("stdout_tail", &stdout_tail)?;
+        validate_task_log_tail("stderr_tail", &stderr_tail)?;
+        let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+        self.quota
+            .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
+        Ok(CoordinatorResponse::TaskLogRecorded {
+            process,
+            task,
+            stdout_bytes,
+            stderr_bytes,
+            stdout_tail: if stdout_truncated {
+                format!("{stdout_tail}\n... truncated")
+            } else {
+                stdout_tail
+            },
+            stderr_tail: if stderr_truncated {
+                format!("{stderr_tail}\n... truncated")
+            } else {
+                stderr_tail
+            },
+            backpressured,
+        })
+    }
+
+    pub(super) fn handle_report_vfs_metadata(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        large_bytes_uploaded: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let artifact_path = artifact_path
+            .map(VfsPath::new)
+            .transpose()
+            .map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
+            self.flush_artifact_metadata(ArtifactFlush {
+                id: artifact_id_from_path(path),
+                tenant,
+                project,
+                process: process.clone(),
+                producer_task: task.clone(),
+                retaining_node: node,
+                digest,
+                size: artifact_size_bytes.unwrap_or_default(),
+            })?;
+        }
+        Ok(CoordinatorResponse::VfsMetadataRecorded {
+            process,
+            task,
+            artifact_path,
+            large_bytes_uploaded,
+        })
+    }
+
+    pub(super) fn handle_task_completed(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        terminal_state: Option<TaskTerminalState>,
+        status_code: Option<i32>,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        result: Option<TaskBoundaryValue>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        validate_task_log_tail("stdout_tail", &stdout_tail)?;
+        validate_task_log_tail("stderr_tail", &stderr_tail)?;
+        let artifact_path = artifact_path
+            .map(VfsPath::new)
+            .transpose()
+            .map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let checkpoint = self
+            .task_restart_checkpoints
+            .get(&super::keys::task_restart_key(
+                &tenant, &project, &process, &task,
+            ))
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "signed node task completion does not name a coordinator-issued task instance"
+                        .to_owned(),
+                )
+            })?;
+        if checkpoint.assignment.node != node {
+            return Err(CoordinatorError::Unauthorized(
+                "signed node task completion came from a node other than the assigned node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let mut event = TaskCompletionEvent {
+            tenant,
+            project,
+            process,
+            node,
+            executor: super::TaskExecutor::Node,
+            task_definition: checkpoint.assignment.task_spec.task_definition.clone(),
+            task,
+            attempt_id: None,
+            placement: None,
+            terminal_state: terminal_state
+                .unwrap_or_else(|| TaskTerminalState::from_status_code(status_code)),
+            status_code,
+            stdout_bytes,
+            stderr_bytes,
+            stdout_tail,
+            stderr_tail,
+            stdout_truncated,
+            stderr_truncated,
+            artifact_path,
+            artifact_digest: artifact_digest.clone(),
+            artifact_size_bytes,
+            result,
+        };
+        let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota.can_charge_log_bytes(
+            &event.tenant,
+            &event.project,
+            reported_bytes,
+            now_epoch_seconds,
+        )?;
+        self.quota.charge_log_bytes(
+            &event.tenant,
+            &event.project,
+            reported_bytes,
+            now_epoch_seconds,
+        )?;
+        let task_key = task_control_key(
+            &event.tenant,
+            &event.project,
+            &event.process,
+            &event.node,
+            &event.task,
+        );
+        let process_key = process_control_key(&event.tenant, &event.project, &event.process);
+        let process_was_aborted = self.process_aborts.contains(&process_key);
+        event.placement = self.task_placements.remove(&task_key);
+        if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
+            self.flush_artifact_metadata(ArtifactFlush {
+                id: artifact_id_from_path(path),
+                tenant: event.tenant.clone(),
+                project: event.project.clone(),
+                process: event.process.clone(),
+                producer_task: event.task.clone(),
+                retaining_node: event.node.clone(),
+                digest,
+                size: artifact_size_bytes.unwrap_or(stdout_bytes),
+            })?;
+        }
+        self.task_cancellations.remove(&task_key);
+        self.task_aborts.remove(&task_key);
+        self.debug_commands.remove(&task_key);
+        self.active_tasks.remove(&task_key);
+        let no_active_tasks =
+            !self
+                .active_tasks
+                .iter()
+                .any(|(task_tenant, task_project, task_process, _, _)| {
+                    task_tenant == &event.tenant
+                        && task_project == &event.project
+                        && task_process == &event.process
+                });
+        if no_active_tasks {
+            self.process_aborts.remove(&process_key);
+            if self.process_cancellations.remove(&process_key)
+                && !self.main_runtime.controls.contains_key(&process_key)
+            {
+                self.coordinator
+                    .abort_process(&event.tenant, &event.project, &event.process)?;
+                self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
+                self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
+            }
+        }
+        if process_was_aborted {
+            let checkpoint_key = super::keys::task_restart_key(
+                &event.tenant,
+                &event.project,
+                &event.process,
+                &event.task,
+            );
+            self.task_restart_checkpoints.remove(&checkpoint_key);
+            self.task_restart_checkpoint_order
+                .retain(|retained| retained != &checkpoint_key);
+        }
+        let awaiting_operator = self.finish_task_attempt(&mut event);
+        self.record_task_completion_event(event.clone());
+        if !awaiting_operator {
+            self.notify_coordinator_main_waiters(&event);
+        }
+        Ok(CoordinatorResponse::TaskRecorded {
+            process: event.process,
+            task: event.task,
+            events_recorded: self.task_events.len(),
+        })
+    }
+
+    pub(super) fn handle_list_task_events(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: Option<String>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = process.map(ProcessId::new);
+        if let Some(process) = &process {
+            self.authorize_task_event_process_scope(&tenant, &project, process)?;
+        }
+        let events = self
+            .task_events
+            .iter()
+            .filter(|event| {
+                event.tenant == tenant
+                    && event.project == project
+                    && process
+                        .as_ref()
+                        .is_none_or(|process| event.process == *process)
+            })
+            .cloned()
+            .collect();
+        Ok(CoordinatorResponse::TaskEvents { events })
+    }
+
+    pub(super) fn handle_list_task_snapshots(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        self.authorize_task_event_process_scope(&tenant, &project, &process)?;
+        let snapshots = self
+            .task_attempts
+            .iter()
+            .filter(
+                |((attempt_tenant, attempt_project, attempt_process, _), _)| {
+                    attempt_tenant == &tenant
+                        && attempt_project == &project
+                        && attempt_process == &process
+                },
+            )
+            .flat_map(|(_, attempts)| attempts.iter().cloned())
+            .collect();
+        Ok(CoordinatorResponse::TaskSnapshots { snapshots })
+    }
+
+    fn authorize_task_event_process_scope(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let active_in_scope = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .is_some();
+        let historical_in_scope = self.task_events.iter().any(|event| {
+            event.tenant == *tenant && event.project == *project && event.process == *process
+        }) || self.process_scope_history.iter().any(
+            |(historical_tenant, historical_project, historical_process)| {
+                historical_tenant == tenant
+                    && historical_project == project
+                    && historical_process == process
+            },
+        );
+        let process_exists_outside_scope = self
+            .coordinator
+            .active_process_exists_outside_scope(tenant, project, process)
+            || self.task_events.iter().any(|event| {
+                event.process == *process && (event.tenant != *tenant || event.project != *project)
+            })
+            || self.process_scope_history.iter().any(
+                |(historical_tenant, historical_project, historical_process)| {
+                    historical_process == process
+                        && (historical_tenant != tenant || historical_project != project)
+                },
+            );
+        if !active_in_scope && !historical_in_scope && process_exists_outside_scope {
+            return Err(CoordinatorError::Unauthorized(
+                "task event access is outside the virtual process tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    pub(super) fn handle_join_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let _actor = UserId::new(actor_user);
+        let process = ProcessId::new(process);
+        let task = TaskInstanceId::new(task);
+        Ok(CoordinatorResponse::TaskJoined {
+            join: self.task_join_result(tenant, project, process, task),
+        })
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_join_child_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let parent_task = TaskInstanceId::new(parent_task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        if !self.active_tasks.contains(&super::keys::task_control_key(
+            &tenant,
+            &project,
+            &process,
+            &node,
+            &parent_task,
+        )) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task join requires a currently active parent task on the signed node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        Ok(CoordinatorResponse::TaskJoined {
+            join: self.task_join_result(tenant, project, process, TaskInstanceId::new(task)),
+        })
+    }
+
+    pub(super) fn task_join_result(
+        &self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        task: TaskInstanceId,
+    ) -> TaskJoinResult {
+        let attempt_key = task_restart_key(&tenant, &project, &process, &task);
+        if self
+            .task_attempts
+            .get(&attempt_key)
+            .is_some_and(|attempts| {
+                attempts
+                    .iter()
+                    .rev()
+                    .find(|attempt| attempt.current)
+                    .is_some_and(|attempt| {
+                        matches!(
+                            attempt.state,
+                            TaskAttemptState::Queued
+                                | TaskAttemptState::Running
+                                | TaskAttemptState::FailedAwaitingAction
+                        )
+                    })
+            })
+        {
+            return TaskJoinResult::pending(
+                process,
+                task,
+                "logical task is still running or awaiting operator action",
+            );
+        }
+        let event = self.task_events.iter().rev().find(|event| {
+            event.tenant == tenant
+                && event.project == project
+                && event.process == process
+                && event.task == task
+        });
+
+        if let Some(event) = event {
+            TaskJoinResult::from_remote_completion(
+                event.process.clone(),
+                event.task.clone(),
+                event.node.clone(),
+                join_state_for_terminal(&event.terminal_state),
+                event.result.clone(),
+                event.status_code,
+                join_message_for_event(event),
+            )
+        } else {
+            let known = self.task_is_known_or_active(&tenant, &project, &process, &task);
+            TaskJoinResult::pending(
+                process,
+                task,
+                if known {
+                    "waiting for signed node task_completed event before join returns"
+                } else {
+                    "no signed node completion event has been observed for this task"
+                },
+            )
+        }
+    }
+
+    pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
+        event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
+        event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
+        let process_scope = (
+            event.tenant.clone(),
+            event.project.clone(),
+            event.process.clone(),
+        );
+        self.process_scope_history
+            .retain(|retained| retained != &process_scope);
+        while self.process_scope_history.len() >= super::MAX_TASK_EVENTS_TOTAL {
+            self.process_scope_history.pop_front();
+        }
+        self.process_scope_history.push_back(process_scope);
+        while self
+            .task_events
+            .iter()
+            .filter(|retained| {
+                retained.tenant == event.tenant
+                    && retained.project == event.project
+                    && retained.process == event.process
+            })
+            .count()
+            >= super::MAX_TASK_EVENTS_PER_PROCESS
+        {
+            let Some(index) = self.task_events.iter().position(|retained| {
+                retained.tenant == event.tenant
+                    && retained.project == event.project
+                    && retained.process == event.process
+            }) else {
+                break;
+            };
+            self.task_events.remove(index);
+        }
+        while self.task_events.len() >= super::MAX_TASK_EVENTS_TOTAL {
+            self.task_events.pop_front();
+        }
+        self.task_events.push_back(event);
+    }
+
+    fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
+        let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
+        let Some(attempt) = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+        else {
+            return false;
+        };
+        event.attempt_id = Some(attempt.attempt_id.clone());
+        attempt.status_code = event.status_code;
+        attempt.artifact_path = event.artifact_path.clone();
+        attempt.artifact_digest = event.artifact_digest.clone();
+        attempt.artifact_size_bytes = event.artifact_size_bytes;
+        attempt.error = (!event.stderr_tail.trim().is_empty()).then(|| event.stderr_tail.clone());
+        let awaiting_operator = event.terminal_state == TaskTerminalState::Failed
+            && attempt.failure_policy == clusterflux_core::TaskFailurePolicy::AwaitOperator;
+        attempt.state = if awaiting_operator {
+            TaskAttemptState::FailedAwaitingAction
+        } else {
+            match event.terminal_state {
+                TaskTerminalState::Completed => TaskAttemptState::Completed,
+                TaskTerminalState::Failed => TaskAttemptState::Failed,
+                TaskTerminalState::Cancelled => TaskAttemptState::Cancelled,
+            }
+        };
+        attempt.command_state = Some(if awaiting_operator {
+            "failed_awaiting_action".to_owned()
+        } else {
+            format!("{:?}", event.terminal_state).to_ascii_lowercase()
+        });
+        awaiting_operator
+    }
+
+    fn flush_artifact_metadata(
+        &mut self,
+        flush: ArtifactFlush,
+    ) -> Result<(), CoordinatorServiceError> {
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.artifact_registry
+            .expire_download_links(now_epoch_seconds);
+        let pinned = self
+            .task_restart_checkpoints
+            .values()
+            .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
+            .chain(
+                self.pending_task_launches
+                    .iter()
+                    .flat_map(|pending| pending.task_spec.required_artifacts.iter()),
+            )
+            .cloned()
+            .collect::<std::collections::BTreeSet<ArtifactId>>();
+        self.artifact_registry
+            .flush_metadata_bounded(flush, &pinned)
+            .map(|_| ())
+            .map_err(CoordinatorServiceError::Protocol)
+    }
+
+    fn task_is_known_or_active(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task: &TaskInstanceId,
+    ) -> bool {
+        self.active_tasks
+            .iter()
+            .any(|(task_tenant, task_project, task_process, _, task_id)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && task_id == task
+            })
+            || self.pending_task_launches.iter().any(|pending| {
+                &pending.tenant == tenant
+                    && &pending.project == project
+                    && &pending.process == process
+                    && &pending.task == task
+            })
+            || self.task_assignments.values().any(|assignments| {
+                assignments.iter().any(|assignment| {
+                    &assignment.tenant == tenant
+                        && &assignment.project == project
+                        && &assignment.process == process
+                        && &assignment.task == task
+                })
+            })
+    }
+}
+
+fn checked_reported_log_bytes(
+    stdout_bytes: u64,
+    stderr_bytes: u64,
+) -> Result<u64, CoordinatorServiceError> {
+    stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
+        CoordinatorServiceError::Protocol(
+            "reported task log byte counts exceed the supported range".to_owned(),
+        )
+    })
+}
+
+fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
+    if value.len() > MAX_TASK_LOG_TAIL_BYTES {
+        return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
+            "{kind} is {} bytes; max is {MAX_TASK_LOG_TAIL_BYTES}",
+            value.len()
+        )));
+    }
+    Ok(())
+}
+
+fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
+    if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
+        return value;
+    }
+    let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
+    while !value.is_char_boundary(boundary) {
+        boundary -= 1;
+    }
+    value.truncate(boundary);
+    *truncated = true;
+    value
+}
+
+fn join_state_for_terminal(terminal: &TaskTerminalState) -> TaskJoinState {
+    match terminal {
+        TaskTerminalState::Completed => TaskJoinState::Completed,
+        TaskTerminalState::Failed => TaskJoinState::Failed,
+        TaskTerminalState::Cancelled => TaskJoinState::Cancelled,
+    }
+}
+
+fn join_message_for_event(event: &TaskCompletionEvent) -> String {
+    match event.terminal_state {
+        TaskTerminalState::Completed => {
+            "joined result from signed node task_completed event".to_owned()
+        }
+        TaskTerminalState::Failed => {
+            let stderr = event.stderr_tail.trim();
+            if stderr.is_empty() {
+                "remote task failed".to_owned()
+            } else {
+                format!("remote task failed: {stderr}")
+            }
+        }
+        TaskTerminalState::Cancelled => "remote task was cancelled".to_owned(),
+    }
+}

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -0,0 +1,452 @@
+use std::collections::BTreeSet;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    generate_opaque_token, verify_node_request_signature, Actor, ArtifactId, CredentialKind,
+    Digest, NodeCapabilities, NodeDescriptor, NodeId, NodeSignedRequest, ProjectId,
+    SourceProviderKind, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::{
+    bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
+    CoordinatorServiceError,
+};
+
+impl CoordinatorService {
+    pub fn set_node_stale_after_seconds(&mut self, seconds: u64) {
+        self.node_stale_after_seconds = seconds.max(1);
+    }
+
+    pub(super) fn liveness_now_epoch_seconds(&self) -> u64 {
+        #[cfg(test)]
+        if let Some(now) = self.server_time_override {
+            return now;
+        }
+        unix_timestamp_seconds()
+    }
+
+    pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
+        self.node_last_seen_epoch_seconds
+            .get(node)
+            .is_some_and(|last_seen| {
+                self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
+                    <= self.node_stale_after_seconds
+            })
+    }
+
+    pub(super) fn live_node_descriptors(&self) -> Vec<NodeDescriptor> {
+        self.node_descriptors
+            .values()
+            .cloned()
+            .map(|mut descriptor| {
+                descriptor.online = self.node_is_live(&descriptor.id);
+                descriptor
+            })
+            .collect()
+    }
+
+    pub(super) fn handle_attach_node(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        self.coordinator.upsert_tenant(tenant.clone());
+        self.coordinator.upsert_user(
+            tenant.clone(),
+            UserId::from("local-user"),
+            CredentialKind::CliDeviceSession,
+        );
+        self.coordinator
+            .upsert_project(tenant.clone(), project.clone(), "local");
+        self.coordinator.upsert_source_provider_config(
+            tenant.clone(),
+            project.clone(),
+            SourceProviderKind::Filesystem,
+            Digest::sha256("local-filesystem"),
+        );
+        self.coordinator.enroll_node(
+            tenant.clone(),
+            project.clone(),
+            node.clone(),
+            public_key,
+            "node:attach",
+        );
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeAttached {
+            node,
+            tenant,
+            project,
+        })
+    }
+
+    pub(super) fn handle_create_node_enrollment_grant(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        ttl_seconds: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        self.coordinator.upsert_tenant(tenant.clone());
+        self.coordinator
+            .upsert_user(tenant.clone(), actor, CredentialKind::CliDeviceSession);
+        self.coordinator
+            .upsert_project(tenant.clone(), project.clone(), "local");
+        self.coordinator.upsert_source_provider_config(
+            tenant.clone(),
+            project.clone(),
+            SourceProviderKind::Filesystem,
+            Digest::sha256("local-filesystem"),
+        );
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.enrollment_grants.retain(|_, grant| {
+            !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
+        });
+        if self
+            .enrollment_grants
+            .values()
+            .filter(|grant| grant.tenant == tenant && grant.project == project)
+            .count()
+            >= super::MAX_ENROLLMENT_GRANTS_PER_PROJECT
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "node enrollment grant limit reached for this project; consume a grant or wait for one to expire"
+                    .to_owned(),
+            ));
+        }
+        let grant =
+            generate_opaque_token("node_grant").map_err(CoordinatorServiceError::Protocol)?;
+        let ttl_seconds = bounded_ttl(ttl_seconds, self.admission.max_node_enrollment_ttl_seconds);
+        let scope = "node:attach".to_owned();
+        let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
+        let enrollment = self.coordinator.create_node_enrollment_grant(
+            tenant.clone(),
+            project.clone(),
+            grant.clone(),
+            scope.clone(),
+            expires_at_epoch_seconds,
+        );
+        self.enrollment_grants
+            .insert(enrollment_grant_key(&tenant, &project, &grant), enrollment);
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeEnrollmentGrantCreated {
+            tenant,
+            project,
+            grant,
+            scope,
+            expires_at_epoch_seconds,
+        })
+    }
+
+    pub(super) fn handle_exchange_node_enrollment_grant(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+        enrollment_grant: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.enrollment_grants.retain(|_, grant| {
+            !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
+        });
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        if self.coordinator.node_identity(&node).is_none() {
+            self.quota.ensure_node_admission(
+                &tenant,
+                self.coordinator.node_identity_count_for_tenant(&tenant),
+            )?;
+        }
+        let grant_key = enrollment_grant_key(&tenant, &project, &enrollment_grant);
+        let grant =
+            self.enrollment_grants
+                .get_mut(&grant_key)
+                .ok_or(CoordinatorError::Enrollment(
+                    clusterflux_core::EnrollmentError::Expired,
+                ))?;
+        let credential = self.coordinator.exchange_node_enrollment_grant(
+            grant,
+            node.clone(),
+            &public_key,
+            "node:attach",
+            now_epoch_seconds,
+        )?;
+        self.enrollment_grants.remove(&grant_key);
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeEnrollmentExchanged {
+            node,
+            tenant,
+            project,
+            credential,
+        })
+    }
+
+    pub(super) fn handle_node_heartbeat(
+        &mut self,
+        node: String,
+        node_signature: Option<NodeSignedRequest>,
+        payload_digest: &Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let node = NodeId::new(node);
+        self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
+        Ok(CoordinatorResponse::NodeHeartbeat {
+            node,
+            epoch: self.coordinator.coordinator_epoch(),
+        })
+    }
+
+    pub(super) fn handle_report_node_capabilities(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        capabilities: NodeCapabilities,
+        cached_environment_digests: Vec<Digest>,
+        dependency_cache_digests: Vec<Digest>,
+        source_snapshots: Vec<Digest>,
+        artifact_locations: Vec<String>,
+        direct_connectivity: bool,
+        _online: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "node capability report is outside the enrolled tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        capabilities.validate_public_report()?;
+        for (kind, count) in [
+            ("cached environments", cached_environment_digests.len()),
+            ("dependency caches", dependency_cache_digests.len()),
+            ("source snapshots", source_snapshots.len()),
+            ("artifact locations", artifact_locations.len()),
+        ] {
+            if count > super::MAX_NODE_REPORTED_OBJECTS_PER_KIND {
+                return Err(CoordinatorServiceError::Protocol(format!(
+                    "node capability report contains {count} {kind}; limit is {}",
+                    super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+                )));
+            }
+        }
+        if cached_environment_digests
+            .iter()
+            .chain(&dependency_cache_digests)
+            .chain(&source_snapshots)
+            .any(|digest| !digest.is_valid_sha256())
+        {
+            return Err(CoordinatorServiceError::Protocol(
+                "node capability report contains an invalid digest".to_owned(),
+            ));
+        }
+        if artifact_locations.iter().any(|artifact| {
+            artifact.trim().is_empty()
+                || artifact.len() > 256
+                || artifact
+                    .chars()
+                    .any(|character| matches!(character, '/' | '\\' | '\0'))
+        }) {
+            return Err(CoordinatorServiceError::Protocol(
+                "node capability report contains an invalid artifact id".to_owned(),
+            ));
+        }
+        let artifact_locations = artifact_locations
+            .into_iter()
+            .map(ArtifactId::new)
+            .collect::<BTreeSet<_>>();
+        self.artifact_registry
+            .reconcile_node_retention(&node, &artifact_locations);
+
+        let online = self.node_is_live(&node);
+        self.node_descriptors.insert(
+            node.clone(),
+            NodeDescriptor {
+                id: node.clone(),
+                tenant,
+                project,
+                capabilities,
+                cached_environments: cached_environment_digests.into_iter().collect(),
+                dependency_caches: dependency_cache_digests.into_iter().collect(),
+                source_snapshots: source_snapshots.into_iter().collect(),
+                artifact_locations,
+                direct_connectivity,
+                online,
+            },
+        );
+        Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
+            node,
+            node_descriptors: self.node_descriptors.len(),
+        })
+    }
+
+    pub(super) fn handle_list_node_descriptors(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let descriptors = self
+            .live_node_descriptors()
+            .into_iter()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .collect();
+        Ok(CoordinatorResponse::NodeDescriptors { descriptors, actor })
+    }
+
+    pub(super) fn handle_revoke_node_credential(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let node = NodeId::new(node);
+        let context = clusterflux_core::AuthContext {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            actor: Actor::User(actor.clone()),
+        };
+        self.coordinator.revoke_node_credential(&context, &node)?;
+        let descriptor_removed = self.node_descriptors.remove(&node).is_some();
+        self.node_last_seen_epoch_seconds.remove(&node);
+        self.artifact_registry.garbage_collect_node(&node);
+        let queued_assignments_removed = self
+            .task_assignments
+            .remove(&(tenant.clone(), project.clone(), node.clone()))
+            .map_or(0, |assignments| assignments.len());
+        self.active_tasks
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_aborts
+            .retain(|(task_tenant, task_project, _, task_node, _)| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.task_placements
+            .retain(|(task_tenant, task_project, _, task_node, _), _| {
+                task_tenant != &tenant || task_project != &project || task_node != &node
+            });
+        self.persist_durable_state()?;
+        Ok(CoordinatorResponse::NodeCredentialRevoked {
+            node,
+            tenant,
+            project,
+            actor,
+            descriptor_removed,
+            queued_assignments_removed,
+        })
+    }
+
+    pub(super) fn authenticate_node_request(
+        &mut self,
+        node: &NodeId,
+        node_signature: Option<NodeSignedRequest>,
+        request_kind: &str,
+        payload_digest: &Digest,
+    ) -> Result<(), CoordinatorServiceError> {
+        let identity = self
+            .coordinator
+            .node_identity(node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        let signature = node_signature.ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "node request requires a signed proof of enrolled private-key possession"
+                    .to_owned(),
+            )
+        })?;
+        if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request nonce is missing or invalid".to_owned(),
+            )
+            .into());
+        }
+        let now_epoch_seconds = unix_timestamp_seconds();
+        if signature
+            .issued_at_epoch_seconds
+            .abs_diff(now_epoch_seconds)
+            > super::NODE_SIGNATURE_WINDOW_SECONDS
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request is expired or outside the allowed clock skew".to_owned(),
+            )
+            .into());
+        }
+        let replay_key = (node.clone(), signature.nonce.clone());
+        self.node_replay_nonces.retain(|_, accepted_at| {
+            now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
+        });
+        if self.node_replay_nonces.contains_key(&replay_key) {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request nonce has already been used".to_owned(),
+            )
+            .into());
+        }
+        verify_node_request_signature(
+            &identity.public_key,
+            node,
+            request_kind,
+            payload_digest,
+            &signature,
+        )
+        .map_err(CoordinatorError::Unauthorized)?;
+        if self
+            .node_replay_nonces
+            .keys()
+            .filter(|(retained_node, _)| retained_node == node)
+            .count()
+            >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "node signed request replay window is full; retry after the bounded signature window advances"
+                    .to_owned(),
+            )
+            .into());
+        }
+        self.node_replay_nonces
+            .insert(replay_key, now_epoch_seconds);
+        let seen_at = self.liveness_now_epoch_seconds();
+        self.node_last_seen_epoch_seconds
+            .insert(node.clone(), seen_at);
+        if let Some(descriptor) = self.node_descriptors.get_mut(node) {
+            descriptor.online = true;
+        }
+        Ok(())
+    }
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}

crates/clusterflux-coordinator/src/service/panels.rs

@@ -0,0 +1,307 @@
+use clusterflux_core::{
+    Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
+    ProcessId, ProjectId, RateLimit, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::artifact_id_from_path;
+use super::keys::panel_stop_key;
+use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+impl CoordinatorService {
+    pub(super) fn clear_operator_panel_state(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) {
+        let stop_key = panel_stop_key(tenant, project, process);
+        self.panel_snapshots.remove(&stop_key);
+        self.stopped_panels.remove(&stop_key);
+        self.panel_event_limits
+            .retain(|(event_tenant, event_project, event_process, _), _| {
+                event_tenant != tenant || event_project != project || event_process != process
+            });
+    }
+
+    pub(super) fn handle_render_operator_panel(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        max_download_bytes: u64,
+        stopped: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let stop_key = panel_stop_key(&tenant, &project, &process);
+
+        let panel = if stopped {
+            self.ensure_operator_panel_scope(&tenant, &project, &process)?;
+            self.stopped_panels.insert(stop_key);
+            let stop_key = panel_stop_key(&tenant, &project, &process);
+            let panel = match self.panel_snapshots.get(&stop_key).cloned() {
+                Some(panel) => stopped_panel_snapshot(panel),
+                None => self.render_operator_panel(
+                    tenant.clone(),
+                    project.clone(),
+                    process.clone(),
+                    UserId::new(actor_user),
+                    max_download_bytes,
+                    true,
+                )?,
+            };
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        } else {
+            self.stopped_panels.remove(&stop_key);
+            let panel = self.render_operator_panel(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                UserId::new(actor_user),
+                max_download_bytes,
+                false,
+            )?;
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        };
+        Ok(CoordinatorResponse::OperatorPanel { panel })
+    }
+
+    pub(super) fn handle_submit_panel_event(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        widget_id: String,
+        kind: PanelEventKind,
+        max_events: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let stop_key = panel_stop_key(&tenant, &project, &process);
+        let stopped = self.stopped_panels.contains(&stop_key);
+        let panel = if stopped {
+            match self.panel_snapshots.get(&stop_key).cloned() {
+                Some(panel) => stopped_panel_snapshot(panel),
+                None => {
+                    let panel = self.render_operator_panel(
+                        tenant.clone(),
+                        project.clone(),
+                        process.clone(),
+                        UserId::from("panel-user"),
+                        self.quota.download_limit(),
+                        true,
+                    )?;
+                    self.panel_snapshots.insert(stop_key.clone(), panel.clone());
+                    panel
+                }
+            }
+        } else {
+            let panel = self.render_operator_panel(
+                tenant.clone(),
+                project.clone(),
+                process.clone(),
+                UserId::from("panel-user"),
+                self.quota.download_limit(),
+                false,
+            )?;
+            self.panel_snapshots.insert(stop_key, panel.clone());
+            panel
+        };
+        let event = PanelEvent {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            widget_id: widget_id.clone(),
+            kind,
+        };
+        let limit_key = (tenant, project, process, widget_id);
+        let mut limit = self
+            .panel_event_limits
+            .get(&limit_key)
+            .cloned()
+            .unwrap_or(RateLimit {
+                max_events,
+                used_events: 0,
+            });
+        limit.max_events = max_events;
+        panel.accept_event(&event, &mut limit)?;
+        self.panel_event_limits.insert(limit_key, limit.clone());
+        Ok(CoordinatorResponse::PanelEventAccepted {
+            used_events: limit.used_events,
+            max_events: limit.max_events,
+        })
+    }
+
+    fn render_operator_panel(
+        &self,
+        tenant: TenantId,
+        project: ProjectId,
+        process: ProcessId,
+        actor_user: UserId,
+        max_download_bytes: u64,
+        stopped: bool,
+    ) -> Result<PanelState, CoordinatorServiceError> {
+        self.ensure_operator_panel_scope(&tenant, &project, &process)?;
+
+        let events = self
+            .task_events
+            .iter()
+            .filter(|event| {
+                event.tenant == tenant && event.project == project && event.process == process
+            })
+            .collect::<Vec<_>>();
+        let completed = events
+            .iter()
+            .filter(|event| event.status_code == Some(0))
+            .count() as u64;
+        let total = events.len().max(1) as u64;
+        let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::<u64>();
+        let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::<u64>();
+        let last_task = events.last().map(|event| event.task.clone());
+
+        let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone());
+        if stopped {
+            panel.freeze_program_ui_events();
+        }
+        panel.add_widget(PanelWidget {
+            id: "process-status".to_owned(),
+            label: "Process Status".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: if stopped {
+                    "stopped".to_owned()
+                } else {
+                    "running".to_owned()
+                },
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "task-progress".to_owned(),
+            label: "Tasks".to_owned(),
+            kind: PanelWidgetKind::Progress {
+                current: completed,
+                total,
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "task-summary".to_owned(),
+            label: "Task Summary".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: if events.is_empty() {
+                    "no task events recorded".to_owned()
+                } else {
+                    events
+                        .iter()
+                        .map(|event| {
+                            format!(
+                                "{} [{}]:{:?}:{}",
+                                event.task_definition, event.task, event.status_code, event.node
+                            )
+                        })
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                },
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "recent-logs".to_owned(),
+            label: "Recent Logs".to_owned(),
+            kind: PanelWidgetKind::Text {
+                value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"),
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "debug-process".to_owned(),
+            label: "Debug Process".to_owned(),
+            kind: PanelWidgetKind::Button {
+                action: "debug-process".to_owned(),
+            },
+        })?;
+        panel.add_widget(PanelWidget {
+            id: "cancel-process".to_owned(),
+            label: "Cancel Process".to_owned(),
+            kind: PanelWidgetKind::Button {
+                action: "cancel-process".to_owned(),
+            },
+        })?;
+        if last_task.is_some() {
+            panel.add_widget(PanelWidget {
+                id: "restart-selected-task".to_owned(),
+                label: "Restart Selected Task".to_owned(),
+                kind: PanelWidgetKind::Button {
+                    action: "restart-task".to_owned(),
+                },
+            })?;
+        }
+
+        let mut actions = vec![
+            clusterflux_core::ControlPlaneAction::DebugProcess,
+            clusterflux_core::ControlPlaneAction::CancelProcess,
+        ];
+        if let Some(task) = last_task.clone() {
+            actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
+        }
+        panel.set_control_plane_actions(actions);
+
+        if let Some(path) = events
+            .iter()
+            .rev()
+            .find_map(|event| event.artifact_path.as_ref())
+        {
+            let artifact = artifact_id_from_path(path);
+            let context = clusterflux_core::AuthContext {
+                tenant,
+                project,
+                actor: Actor::User(actor_user),
+            };
+            panel.add_download_widget_from_action(
+                "download-artifact",
+                "Download Artifact",
+                self.artifact_registry.download_action(
+                    &context,
+                    &artifact,
+                    &DownloadPolicy {
+                        max_bytes: max_download_bytes,
+                    },
+                ),
+            )?;
+        }
+
+        Ok(panel)
+    }
+
+    fn ensure_operator_panel_scope(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+    ) -> Result<(), CoordinatorServiceError> {
+        let active = self
+            .coordinator
+            .active_process(tenant, project, process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "operator panel requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, *tenant);
+        debug_assert_eq!(active.project, *project);
+        Ok(())
+    }
+}
+
+fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState {
+    panel.freeze_program_ui_events();
+    if let Some(status) = panel.widgets.get_mut("process-status") {
+        status.kind = PanelWidgetKind::Text {
+            value: "stopped".to_owned(),
+        };
+    }
+    panel
+}

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -0,0 +1,804 @@
+use std::collections::BTreeMap;
+
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
+use clusterflux_core::{
+    AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
+    NodeDescriptor, NodeId, Placement, PlacementError, PlacementRequest, ProcessId, ProjectId,
+    Scheduler, TaskBoundaryValue, TaskCheckpoint, TaskDispatch, TaskInstanceId, TaskSpec, TenantId,
+    VfsManifest, VfsObject, VfsPath, WasmTaskInvocation,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key, task_restart_key};
+use super::{
+    CoordinatorResponse, CoordinatorService, CoordinatorServiceError, TaskAssignment, WorkflowActor,
+};
+
+use super::processes::*;
+use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
+
+fn select_task_placement(
+    candidates: &[Placement],
+    active_by_node: &BTreeMap<NodeId, usize>,
+) -> Option<Placement> {
+    candidates
+        .iter()
+        .min_by(|left, right| {
+            right
+                .score
+                .cmp(&left.score)
+                .then_with(|| {
+                    active_by_node
+                        .get(&left.node)
+                        .copied()
+                        .unwrap_or_default()
+                        .cmp(&active_by_node.get(&right.node).copied().unwrap_or_default())
+                })
+                .then_with(|| left.node.cmp(&right.node))
+        })
+        .cloned()
+}
+
+impl CoordinatorService {
+    fn place_workflow_task(
+        &self,
+        nodes: &[NodeDescriptor],
+        request: &PlacementRequest,
+    ) -> Result<Placement, PlacementError> {
+        let candidates = nodes
+            .iter()
+            .filter_map(|node| {
+                DefaultScheduler
+                    .place(std::slice::from_ref(node), request)
+                    .ok()
+            })
+            .collect::<Vec<_>>();
+        if candidates.is_empty() {
+            return DefaultScheduler.place(nodes, request);
+        }
+        let active_by_node = self
+            .active_tasks
+            .iter()
+            .filter(|(tenant, project, _, _, _)| {
+                tenant == &request.tenant && project == &request.project
+            })
+            .fold(BTreeMap::<NodeId, usize>::new(), |mut counts, key| {
+                *counts.entry(key.3.clone()).or_default() += 1;
+                counts
+            });
+        let mut selected = select_task_placement(&candidates, &active_by_node)
+            .expect("one or more compatible task-placement candidates should remain");
+        let selected_load = active_by_node
+            .get(&selected.node)
+            .copied()
+            .unwrap_or_default();
+        if candidates.iter().any(|candidate| {
+            candidate.score == selected.score
+                && active_by_node
+                    .get(&candidate.node)
+                    .copied()
+                    .unwrap_or_default()
+                    > selected_load
+        }) {
+            selected.reasons.push(format!(
+                "least active equal-locality node ({selected_load} active assignment(s))"
+            ));
+        }
+        Ok(selected)
+    }
+
+    pub(super) fn capture_task_restart_checkpoint(
+        &mut self,
+        assignment: &TaskAssignment,
+    ) -> Result<(), CoordinatorServiceError> {
+        let task_spec = &assignment.task_spec;
+        let environment_digest = task_spec.environment_digest.clone().unwrap_or_else(|| {
+            task_spec.environment.as_ref().map_or_else(
+                || Digest::sha256("clusterflux.environment.unconstrained.v1"),
+                |environment| {
+                    Digest::sha256(
+                        serde_json::to_vec(environment)
+                            .expect("serializable environment requirements"),
+                    )
+                },
+            )
+        });
+        let task_entrypoint = match &task_spec.dispatch {
+            clusterflux_core::TaskDispatch::CoordinatorNodeWasm { export, .. } => export
+                .clone()
+                .or_else(|| assignment_task_descriptor(assignment)?.get("export")?.as_str().map(str::to_owned))
+                .ok_or_else(|| {
+                    CoordinatorServiceError::Protocol(format!(
+                        "cannot capture restart checkpoint for task `{}`: bundle descriptor omitted its Wasm export",
+                        task_spec.task_definition
+                    ))
+                })?,
+        };
+        let mut objects = BTreeMap::new();
+        let mut missing_required_artifact = false;
+        for artifact in &task_spec.required_artifacts {
+            let Some(metadata) = self.artifact_registry.metadata(artifact) else {
+                missing_required_artifact = true;
+                continue;
+            };
+            if metadata.tenant != assignment.tenant
+                || metadata.project != assignment.project
+                || metadata.retaining_nodes.is_empty()
+            {
+                missing_required_artifact = true;
+                continue;
+            }
+            let path = VfsPath::new(format!("/vfs/artifacts/{artifact}"))
+                .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
+            objects.insert(
+                path.clone(),
+                VfsObject {
+                    path,
+                    digest: metadata.digest.clone(),
+                    size: metadata.size,
+                    producer: metadata.producer_task.clone(),
+                    node: metadata.producer_node.clone(),
+                },
+            );
+        }
+        let checkpoint = TaskCheckpoint {
+            task: assignment.task.clone(),
+            boundary: CheckpointBoundary {
+                task_entrypoint,
+                serialized_args: Digest::sha256(serde_json::to_vec(&task_spec.args)?),
+                environment_digest,
+                vfs_epoch: task_spec.vfs_epoch,
+                task_abi: assignment_task_compatibility(assignment)
+                    .unwrap_or(Digest::sha256(serde_json::to_vec(&task_spec.dispatch)?)),
+            },
+            vfs_manifest: VfsManifest {
+                epoch: task_spec.vfs_epoch,
+                producer: assignment.task.clone(),
+                node: assignment.node.clone(),
+                objects,
+                large_bytes_uploaded: false,
+            },
+            depends_on_live_stack: false,
+            depends_on_live_socket: false,
+            depends_on_ephemeral_artifact_durability: missing_required_artifact,
+        };
+        let key = task_restart_key(
+            &assignment.tenant,
+            &assignment.project,
+            &assignment.process,
+            &assignment.task,
+        );
+        self.task_restart_checkpoint_order
+            .retain(|retained| retained != &key);
+        self.task_restart_checkpoints.insert(
+            key.clone(),
+            TaskRestartCheckpoint {
+                checkpoint,
+                assignment: assignment.clone(),
+            },
+        );
+        self.task_restart_checkpoint_order.push_back(key);
+        while self
+            .task_restart_checkpoint_order
+            .iter()
+            .filter(|(tenant, project, process, _)| {
+                tenant == &assignment.tenant
+                    && project == &assignment.project
+                    && process == &assignment.process
+            })
+            .count()
+            > super::MAX_RESTART_CHECKPOINTS_PER_PROCESS
+        {
+            let Some(index) = self.task_restart_checkpoint_order.iter().position(
+                |(tenant, project, process, _)| {
+                    tenant == &assignment.tenant
+                        && project == &assignment.project
+                        && process == &assignment.process
+                },
+            ) else {
+                break;
+            };
+            if let Some(expired) = self.task_restart_checkpoint_order.remove(index) {
+                self.task_restart_checkpoints.remove(&expired);
+            }
+        }
+        while self.task_restart_checkpoint_order.len() > super::MAX_RESTART_CHECKPOINTS_TOTAL {
+            if let Some(expired) = self.task_restart_checkpoint_order.pop_front() {
+                self.task_restart_checkpoints.remove(&expired);
+            }
+        }
+        Ok(())
+    }
+
+    pub(super) fn handle_schedule_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        environment: Option<clusterflux_core::EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<clusterflux_core::Capability>,
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let request = PlacementRequest {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            environment,
+            environment_digest,
+            environment_cache_required: false,
+            required_capabilities: required_capabilities.into_iter().collect(),
+            dependency_cache,
+            source_snapshot,
+            required_artifacts: required_artifacts
+                .into_iter()
+                .map(ArtifactId::new)
+                .collect(),
+            quota_available: self
+                .quota
+                .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
+                .is_ok(),
+            policy_allowed: self.admission.workflow_placement_allowed,
+            prefer_node: prefer_node.map(NodeId::new),
+        };
+        let nodes = self.live_node_descriptors();
+        let placement = DefaultScheduler.place(&nodes, &request)?;
+        Ok(CoordinatorResponse::TaskPlacement { placement })
+    }
+
+    pub(super) fn handle_launch_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        task_spec: TaskSpec,
+        _wait_for_node: bool,
+        _artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        task_spec
+            .validate_boundary_authority()
+            .map_err(CoordinatorServiceError::Protocol)?;
+        if matches!(
+            &task_spec.dispatch,
+            TaskDispatch::CoordinatorNodeWasm {
+                abi: clusterflux_core::WasmExportAbi::EntrypointV1,
+                ..
+            }
+        ) {
+            return self.handle_launch_coordinator_main(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                request_payload_digest,
+                task_spec,
+                wasm_module_base64,
+            );
+        }
+        Err(CoordinatorError::Unauthorized(
+            "external callers may launch only EntrypointV1; TaskV1 requires an authenticated live parent runtime or validated restart"
+                .to_owned(),
+        )
+        .into())
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn handle_launch_child_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task_spec: TaskSpec,
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let parent_task = TaskInstanceId::new(parent_task);
+        if task_spec.process != process {
+            return Err(CoordinatorError::Unauthorized(
+                "child task must remain in its parent virtual process".to_owned(),
+            )
+            .into());
+        }
+        if !matches!(
+            task_spec.dispatch,
+            TaskDispatch::CoordinatorNodeWasm {
+                abi: clusterflux_core::WasmExportAbi::TaskV1,
+                ..
+            }
+        ) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task launch requires the TaskV1 ABI".to_owned(),
+            )
+            .into());
+        }
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let parent_key = task_control_key(&tenant, &project, &process, &node, &parent_task);
+        if !self.active_tasks.contains(&parent_key) {
+            return Err(CoordinatorError::Unauthorized(
+                "child task launch requires a currently active parent task on the signed node"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let actor = WorkflowActor {
+            kind: "task".to_owned(),
+            user: None,
+            agent: None,
+            credential_kind: CredentialKind::TaskCredential,
+            public_key_fingerprint: None,
+            authenticated_without_browser: true,
+            scopes: vec!["process:spawn-child".to_owned()],
+        };
+        self.handle_launch_task_with_actor(
+            tenant,
+            project,
+            actor,
+            task_spec,
+            wait_for_node,
+            artifact_path,
+            wasm_module_base64,
+        )
+    }
+
+    pub(super) fn handle_launch_task_with_actor(
+        &mut self,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: WorkflowActor,
+        task_spec: TaskSpec,
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        task_spec
+            .validate_boundary_authority()
+            .map_err(CoordinatorServiceError::Protocol)?;
+        if task_spec.tenant != tenant || task_spec.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification is outside the authenticated tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        if !task_spec.product_mode_uses_remote_dispatch() {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification must use the Wasm coordinator/node dispatch ABI".to_owned(),
+            )
+            .into());
+        }
+        if task_spec
+            .environment_id
+            .as_deref()
+            .is_some_and(|environment| environment.trim().is_empty() || environment.len() > 128)
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "task specification environment id is invalid".to_owned(),
+            )
+            .into());
+        }
+        let process = task_spec.process.clone();
+        let task = task_spec.task_instance.clone();
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .cloned()
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "task launch requires an active coordinator-side virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+        if self
+            .process_cancellations
+            .contains(&process_control_key(&tenant, &project, &process))
+        {
+            return Err(CoordinatorError::Unauthorized(
+                "task launch is blocked because the virtual process is cancelling".to_owned(),
+            )
+            .into());
+        }
+        if self.task_instance_exists(&tenant, &project, &process, &task) {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "task instance {task} already exists in virtual process {process}; every spawn must use a fresh task-instance id"
+            )));
+        }
+        let in_flight = self
+            .active_tasks
+            .iter()
+            .filter(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant == &tenant && task_project == &project && task_process == &process
+            })
+            .count()
+            + self
+                .pending_task_launches
+                .iter()
+                .filter(|pending| {
+                    pending.tenant == tenant
+                        && pending.project == project
+                        && pending.process == process
+                })
+                .count();
+        if in_flight >= super::MAX_IN_FLIGHT_TASKS_PER_PROCESS {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "virtual process task limit of {} reached; join or cancel existing work before spawning more",
+                super::MAX_IN_FLIGHT_TASKS_PER_PROCESS
+            )));
+        }
+        if task_spec.vfs_epoch != active.coordinator_epoch {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "task specification VFS epoch {} does not match active process epoch {}",
+                task_spec.vfs_epoch, active.coordinator_epoch
+            ))
+            .into());
+        }
+        let bundle_digest = task_spec.bundle_digest.as_ref().ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "Wasm task specification is missing its bundle digest".to_owned(),
+            )
+        })?;
+        if !bundle_digest.is_valid_sha256() {
+            return Err(CoordinatorError::Unauthorized(
+                "Wasm task specification has an invalid bundle digest".to_owned(),
+            )
+            .into());
+        }
+        let module = BASE64_STANDARD
+            .decode(&wasm_module_base64)
+            .map_err(|error| {
+                CoordinatorServiceError::Protocol(format!(
+                    "Wasm task module is not valid base64: {error}"
+                ))
+            })?;
+        let actual_digest = Digest::sha256(&module);
+        if &actual_digest != bundle_digest {
+            return Err(CoordinatorError::Unauthorized(format!(
+                "Wasm task module digest does not match bundle digest: expected {bundle_digest}, actual {actual_digest}"
+            ))
+            .into());
+        }
+        WasmTaskInvocation::new(
+            task_spec.task_definition.clone(),
+            task.clone(),
+            task_spec.args.clone(),
+        )
+        .validate()
+        .map_err(CoordinatorServiceError::Protocol)?;
+        for artifact in &task_spec.required_artifacts {
+            let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
+                CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} is unavailable or has expired"
+                ))
+            })?;
+            if metadata.tenant != tenant || metadata.project != project {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} is outside the task tenant/project scope"
+                ))
+                .into());
+            }
+            if metadata.retaining_nodes.is_empty() {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "required artifact {artifact} has no retaining node"
+                ))
+                .into());
+            }
+        }
+        VfsPath::new(&artifact_path)
+            .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        let request = PlacementRequest {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            environment: task_spec.environment.clone(),
+            environment_digest: task_spec.environment_digest.clone(),
+            environment_cache_required: task_spec.environment_id.is_some()
+                && task_spec.environment.is_none(),
+            required_capabilities: task_spec.required_capabilities.clone(),
+            dependency_cache: task_spec.dependency_cache.clone(),
+            source_snapshot: task_spec.source_snapshot.clone(),
+            required_artifacts: task_spec.required_artifacts.iter().cloned().collect(),
+            quota_available: self
+                .quota
+                .can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
+                .is_ok(),
+            policy_allowed: self.admission.workflow_placement_allowed,
+            prefer_node: None,
+        };
+        let nodes = self.live_node_descriptors();
+        let placement = match self.place_workflow_task(&nodes, &request) {
+            Ok(placement) => placement,
+            Err(err) if wait_for_node => {
+                let reason = if err.message.is_empty() {
+                    "waiting for any capable node".to_owned()
+                } else {
+                    err.message
+                };
+                let charged_spawns =
+                    self.quota
+                        .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+                self.begin_task_attempt(&task_spec, None, Some(&artifact_path), true)?;
+                self.pending_task_launches.push_back(PendingTaskLaunch {
+                    tenant: tenant.clone(),
+                    project: project.clone(),
+                    process: process.clone(),
+                    task: task.clone(),
+                    request,
+                    epoch: active.coordinator_epoch,
+                    artifact_path,
+                    task_spec,
+                    wasm_module_base64,
+                });
+                return Ok(CoordinatorResponse::TaskQueued {
+                    process,
+                    task,
+                    actor,
+                    reason,
+                    charged_spawns,
+                    queued_tasks: self.pending_task_launches.len(),
+                });
+            }
+            Err(err) => return Err(err.into()),
+        };
+        let charged_spawns =
+            self.quota
+                .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        let assignment = TaskAssignment {
+            tenant: tenant.clone(),
+            project: project.clone(),
+            process: process.clone(),
+            task: task.clone(),
+            node: placement.node.clone(),
+            epoch: active.coordinator_epoch,
+            artifact_path,
+            task_spec,
+            wasm_module_base64,
+        };
+        self.begin_task_attempt(
+            &assignment.task_spec,
+            Some(assignment.node.clone()),
+            Some(&assignment.artifact_path),
+            false,
+        )?;
+        self.capture_task_restart_checkpoint(&assignment)?;
+        let task_key = task_control_key(&tenant, &project, &process, &placement.node, &task);
+        self.task_placements
+            .insert(task_key.clone(), placement.clone());
+        self.active_tasks.insert(task_key);
+        self.task_assignments
+            .entry((tenant, project, placement.node.clone()))
+            .or_default()
+            .push_back(assignment.clone());
+        Ok(CoordinatorResponse::TaskLaunched {
+            process,
+            task,
+            actor,
+            placement,
+            assignment: Box::new(assignment),
+            charged_spawns,
+        })
+    }
+
+    fn task_instance_exists(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        process: &ProcessId,
+        task_instance: &clusterflux_core::TaskInstanceId,
+    ) -> bool {
+        self.active_tasks.iter().any(
+            |(task_tenant, task_project, task_process, _, existing_instance)| {
+                task_tenant == tenant
+                    && task_project == project
+                    && task_process == process
+                    && existing_instance == task_instance
+            },
+        ) || self.pending_task_launches.iter().any(|pending| {
+            &pending.tenant == tenant
+                && &pending.project == project
+                && &pending.process == process
+                && &pending.task == task_instance
+        }) || (!self.restart_launches.contains(&task_restart_key(
+            tenant,
+            project,
+            process,
+            task_instance,
+        )) && self.task_events.iter().any(|event| {
+            &event.tenant == tenant
+                && &event.project == project
+                && &event.process == process
+                && &event.task == task_instance
+        }))
+    }
+
+    pub(super) fn begin_task_attempt(
+        &mut self,
+        task_spec: &TaskSpec,
+        node: Option<NodeId>,
+        artifact_path: Option<&str>,
+        queued: bool,
+    ) -> Result<String, CoordinatorServiceError> {
+        let key = task_restart_key(
+            &task_spec.tenant,
+            &task_spec.project,
+            &task_spec.process,
+            &task_spec.task_instance,
+        );
+        while !self.task_attempts.contains_key(&key)
+            && self.task_attempts.len() >= super::MAX_TASK_ATTEMPT_HISTORIES
+        {
+            let removable = self.task_attempts.iter().find_map(|(candidate, attempts)| {
+                attempts
+                    .iter()
+                    .all(|attempt| {
+                        !matches!(
+                            &attempt.state,
+                            TaskAttemptState::Queued
+                                | TaskAttemptState::Running
+                                | TaskAttemptState::FailedAwaitingAction
+                        )
+                    })
+                    .then(|| candidate.clone())
+            });
+            let Some(removable) = removable else {
+                return Err(CoordinatorServiceError::Protocol(
+                    "task attempt history capacity is exhausted by active attempts".to_owned(),
+                ));
+            };
+            self.task_attempts.remove(&removable);
+        }
+        let attempts = self.task_attempts.entry(key).or_default();
+        for attempt in attempts.iter_mut() {
+            attempt.current = false;
+        }
+        let attempt_id = clusterflux_core::generate_opaque_token("ta")
+            .map_err(CoordinatorServiceError::Protocol)?;
+        let attempt_number = u32::try_from(attempts.len() + 1).unwrap_or(u32::MAX);
+        let mut argument_summary = task_spec
+            .args
+            .iter()
+            .map(|argument| {
+                let mut value = serde_json::to_string(argument)
+                    .unwrap_or_else(|_| "<invalid canonical argument>".to_owned());
+                value.truncate(value.len().min(1024));
+                value
+            })
+            .collect::<Vec<_>>();
+        argument_summary.truncate(64);
+        let mut handle_summary = task_spec
+            .required_artifacts
+            .iter()
+            .map(|artifact| format!("artifact:{artifact}"))
+            .collect::<Vec<_>>();
+        for argument in &task_spec.args {
+            if let TaskBoundaryValue::Structured(boundary) = argument {
+                handle_summary.extend(boundary.handles.iter().map(|handle| format!("{handle:?}")));
+            }
+        }
+        handle_summary.truncate(256);
+        attempts.push(TaskAttemptSnapshot {
+            process: task_spec.process.clone(),
+            task: task_spec.task_instance.clone(),
+            attempt_id: attempt_id.clone(),
+            attempt_number,
+            task_definition: task_spec.task_definition.clone(),
+            display_name: task_spec.task_definition.as_str().replace(['_', '-'], " "),
+            state: if queued {
+                TaskAttemptState::Queued
+            } else {
+                TaskAttemptState::Running
+            },
+            current: true,
+            node,
+            environment_id: task_spec.environment_id.clone(),
+            environment_digest: task_spec.environment_digest.clone(),
+            argument_summary,
+            handle_summary,
+            command_state: Some(if queued { "queued" } else { "running" }.to_owned()),
+            vfs_checkpoint: format!("vfs-epoch:{}", task_spec.vfs_epoch),
+            probe_symbol: None,
+            source_path: None,
+            source_line: None,
+            restart_compatible: true,
+            failure_policy: task_spec.failure_policy,
+            artifact_path: artifact_path.and_then(|path| VfsPath::new(path).ok()),
+            artifact_digest: None,
+            artifact_size_bytes: None,
+            status_code: None,
+            error: None,
+        });
+        if attempts.len() > 128 {
+            attempts.remove(0);
+        }
+        Ok(attempt_id)
+    }
+
+    pub(super) fn assign_task_attempt(&mut self, task_spec: &TaskSpec, node: NodeId) {
+        let key = task_restart_key(
+            &task_spec.tenant,
+            &task_spec.project,
+            &task_spec.process,
+            &task_spec.task_instance,
+        );
+        if let Some(attempt) = self
+            .task_attempts
+            .get_mut(&key)
+            .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
+        {
+            attempt.node = Some(node);
+            attempt.state = TaskAttemptState::Running;
+            attempt.command_state = Some("running".to_owned());
+        }
+    }
+}
+
+fn assignment_task_compatibility(assignment: &TaskAssignment) -> Option<Digest> {
+    let descriptor = assignment_task_descriptor(assignment)?;
+    serde_json::from_value(descriptor.get("restart_compatibility_hash")?.clone()).ok()
+}
+
+fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option<serde_json::Value> {
+    let module = BASE64_STANDARD
+        .decode(&assignment.wasm_module_base64)
+        .ok()?;
+    let mut descriptors = super::main_runtime::task_descriptors(&module).ok()?;
+    descriptors.remove(assignment.task_spec.task_definition.as_str())
+}
+
+#[cfg(test)]
+mod placement_tests {
+    use super::*;
+
+    fn placement(node: &str, score: i64) -> Placement {
+        Placement {
+            node: NodeId::from(node),
+            score,
+            reasons: Vec::new(),
+        }
+    }
+
+    #[test]
+    fn equal_locality_prefers_the_least_active_node() {
+        let candidates = vec![placement("busy", 40), placement("idle", 40)];
+        let active = BTreeMap::from([(NodeId::from("busy"), 1)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("idle")
+        );
+    }
+
+    #[test]
+    fn locality_score_remains_stronger_than_load_balancing() {
+        let candidates = vec![placement("warm", 50), placement("cold", 40)];
+        let active = BTreeMap::from([(NodeId::from("warm"), 2)]);
+
+        assert_eq!(
+            select_task_placement(&candidates, &active)
+                .expect("one placement")
+                .node,
+            NodeId::from("warm")
+        );
+    }
+}

crates/clusterflux-coordinator/src/service/processes.rs

@@ -0,0 +1,815 @@
+use std::collections::{BTreeSet, VecDeque};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use clusterflux_core::{
+    AgentId, AgentSignedRequest, CredentialKind, DefaultScheduler, Digest, NodeId,
+    PlacementRequest, ProcessId, ProjectId, RendezvousRequest, Scheduler, SourcePreparation,
+    TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
+};
+
+use crate::CoordinatorError;
+
+use super::keys::{process_control_key, task_control_key};
+use super::{
+    CoordinatorResponse, CoordinatorService, CoordinatorServiceError, SourcePreparationDisposition,
+    SourcePreparationStatus, TaskAssignment, TaskCancellationTarget, VirtualProcessStatus,
+    WorkflowActor,
+};
+
+#[derive(Clone, Debug)]
+pub(super) struct PendingTaskLaunch {
+    pub(super) tenant: TenantId,
+    pub(super) project: ProjectId,
+    pub(super) process: ProcessId,
+    pub(super) task: TaskInstanceId,
+    pub(super) request: PlacementRequest,
+    pub(super) epoch: u64,
+    pub(super) artifact_path: String,
+    pub(super) task_spec: TaskSpec,
+    pub(super) wasm_module_base64: String,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct TaskRestartCheckpoint {
+    pub(super) checkpoint: TaskCheckpoint,
+    pub(super) assignment: TaskAssignment,
+}
+
+impl CoordinatorService {
+    pub(super) fn handle_poll_task_assignment(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "task assignment poll is outside the enrolled tenant/project scope".to_owned(),
+            )
+            .into());
+        }
+        let assignment_key = (tenant.clone(), project.clone(), node.clone());
+        let assignment = self
+            .task_assignments
+            .get_mut(&assignment_key)
+            .and_then(VecDeque::pop_front);
+        if assignment.is_some() {
+            return Ok(CoordinatorResponse::TaskAssignment {
+                assignment: assignment.map(Box::new),
+            });
+        }
+        let assignment = self.assign_pending_task_to_node(&tenant, &project, &node)?;
+        Ok(CoordinatorResponse::TaskAssignment {
+            assignment: assignment.map(Box::new),
+        })
+    }
+
+    fn assign_pending_task_to_node(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        node: &NodeId,
+    ) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
+        let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
+            return Ok(None);
+        };
+        let mut remaining = VecDeque::new();
+        let mut selected = None;
+
+        while let Some(pending) = self.pending_task_launches.pop_front() {
+            if selected.is_some() {
+                remaining.push_back(pending);
+                continue;
+            }
+            if &pending.tenant != tenant || &pending.project != project {
+                remaining.push_back(pending);
+                continue;
+            }
+            if self.process_cancellations.contains(&process_control_key(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+            )) {
+                continue;
+            }
+            let Some(active) = self.coordinator.active_process(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+            ) else {
+                continue;
+            };
+            if active.tenant != pending.tenant
+                || active.project != pending.project
+                || active.coordinator_epoch != pending.epoch
+            {
+                continue;
+            }
+            let Ok(placement) =
+                DefaultScheduler.place(std::slice::from_ref(&descriptor), &pending.request)
+            else {
+                remaining.push_back(pending);
+                continue;
+            };
+            let assignment = TaskAssignment {
+                tenant: pending.tenant.clone(),
+                project: pending.project.clone(),
+                process: pending.process.clone(),
+                task: pending.task.clone(),
+                node: placement.node.clone(),
+                epoch: pending.epoch,
+                artifact_path: pending.artifact_path,
+                task_spec: pending.task_spec,
+                wasm_module_base64: pending.wasm_module_base64,
+            };
+            self.assign_task_attempt(&assignment.task_spec, assignment.node.clone());
+            self.capture_task_restart_checkpoint(&assignment)?;
+            let task_key = task_control_key(
+                &pending.tenant,
+                &pending.project,
+                &pending.process,
+                &placement.node,
+                &pending.task,
+            );
+            self.task_placements.insert(task_key.clone(), placement);
+            self.active_tasks.insert(task_key);
+            selected = Some(assignment);
+        }
+
+        self.pending_task_launches = remaining;
+        Ok(selected)
+    }
+
+    pub(super) fn handle_request_rendezvous(
+        &mut self,
+        scope: clusterflux_core::DataPlaneScope,
+        source: clusterflux_core::NodeEndpoint,
+        destination: clusterflux_core::NodeEndpoint,
+        direct_connectivity: bool,
+        failure_reason: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let charged_rendezvous_attempts = self.quota.charge_rendezvous_attempt(
+            &scope.tenant,
+            &scope.project,
+            now_epoch_seconds,
+        )?;
+        let plan = self.transport.plan_authenticated_direct_bulk_transfer(
+            RendezvousRequest {
+                scope,
+                source,
+                destination,
+            },
+            direct_connectivity,
+            failure_reason,
+        )?;
+        Ok(CoordinatorResponse::RendezvousPlan {
+            plan,
+            charged_rendezvous_attempts,
+        })
+    }
+
+    pub(super) fn handle_request_source_preparation(
+        &mut self,
+        tenant: String,
+        project: String,
+        provider: clusterflux_core::SourceProviderKind,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let preparation = SourcePreparation::node_task(tenant.clone(), project.clone(), provider);
+        let request = PlacementRequest {
+            tenant,
+            project,
+            environment: None,
+            environment_digest: None,
+            environment_cache_required: false,
+            required_capabilities: preparation.required_capabilities.clone(),
+            dependency_cache: None,
+            source_snapshot: None,
+            required_artifacts: Default::default(),
+            quota_available: true,
+            policy_allowed: true,
+            prefer_node: None,
+        };
+        let nodes = self.live_node_descriptors();
+        let disposition = match DefaultScheduler.place(&nodes, &request) {
+            Ok(placement) => SourcePreparationDisposition::Assigned {
+                node: placement.node,
+            },
+            Err(err) => SourcePreparationDisposition::Pending {
+                reason: if err.message.is_empty() {
+                    "waiting for any capable node to prepare source".to_owned()
+                } else {
+                    err.message
+                },
+            },
+        };
+        Ok(CoordinatorResponse::SourcePreparation {
+            status: SourcePreparationStatus {
+                preparation,
+                disposition,
+            },
+        })
+    }
+
+    pub(super) fn handle_complete_source_preparation(
+        &mut self,
+        tenant: String,
+        project: String,
+        node: String,
+        provider: clusterflux_core::SourceProviderKind,
+        source_snapshot: Digest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let node = NodeId::new(node);
+        let identity = self
+            .coordinator
+            .node_identity(&node)
+            .ok_or(CoordinatorError::UnknownNode)?;
+        if identity.tenant != tenant || identity.project != project {
+            return Err(CoordinatorError::Unauthorized(
+                "source preparation completion is outside the enrolled tenant/project scope"
+                    .to_owned(),
+            )
+            .into());
+        }
+        let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
+            CoordinatorError::Unauthorized(
+                "source preparation completion requires a node capability report".to_owned(),
+            )
+        })?;
+        if !descriptor.source_snapshots.contains(&source_snapshot)
+            && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+        {
+            return Err(CoordinatorServiceError::Protocol(format!(
+                "node source snapshot retention limit of {} reached; refresh the node capability report",
+                super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
+            )));
+        }
+        descriptor.source_snapshots.insert(source_snapshot.clone());
+        Ok(CoordinatorResponse::SourcePreparationCompleted {
+            node,
+            provider,
+            source_snapshot,
+        })
+    }
+
+    pub(super) fn handle_start_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        process: String,
+        restart: bool,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        self.coordinator.ensure_tenant_active(&tenant)?;
+        let actor = self.workflow_actor(
+            &tenant,
+            &project,
+            actor_user,
+            actor_agent,
+            agent_public_key_fingerprint,
+            agent_signature,
+            request_payload_digest,
+            "start_process",
+            &process,
+            None,
+        )?;
+        let replacing_existing = if let Some(active) = self
+            .coordinator
+            .active_process_for_project(&tenant, &project)
+        {
+            if active.id != process || !restart {
+                return Err(CoordinatorError::Unauthorized(format!(
+                    "project already has active virtual process {}; attach to or restart it, request cooperative cancellation, abort it, or use another Coordinator Project",
+                    active.id
+                ))
+                .into());
+            }
+            true
+        } else {
+            false
+        };
+        if !replacing_existing {
+            self.quota.ensure_process_admission(
+                &tenant,
+                self.coordinator.active_process_count_for_tenant(&tenant),
+            )?;
+        }
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        let charged_spawns =
+            self.quota
+                .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
+        if replacing_existing {
+            self.main_runtime.interrupt_process(
+                &tenant,
+                &project,
+                &process,
+                "virtual process incarnation replaced",
+            );
+            self.main_runtime
+                .controls
+                .remove(&process_control_key(&tenant, &project, &process));
+        }
+        self.process_cancellations
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.process_aborts
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.clear_operator_panel_state(&tenant, &project, &process);
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_aborts
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.active_tasks
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_placements
+            .retain(|(task_tenant, task_project, task_process, _, _), _| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.task_assignments.retain(|_, assignments| {
+            assignments.retain(|assignment| {
+                assignment.tenant != tenant
+                    || assignment.project != project
+                    || assignment.process != process
+            });
+            !assignments.is_empty()
+        });
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        self.task_restart_checkpoints.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, _), _| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+            },
+        );
+        self.task_restart_checkpoint_order.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, _)| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+            },
+        );
+        self.task_events.retain(|event| {
+            event.tenant != tenant || event.project != project || event.process != process
+        });
+        self.task_attempts
+            .retain(|(attempt_tenant, attempt_project, attempt_process, _), _| {
+                attempt_tenant != &tenant
+                    || attempt_project != &project
+                    || attempt_process != &process
+            });
+        self.restart_launches
+            .retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
+                attempt_tenant != &tenant
+                    || attempt_project != &project
+                    || attempt_process != &process
+            });
+        self.debug_audit_events.retain(|event| {
+            event.tenant != tenant || event.project != project || event.process != process
+        });
+        self.coordinator
+            .start_process(tenant, project, process.clone());
+        Ok(CoordinatorResponse::ProcessStarted {
+            process,
+            epoch: self.coordinator.coordinator_epoch(),
+            actor,
+            charged_spawns,
+        })
+    }
+
+    pub(super) fn handle_reconnect_node(
+        &mut self,
+        node: String,
+        process: String,
+        epoch: u64,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let node = NodeId::new(node);
+        let process = ProcessId::new(process);
+        self.coordinator
+            .reconnect_node(&node, Some((&process, epoch)))?;
+        Ok(CoordinatorResponse::NodeReconnected { node, process })
+    }
+
+    pub(super) fn handle_cancel_task(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.coordinator
+            .authorize_node_for_process(&node, &tenant, &project, &process)?;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "task cancellation requires an active virtual process".to_owned(),
+                )
+            })?;
+        if !active.connected_nodes.contains(&node) {
+            return Err(CoordinatorError::Unauthorized(
+                "task cancellation target node is not connected to the virtual process".to_owned(),
+            )
+            .into());
+        }
+        self.task_cancellations
+            .insert(task_control_key(&tenant, &project, &process, &node, &task));
+        Ok(CoordinatorResponse::TaskCancellationRequested {
+            process,
+            task,
+            node,
+        })
+    }
+
+    pub(super) fn handle_cancel_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let _actor_user = actor_user;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "process cancellation requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+        self.process_cancellations
+            .insert(process_control_key(&tenant, &project, &process));
+        self.main_runtime.interrupt_process(
+            &tenant,
+            &project,
+            &process,
+            "virtual process cancellation requested",
+        );
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        let mut cancelled_tasks = Vec::new();
+        let mut affected_nodes = BTreeSet::new();
+        for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
+            if task_tenant == &tenant && task_project == &project && task_process == &process {
+                self.task_cancellations
+                    .insert(task_control_key(&tenant, &project, &process, node, task));
+                affected_nodes.insert(node.clone());
+                cancelled_tasks.push(TaskCancellationTarget {
+                    process: process.clone(),
+                    task: task.clone(),
+                    node: node.clone(),
+                });
+            }
+        }
+        let process_key = process_control_key(&tenant, &project, &process);
+        if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
+            self.coordinator
+                .abort_process(&tenant, &project, &process)?;
+            self.clear_operator_panel_state(&tenant, &project, &process);
+        }
+        Ok(CoordinatorResponse::ProcessCancellationRequested {
+            process,
+            cancelled_tasks,
+            affected_nodes: affected_nodes.into_iter().collect(),
+        })
+    }
+
+    pub(super) fn handle_abort_process(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let _actor_user = actor_user;
+        let active = self
+            .coordinator
+            .active_process(&tenant, &project, &process)
+            .ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "process abort requires an active virtual process".to_owned(),
+                )
+            })?;
+        debug_assert_eq!(active.tenant, tenant);
+        debug_assert_eq!(active.project, project);
+
+        let process_key = process_control_key(&tenant, &project, &process);
+        self.process_cancellations.remove(&process_key);
+        self.task_cancellations
+            .retain(|(task_tenant, task_project, task_process, _, _)| {
+                task_tenant != &tenant || task_project != &project || task_process != &process
+            });
+        self.process_aborts.insert(process_key);
+        self.main_runtime
+            .interrupt_process(&tenant, &project, &process, "virtual process aborted");
+        self.main_runtime
+            .controls
+            .remove(&process_control_key(&tenant, &project, &process));
+        self.clear_debug_state_for_process(&tenant, &project, &process);
+        self.clear_operator_panel_state(&tenant, &project, &process);
+        self.pending_task_launches.retain(|pending| {
+            pending.tenant != tenant || pending.project != project || pending.process != process
+        });
+        self.task_assignments.retain(|_, assignments| {
+            assignments.retain(|assignment| {
+                assignment.tenant != tenant
+                    || assignment.project != project
+                    || assignment.process != process
+            });
+            !assignments.is_empty()
+        });
+        let mut aborted_tasks = Vec::new();
+        let mut affected_nodes = BTreeSet::new();
+        for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
+            if task_tenant == &tenant && task_project == &project && task_process == &process {
+                self.task_aborts
+                    .insert(task_control_key(&tenant, &project, &process, node, task));
+                affected_nodes.insert(node.clone());
+                aborted_tasks.push(TaskCancellationTarget {
+                    process: process.clone(),
+                    task: task.clone(),
+                    node: node.clone(),
+                });
+            }
+        }
+
+        self.coordinator
+            .abort_process(&tenant, &project, &process)?;
+        let active_restart_tasks = aborted_tasks
+            .iter()
+            .map(|target| target.task.clone())
+            .collect::<BTreeSet<_>>();
+        self.task_restart_checkpoints.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task), _| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+                    || active_restart_tasks.contains(checkpoint_task)
+            },
+        );
+        self.task_restart_checkpoint_order.retain(
+            |(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task)| {
+                checkpoint_tenant != &tenant
+                    || checkpoint_project != &project
+                    || checkpoint_process != &process
+                    || active_restart_tasks.contains(checkpoint_task)
+            },
+        );
+        if aborted_tasks.is_empty() {
+            self.process_aborts
+                .remove(&process_control_key(&tenant, &project, &process));
+        }
+        Ok(CoordinatorResponse::ProcessAborted {
+            process,
+            aborted_tasks,
+            affected_nodes: affected_nodes.into_iter().collect(),
+        })
+    }
+
+    pub(super) fn handle_list_processes(
+        &mut self,
+        tenant: String,
+        project: String,
+        actor_user: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let actor = UserId::new(actor_user);
+        let processes = self
+            .coordinator
+            .active_processes_for_project(&tenant, &project)
+            .into_iter()
+            .map(|active| {
+                let process_key = process_control_key(&active.tenant, &active.project, &active.id);
+                let main = self.main_runtime.controls.get(&process_key);
+                let state = if self.process_cancellations.contains(&process_key) {
+                    "cancelling"
+                } else {
+                    main.map_or("running", |main| main.state.as_str())
+                };
+                let main_wait_state = main.and_then(|main| {
+                    if main.state != "running" {
+                        return None;
+                    }
+                    if self.pending_task_launches.iter().any(|pending| {
+                        pending.tenant == active.tenant
+                            && pending.project == active.project
+                            && pending.process == active.id
+                    }) {
+                        Some("waiting_for_node".to_owned())
+                    } else if self.main_runtime.is_waiting_for_task(
+                        &active.tenant,
+                        &active.project,
+                        &active.id,
+                    ) {
+                        Some("waiting_for_task".to_owned())
+                    } else {
+                        Some("executing".to_owned())
+                    }
+                });
+                VirtualProcessStatus {
+                    process: active.id,
+                    state: state.to_owned(),
+                    main_task_definition: main.map(|main| main.task_definition.clone()),
+                    main_task_instance: main.map(|main| main.task_instance.clone()),
+                    main_state: main.map(|main| main.state.clone()),
+                    main_wait_state,
+                    main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
+                    connected_nodes: active.connected_nodes.into_iter().collect(),
+                    coordinator_epoch: active.coordinator_epoch,
+                }
+            })
+            .collect();
+        Ok(CoordinatorResponse::ProcessStatuses { processes, actor })
+    }
+
+    pub(super) fn handle_poll_task_control(
+        &mut self,
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let tenant = TenantId::new(tenant);
+        let project = ProjectId::new(project);
+        let process = ProcessId::new(process);
+        let node = NodeId::new(node);
+        let task = TaskInstanceId::new(task);
+        self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
+        let cancel_requested = self
+            .task_cancellations
+            .contains(&task_control_key(&tenant, &project, &process, &node, &task))
+            || self
+                .process_cancellations
+                .contains(&process_control_key(&tenant, &project, &process));
+        let abort_requested = self
+            .task_aborts
+            .contains(&task_control_key(&tenant, &project, &process, &node, &task))
+            || self
+                .process_aborts
+                .contains(&process_control_key(&tenant, &project, &process));
+        Ok(CoordinatorResponse::TaskControl {
+            process,
+            task,
+            cancel_requested,
+            abort_requested,
+        })
+    }
+
+    pub(super) fn workflow_actor(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        actor_user: Option<String>,
+        actor_agent: Option<String>,
+        agent_public_key_fingerprint: Option<Digest>,
+        agent_signature: Option<AgentSignedRequest>,
+        request_payload_digest: Option<&Digest>,
+        request_kind: &str,
+        process: &ProcessId,
+        task: Option<&TaskInstanceId>,
+    ) -> Result<WorkflowActor, CoordinatorServiceError> {
+        if let Some(agent) = actor_agent {
+            let agent = AgentId::new(agent);
+            let signature = agent_signature.ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent workflow dispatch requires a signed request proving private-key possession"
+                        .to_owned(),
+                )
+            })?;
+            let request_payload_digest = request_payload_digest.ok_or_else(|| {
+                CoordinatorError::Unauthorized(
+                    "agent workflow dispatch requires a canonical signed request payload"
+                        .to_owned(),
+                )
+            })?;
+            if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request nonce is missing or invalid".to_owned(),
+                )
+                .into());
+            }
+            let now_epoch_seconds = unix_timestamp_seconds();
+            let replay_key = (
+                tenant.clone(),
+                project.clone(),
+                agent.clone(),
+                signature.nonce.clone(),
+            );
+            const AGENT_SIGNATURE_WINDOW_SECONDS: u64 = 300;
+            self.agent_replay_nonces.retain(|_, issued_at| {
+                now_epoch_seconds <= issued_at.saturating_add(AGENT_SIGNATURE_WINDOW_SECONDS)
+            });
+            if self.agent_replay_nonces.contains_key(&replay_key) {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request nonce has already been used".to_owned(),
+                )
+                .into());
+            }
+            let canonical_scope = clusterflux_core::AgentWorkflowRequestScope::new(
+                tenant.clone(),
+                project.clone(),
+                request_kind,
+                process.clone(),
+                task.cloned(),
+            )
+            .map_err(CoordinatorError::Unauthorized)?;
+            let record = self.coordinator.authorize_agent_project_run(
+                canonical_scope.for_agent(&agent),
+                agent_public_key_fingerprint.as_ref(),
+                request_payload_digest,
+                &signature,
+                now_epoch_seconds,
+            )?;
+            if self
+                .agent_replay_nonces
+                .keys()
+                .filter(|(retained_tenant, retained_project, retained_agent, _)| {
+                    retained_tenant == tenant
+                        && retained_project == project
+                        && retained_agent == &agent
+                })
+                .count()
+                >= super::MAX_REPLAY_NONCES_PER_AUTHORITY
+            {
+                return Err(CoordinatorError::Unauthorized(
+                    "agent signed request replay window is full; retry after the bounded signature window advances"
+                        .to_owned(),
+                )
+                .into());
+            }
+            self.agent_replay_nonces
+                .insert(replay_key, signature.issued_at_epoch_seconds);
+            return Ok(WorkflowActor {
+                kind: "agent".to_owned(),
+                user: Some(record.user),
+                agent: Some(agent),
+                credential_kind: CredentialKind::PublicKey,
+                public_key_fingerprint: Some(record.public_key_fingerprint),
+                authenticated_without_browser: true,
+                scopes: record.scopes,
+            });
+        }
+
+        let actor = UserId::new(actor_user.unwrap_or_else(|| "user".to_owned()));
+        Ok(WorkflowActor {
+            kind: "user".to_owned(),
+            user: Some(actor),
+            agent: None,
+            credential_kind: CredentialKind::BrowserSession,
+            public_key_fingerprint: None,
+            authenticated_without_browser: false,
+            scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
+        })
+    }
+}
+
+fn unix_timestamp_seconds() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}

crates/clusterflux-coordinator/src/service/protocol.rs

@@ -0,0 +1,673 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{
+    AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind,
+    DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements,
+    LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest,
+    PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation,
+    SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId,
+    UserId, VfsPath,
+};
+use serde::{Deserialize, Serialize};
+
+mod responses;
+pub use responses::*;
+
+use crate::{AgentPublicKeyRecord, ProjectRecord};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TaskReplacementBundle {
+    pub bundle_digest: Digest,
+    pub wasm_module_base64: String,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskFailureResolution {
+    AcceptFailure,
+    Cancel,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum CoordinatorRequest {
+    Ping,
+    Authenticated {
+        session_secret: String,
+        request: AuthenticatedCoordinatorRequest,
+    },
+    AuthStatus {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    AdminStatus {
+        tenant: String,
+        actor_user: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    },
+    SuspendTenant {
+        tenant: String,
+        actor_user: String,
+        target_tenant: String,
+        admin_proof: Digest,
+        admin_nonce: String,
+        issued_at_epoch_seconds: u64,
+    },
+    CreateProject {
+        tenant: String,
+        actor_user: String,
+        project: String,
+        name: String,
+    },
+    SelectProject {
+        tenant: String,
+        actor_user: String,
+        project: String,
+    },
+    ListProjects {
+        tenant: String,
+        actor_user: String,
+    },
+    RegisterAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+        public_key: String,
+    },
+    ListAgentPublicKeys {
+        tenant: String,
+        project: String,
+        user: String,
+    },
+    RotateAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        tenant: String,
+        project: String,
+        user: String,
+        agent: String,
+    },
+    AttachNode {
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+    },
+    CreateNodeEnrollmentGrant {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        #[serde(default = "default_node_enrollment_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    ExchangeNodeEnrollmentGrant {
+        tenant: String,
+        project: String,
+        node: String,
+        public_key: String,
+        enrollment_grant: String,
+    },
+    NodeHeartbeat {
+        node: String,
+        #[serde(default)]
+        node_signature: Option<NodeSignedRequest>,
+    },
+    SignedNode {
+        node: String,
+        node_signature: NodeSignedRequest,
+        request: Box<CoordinatorRequest>,
+    },
+    ReportNodeCapabilities {
+        tenant: String,
+        project: String,
+        node: String,
+        capabilities: NodeCapabilities,
+        cached_environment_digests: Vec<Digest>,
+        #[serde(default)]
+        dependency_cache_digests: Vec<Digest>,
+        source_snapshots: Vec<Digest>,
+        artifact_locations: Vec<String>,
+        direct_connectivity: bool,
+        online: bool,
+    },
+    ListNodeDescriptors {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    RevokeNodeCredential {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        node: String,
+    },
+    ScheduleTask {
+        tenant: String,
+        project: String,
+        environment: Option<EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<Capability>,
+        #[serde(default)]
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    },
+    LaunchTask {
+        tenant: String,
+        project: String,
+        #[serde(default)]
+        actor_user: Option<String>,
+        #[serde(default)]
+        actor_agent: Option<String>,
+        #[serde(default)]
+        agent_public_key_fingerprint: Option<Digest>,
+        #[serde(default)]
+        agent_signature: Option<AgentSignedRequest>,
+        task_spec: TaskSpec,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    LaunchChildTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task_spec: TaskSpec,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    JoinChildTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        parent_task: String,
+        task: String,
+    },
+    PollTaskAssignment {
+        tenant: String,
+        project: String,
+        node: String,
+    },
+    PollArtifactTransfer {
+        tenant: String,
+        project: String,
+        node: String,
+    },
+    UploadArtifactTransferChunk {
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        offset: u64,
+        content_base64: String,
+        chunk_digest: Digest,
+        eof: bool,
+    },
+    FailArtifactTransfer {
+        tenant: String,
+        project: String,
+        node: String,
+        transfer_id: String,
+        artifact: String,
+        message: String,
+    },
+    RequestRendezvous {
+        scope: DataPlaneScope,
+        source: NodeEndpoint,
+        destination: NodeEndpoint,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+    RequestSourcePreparation {
+        tenant: String,
+        project: String,
+        provider: SourceProviderKind,
+    },
+    CompleteSourcePreparation {
+        tenant: String,
+        project: String,
+        node: String,
+        provider: SourceProviderKind,
+        source_snapshot: Digest,
+    },
+    StartProcess {
+        tenant: String,
+        project: String,
+        #[serde(default)]
+        actor_user: Option<String>,
+        #[serde(default)]
+        actor_agent: Option<String>,
+        #[serde(default)]
+        agent_public_key_fingerprint: Option<Digest>,
+        #[serde(default)]
+        agent_signature: Option<AgentSignedRequest>,
+        process: String,
+        #[serde(default)]
+        restart: bool,
+    },
+    ReconnectNode {
+        node: String,
+        process: String,
+        epoch: u64,
+    },
+    CancelTask {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    CancelProcess {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    AbortProcess {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    ListProcesses {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    QuotaStatus {
+        tenant: String,
+        project: String,
+        actor_user: String,
+    },
+    PollTaskControl {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    RestartTask {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        #[serde(default)]
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    SetDebugBreakpoints {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        probe_symbols: Vec<String>,
+    },
+    InspectDebugBreakpoints {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    CreateDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        stopped_task: String,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        epoch: u64,
+    },
+    PollDebugCommand {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+    },
+    ReportDebugState {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        epoch: u64,
+        state: DebugAcknowledgementState,
+        #[serde(default)]
+        stack_frames: Vec<String>,
+        #[serde(default)]
+        local_values: Vec<(String, String)>,
+        #[serde(default)]
+        task_args: Vec<(String, String)>,
+        #[serde(default)]
+        handles: Vec<(String, String)>,
+        #[serde(default)]
+        command_status: Option<String>,
+        #[serde(default)]
+        recent_output: Vec<String>,
+        #[serde(default)]
+        message: Option<String>,
+    },
+    ReportDebugProbeHit {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        probe_symbol: String,
+    },
+    ReportTaskLog {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        #[serde(default)]
+        stdout_tail: String,
+        #[serde(default)]
+        stderr_tail: String,
+        stdout_truncated: bool,
+        stderr_truncated: bool,
+        backpressured: bool,
+    },
+    ReportVfsMetadata {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        large_bytes_uploaded: bool,
+    },
+    TaskCompleted {
+        tenant: String,
+        project: String,
+        process: String,
+        node: String,
+        task: String,
+        #[serde(default)]
+        terminal_state: Option<TaskTerminalState>,
+        status_code: Option<i32>,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        #[serde(default)]
+        stdout_tail: String,
+        #[serde(default)]
+        stderr_tail: String,
+        #[serde(default)]
+        stdout_truncated: bool,
+        #[serde(default)]
+        stderr_truncated: bool,
+        artifact_path: Option<String>,
+        artifact_digest: Option<Digest>,
+        artifact_size_bytes: Option<u64>,
+        #[serde(default)]
+        result: Option<TaskBoundaryValue>,
+    },
+    ListTaskEvents {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        #[serde(default)]
+        process: Option<String>,
+    },
+    ListTaskSnapshots {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+    },
+    JoinTask {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        process: String,
+        task: String,
+    },
+    RenderOperatorPanel {
+        tenant: String,
+        project: String,
+        process: String,
+        actor_user: String,
+        max_download_bytes: u64,
+        stopped: bool,
+    },
+    SubmitPanelEvent {
+        tenant: String,
+        project: String,
+        process: String,
+        widget_id: String,
+        kind: PanelEventKind,
+        max_events: u64,
+    },
+    CreateArtifactDownloadLink {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        #[serde(default = "default_download_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        token_digest: Digest,
+    },
+    ExportArtifactToNode {
+        tenant: String,
+        project: String,
+        actor_user: String,
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+}
+
+impl CoordinatorRequest {
+    pub fn operation(&self) -> Result<String, String> {
+        serde_json::to_value(self)
+            .map_err(|err| format!("failed to encode coordinator request operation: {err}"))
+            .map(|value| clusterflux_core::coordinator_payload_operation(&value))
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum AuthenticatedCoordinatorRequest {
+    AuthStatus,
+    RevokeCliSession,
+    CreateProject {
+        project: String,
+        name: String,
+    },
+    SelectProject {
+        project: String,
+    },
+    ListProjects,
+    RegisterAgentPublicKey {
+        agent: String,
+        public_key: String,
+    },
+    ListAgentPublicKeys,
+    RotateAgentPublicKey {
+        agent: String,
+        public_key: String,
+    },
+    RevokeAgentPublicKey {
+        agent: String,
+    },
+    CreateNodeEnrollmentGrant {
+        #[serde(default = "default_node_enrollment_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    ListNodeDescriptors,
+    RevokeNodeCredential {
+        node: String,
+    },
+    StartProcess {
+        process: String,
+        #[serde(default)]
+        restart: bool,
+    },
+    ScheduleTask {
+        environment: Option<EnvironmentRequirements>,
+        environment_digest: Option<Digest>,
+        required_capabilities: Vec<Capability>,
+        #[serde(default)]
+        dependency_cache: Option<Digest>,
+        source_snapshot: Option<Digest>,
+        required_artifacts: Vec<String>,
+        prefer_node: Option<String>,
+    },
+    LaunchTask {
+        task_spec: Box<TaskSpec>,
+        #[serde(default)]
+        wait_for_node: bool,
+        artifact_path: String,
+        wasm_module_base64: String,
+    },
+    CancelProcess {
+        process: String,
+    },
+    AbortProcess {
+        process: String,
+    },
+    ListProcesses,
+    QuotaStatus,
+    RestartTask {
+        process: String,
+        task: String,
+        #[serde(default)]
+        replacement_bundle: Option<TaskReplacementBundle>,
+    },
+    ResolveTaskFailure {
+        process: String,
+        task: String,
+        resolution: TaskFailureResolution,
+    },
+    DebugAttach {
+        process: String,
+    },
+    SetDebugBreakpoints {
+        process: String,
+        probe_symbols: Vec<String>,
+    },
+    InspectDebugBreakpoints {
+        process: String,
+    },
+    CreateDebugEpoch {
+        process: String,
+        stopped_task: String,
+        reason: String,
+    },
+    ResumeDebugEpoch {
+        process: String,
+        epoch: u64,
+    },
+    InspectDebugEpoch {
+        process: String,
+        epoch: u64,
+    },
+    ListTaskEvents {
+        #[serde(default)]
+        process: Option<String>,
+    },
+    ListTaskSnapshots {
+        process: String,
+    },
+    JoinTask {
+        process: String,
+        task: String,
+    },
+    CreateArtifactDownloadLink {
+        artifact: String,
+        max_bytes: u64,
+        #[serde(default = "default_download_ttl_seconds")]
+        ttl_seconds: u64,
+    },
+    OpenArtifactDownloadStream {
+        artifact: String,
+        max_bytes: u64,
+        token_digest: Digest,
+        chunk_bytes: u64,
+    },
+    RevokeArtifactDownloadLink {
+        artifact: String,
+        token_digest: Digest,
+    },
+    ExportArtifactToNode {
+        artifact: String,
+        receiver_node: String,
+        direct_connectivity: bool,
+        failure_reason: String,
+    },
+}
+
+fn default_download_ttl_seconds() -> u64 {
+    900
+}
+
+fn default_node_enrollment_ttl_seconds() -> u64 {
+    900
+}

crates/clusterflux-coordinator/src/service/protocol/responses.rs

@@ -0,0 +1,545 @@
+use super::*;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskTerminalState {
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+impl TaskTerminalState {
+    pub(crate) fn from_status_code(status_code: Option<i32>) -> Self {
+        match status_code {
+            Some(0) => Self::Completed,
+            _ => Self::Failed,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskExecutor {
+    CoordinatorMain,
+    Node,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCompletionEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub node: NodeId,
+    pub executor: TaskExecutor,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub task: TaskInstanceId,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub attempt_id: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub placement: Option<Placement>,
+    pub terminal_state: TaskTerminalState,
+    pub status_code: Option<i32>,
+    pub stdout_bytes: u64,
+    pub stderr_bytes: u64,
+    pub stdout_tail: String,
+    pub stderr_tail: String,
+    pub stdout_truncated: bool,
+    pub stderr_truncated: bool,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub result: Option<TaskBoundaryValue>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TaskAttemptState {
+    Queued,
+    Running,
+    FailedAwaitingAction,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAttemptSnapshot {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub attempt_id: String,
+    pub attempt_number: u32,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub display_name: String,
+    pub state: TaskAttemptState,
+    pub current: bool,
+    pub node: Option<NodeId>,
+    pub environment_id: Option<String>,
+    pub environment_digest: Option<Digest>,
+    pub argument_summary: Vec<String>,
+    pub handle_summary: Vec<String>,
+    pub command_state: Option<String>,
+    pub vfs_checkpoint: String,
+    pub probe_symbol: Option<String>,
+    pub source_path: Option<String>,
+    pub source_line: Option<u32>,
+    pub restart_compatible: bool,
+    pub failure_policy: clusterflux_core::TaskFailurePolicy,
+    pub artifact_path: Option<VfsPath>,
+    pub artifact_digest: Option<Digest>,
+    pub artifact_size_bytes: Option<u64>,
+    pub status_code: Option<i32>,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugAuditEvent {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: Option<TaskInstanceId>,
+    pub actor: UserId,
+    pub operation: String,
+    pub allowed: bool,
+    pub reason: String,
+    pub charged_debug_read_bytes: u64,
+    pub used_debug_read_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WorkflowActor {
+    pub kind: String,
+    pub user: Option<UserId>,
+    pub agent: Option<AgentId>,
+    pub credential_kind: CredentialKind,
+    pub public_key_fingerprint: Option<Digest>,
+    pub authenticated_without_browser: bool,
+    pub scopes: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskAssignment {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+    pub epoch: u64,
+    pub artifact_path: String,
+    pub task_spec: TaskSpec,
+    pub wasm_module_base64: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactTransferAssignment {
+    pub transfer_id: String,
+    pub artifact: ArtifactId,
+    pub expected_digest: Digest,
+    pub expected_size_bytes: u64,
+    pub offset: u64,
+    pub max_chunk_bytes: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TaskCancellationTarget {
+    pub process: ProcessId,
+    pub task: TaskInstanceId,
+    pub node: NodeId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DebugAcknowledgementState {
+    Frozen,
+    Running,
+    Failed,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DebugParticipantAcknowledgement {
+    pub node: NodeId,
+    pub task_definition: clusterflux_core::TaskDefinitionId,
+    pub task: TaskInstanceId,
+    pub epoch: u64,
+    pub state: DebugAcknowledgementState,
+    pub stack_frames: Vec<String>,
+    pub local_values: Vec<(String, String)>,
+    pub task_args: Vec<(String, String)>,
+    pub handles: Vec<(String, String)>,
+    pub command_status: Option<String>,
+    pub recent_output: Vec<String>,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct VirtualProcessStatus {
+    pub process: ProcessId,
+    pub state: String,
+    pub main_task_definition: Option<clusterflux_core::TaskDefinitionId>,
+    pub main_task_instance: Option<TaskInstanceId>,
+    pub main_state: Option<String>,
+    pub main_wait_state: Option<String>,
+    pub main_debug_epoch: Option<u64>,
+    pub connected_nodes: Vec<NodeId>,
+    pub coordinator_epoch: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SourcePreparationDisposition {
+    Pending { reason: String },
+    Assigned { node: NodeId },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SourcePreparationStatus {
+    pub preparation: SourcePreparation,
+    pub disposition: SourcePreparationDisposition,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum CoordinatorResponse {
+    Pong {
+        epoch: u64,
+    },
+    AuthStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        authenticated: bool,
+        account_status: String,
+        suspended: bool,
+        disabled: bool,
+        deleted: bool,
+        manual_review: bool,
+        sanitized_reason: Option<String>,
+        next_actions: Vec<String>,
+        private_moderation_details_exposed: bool,
+        signup_failure_details_exposed: bool,
+    },
+    AdminStatus {
+        tenant: TenantId,
+        actor: UserId,
+        suspended: bool,
+        safe_default: String,
+    },
+    TenantSuspended {
+        tenant: TenantId,
+        actor: UserId,
+        policy: crate::ServicePolicyRecord,
+    },
+    ProjectCreated {
+        project: ProjectRecord,
+        actor: UserId,
+    },
+    ProjectSelected {
+        project: ProjectRecord,
+        actor: UserId,
+    },
+    Projects {
+        projects: Vec<ProjectRecord>,
+        actor: UserId,
+    },
+    CliSessionRevoked {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+    },
+    AgentPublicKey {
+        record: AgentPublicKeyRecord,
+        actor: UserId,
+    },
+    AgentPublicKeys {
+        records: Vec<AgentPublicKeyRecord>,
+        actor: UserId,
+    },
+    NodeAttached {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+    },
+    NodeEnrollmentGrantCreated {
+        tenant: TenantId,
+        project: ProjectId,
+        grant: String,
+        scope: String,
+        expires_at_epoch_seconds: u64,
+    },
+    NodeEnrollmentExchanged {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        credential: clusterflux_core::NodeCredential,
+    },
+    NodeHeartbeat {
+        node: NodeId,
+        epoch: u64,
+    },
+    NodeCapabilitiesRecorded {
+        node: NodeId,
+        node_descriptors: usize,
+    },
+    NodeDescriptors {
+        descriptors: Vec<NodeDescriptor>,
+        actor: UserId,
+    },
+    NodeCredentialRevoked {
+        node: NodeId,
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        descriptor_removed: bool,
+        queued_assignments_removed: usize,
+    },
+    TaskPlacement {
+        placement: Placement,
+    },
+    TaskLaunched {
+        process: ProcessId,
+        task: TaskInstanceId,
+        actor: WorkflowActor,
+        placement: Placement,
+        assignment: Box<TaskAssignment>,
+        charged_spawns: u64,
+    },
+    MainLaunched {
+        process: ProcessId,
+        task_definition: clusterflux_core::TaskDefinitionId,
+        task_instance: TaskInstanceId,
+        actor: WorkflowActor,
+        state: String,
+    },
+    TaskQueued {
+        process: ProcessId,
+        task: TaskInstanceId,
+        actor: WorkflowActor,
+        reason: String,
+        charged_spawns: u64,
+        queued_tasks: usize,
+    },
+    TaskAssignment {
+        assignment: Option<Box<TaskAssignment>>,
+    },
+    ArtifactTransferAssignment {
+        transfer: Option<ArtifactTransferAssignment>,
+    },
+    ArtifactTransferChunkAccepted {
+        transfer_id: String,
+        next_offset: u64,
+        complete: bool,
+    },
+    ArtifactTransferFailed {
+        transfer_id: String,
+    },
+    RendezvousPlan {
+        plan: DirectBulkTransferPlan,
+        charged_rendezvous_attempts: u64,
+    },
+    SourcePreparation {
+        status: SourcePreparationStatus,
+    },
+    SourcePreparationCompleted {
+        node: NodeId,
+        provider: SourceProviderKind,
+        source_snapshot: Digest,
+    },
+    ProcessStarted {
+        process: ProcessId,
+        epoch: u64,
+        actor: WorkflowActor,
+        charged_spawns: u64,
+    },
+    NodeReconnected {
+        node: NodeId,
+        process: ProcessId,
+    },
+    TaskCancellationRequested {
+        process: ProcessId,
+        task: TaskInstanceId,
+        node: NodeId,
+    },
+    ProcessCancellationRequested {
+        process: ProcessId,
+        cancelled_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessAborted {
+        process: ProcessId,
+        aborted_tasks: Vec<TaskCancellationTarget>,
+        affected_nodes: Vec<NodeId>,
+    },
+    ProcessStatuses {
+        processes: Vec<VirtualProcessStatus>,
+        actor: UserId,
+    },
+    QuotaStatus {
+        tenant: TenantId,
+        project: ProjectId,
+        actor: UserId,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        policy_label: Option<String>,
+        limits: ResourceLimits,
+        window_seconds: BTreeMap<LimitKind, u64>,
+        usage: BTreeMap<LimitKind, u64>,
+        window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+    },
+    TaskControl {
+        process: ProcessId,
+        task: TaskInstanceId,
+        cancel_requested: bool,
+        abort_requested: bool,
+    },
+    TaskRestart {
+        process: ProcessId,
+        task: TaskInstanceId,
+        restarted_task_instance: Option<clusterflux_core::TaskInstanceId>,
+        restarted_attempt_id: Option<String>,
+        actor: UserId,
+        accepted: bool,
+        clean_boundary_available: bool,
+        active_task: bool,
+        completed_event_observed: bool,
+        requires_whole_process_restart: bool,
+        message: String,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugCommand {
+        process: ProcessId,
+        task: TaskInstanceId,
+        epoch: Option<u64>,
+        command: Option<String>,
+    },
+    DebugStateRecorded {
+        process: ProcessId,
+        node: NodeId,
+        task: TaskInstanceId,
+        epoch: u64,
+        state: DebugAcknowledgementState,
+    },
+    DebugAttach {
+        process: ProcessId,
+        actor: UserId,
+        authorization: Authorization,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugBreakpoints {
+        process: ProcessId,
+        actor: UserId,
+        probe_symbols: Vec<String>,
+        hit_epoch: Option<u64>,
+        hit_task: Option<TaskInstanceId>,
+        hit_probe_symbol: Option<String>,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugProbeHit {
+        process: ProcessId,
+        node: NodeId,
+        task: TaskInstanceId,
+        probe_symbol: String,
+        breakpoint_matched: bool,
+        debug_epoch: Option<u64>,
+    },
+    DebugEpoch {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        affected_tasks: Vec<TaskCancellationTarget>,
+        all_stop_requested: bool,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    DebugEpochStatus {
+        process: ProcessId,
+        actor: UserId,
+        epoch: u64,
+        command: String,
+        expected_tasks: Vec<TaskCancellationTarget>,
+        acknowledgements: Vec<DebugParticipantAcknowledgement>,
+        fully_frozen: bool,
+        partially_frozen: bool,
+        fully_resumed: bool,
+        failed: bool,
+        failure_messages: Vec<String>,
+        audit_event: DebugAuditEvent,
+        charged_debug_read_bytes: u64,
+        used_debug_read_bytes: u64,
+    },
+    TaskLogRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        stdout_bytes: u64,
+        stderr_bytes: u64,
+        stdout_tail: String,
+        stderr_tail: String,
+        backpressured: bool,
+    },
+    VfsMetadataRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        artifact_path: Option<VfsPath>,
+        large_bytes_uploaded: bool,
+    },
+    TaskRecorded {
+        process: ProcessId,
+        task: TaskInstanceId,
+        events_recorded: usize,
+    },
+    TaskEvents {
+        events: Vec<TaskCompletionEvent>,
+    },
+    TaskSnapshots {
+        snapshots: Vec<TaskAttemptSnapshot>,
+    },
+    TaskFailureResolved {
+        process: ProcessId,
+        task: TaskInstanceId,
+        attempt_id: String,
+        resolution: TaskFailureResolution,
+    },
+    TaskJoined {
+        join: TaskJoinResult,
+    },
+    OperatorPanel {
+        panel: PanelState,
+    },
+    PanelEventAccepted {
+        used_events: u64,
+        max_events: u64,
+    },
+    ArtifactDownloadLink {
+        link: DownloadLink,
+    },
+    ArtifactDownloadLinkRevoked {
+        link: DownloadLink,
+    },
+    ArtifactDownloadStream {
+        link: DownloadLink,
+        streamed_bytes: u64,
+        charged_download_bytes: u64,
+        content_bytes_available: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_offset: Option<u64>,
+        #[serde(default)]
+        content_eof: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_base64: Option<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        content_source: Option<String>,
+    },
+    ArtifactExportPlan {
+        plan: DirectBulkTransferPlan,
+        source_node: NodeId,
+        receiver_node: NodeId,
+        artifact_size_bytes: u64,
+    },
+    Error {
+        message: String,
+    },
+}

crates/clusterflux-coordinator/src/service/quota.rs

@@ -0,0 +1,484 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{LimitError, LimitKind, ProjectId, ResourceLimits, ResourceMeter, TenantId};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CoordinatorQuotaConfiguration {
+    pub limits: ResourceLimits,
+    pub window_seconds: BTreeMap<LimitKind, u64>,
+    pub policy_label: Option<String>,
+    pub max_projects_per_tenant: usize,
+    pub max_nodes_per_tenant: usize,
+    pub max_active_processes_per_tenant: usize,
+}
+
+pub(super) struct CoordinatorQuotaStatus {
+    pub(super) policy_label: Option<String>,
+    pub(super) limits: ResourceLimits,
+    pub(super) window_seconds: BTreeMap<LimitKind, u64>,
+    pub(super) usage: BTreeMap<LimitKind, u64>,
+    pub(super) window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
+}
+
+impl CoordinatorQuotaConfiguration {
+    pub fn new(
+        limits: ResourceLimits,
+        window_seconds: impl IntoIterator<Item = (LimitKind, u64)>,
+    ) -> Result<Self, String> {
+        let window_seconds = window_seconds.into_iter().collect::<BTreeMap<_, _>>();
+        if window_seconds.values().any(|seconds| *seconds == 0) {
+            return Err("quota windows must be at least one second".to_owned());
+        }
+        Ok(Self {
+            limits,
+            window_seconds,
+            policy_label: None,
+            max_projects_per_tenant: usize::MAX,
+            max_nodes_per_tenant: usize::MAX,
+            max_active_processes_per_tenant: usize::MAX,
+        })
+    }
+
+    pub fn with_policy_label(mut self, label: impl Into<String>) -> Self {
+        let label = label.into();
+        self.policy_label = (!label.trim().is_empty()).then_some(label);
+        self
+    }
+
+    pub fn with_admission_limits(
+        mut self,
+        max_projects_per_tenant: usize,
+        max_nodes_per_tenant: usize,
+        max_active_processes_per_tenant: usize,
+    ) -> Self {
+        self.max_projects_per_tenant = max_projects_per_tenant;
+        self.max_nodes_per_tenant = max_nodes_per_tenant;
+        self.max_active_processes_per_tenant = max_active_processes_per_tenant;
+        self
+    }
+
+    pub fn unlimited() -> Self {
+        Self {
+            limits: ResourceLimits::unlimited(),
+            window_seconds: LimitKind::ALL
+                .into_iter()
+                .map(|kind| (kind, u64::MAX))
+                .collect(),
+            policy_label: None,
+            max_projects_per_tenant: usize::MAX,
+            max_nodes_per_tenant: usize::MAX,
+            max_active_processes_per_tenant: usize::MAX,
+        }
+    }
+
+    pub fn window_seconds(&self, kind: LimitKind) -> u64 {
+        self.window_seconds
+            .get(&kind)
+            .copied()
+            .unwrap_or(u64::MAX)
+            .max(1)
+    }
+}
+
+impl Default for CoordinatorQuotaConfiguration {
+    fn default() -> Self {
+        Self::unlimited()
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct ProjectQuotaScope {
+    tenant: TenantId,
+    project: ProjectId,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct MeterKey {
+    scope: ProjectQuotaScope,
+    kind: LimitKind,
+    window: u64,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct CoordinatorQuota {
+    configuration: CoordinatorQuotaConfiguration,
+    meters: BTreeMap<MeterKey, ResourceMeter>,
+}
+
+impl Default for CoordinatorQuota {
+    fn default() -> Self {
+        Self::new(CoordinatorQuotaConfiguration::default())
+    }
+}
+
+impl CoordinatorQuota {
+    pub(super) fn new(configuration: CoordinatorQuotaConfiguration) -> Self {
+        Self {
+            configuration,
+            meters: BTreeMap::new(),
+        }
+    }
+
+    pub(super) fn ensure_project_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_projects_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.project_limit: tenant {tenant} already has the maximum of {maximum} projects"
+            )));
+        }
+        Ok(())
+    }
+
+    pub(super) fn ensure_node_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_nodes_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.node_limit: tenant {tenant} already has the maximum of {maximum} nodes"
+            )));
+        }
+        Ok(())
+    }
+
+    pub(super) fn ensure_process_admission(
+        &self,
+        tenant: &TenantId,
+        current: usize,
+    ) -> Result<(), super::CoordinatorServiceError> {
+        let maximum = self.configuration.max_active_processes_per_tenant;
+        if current >= maximum {
+            return Err(super::CoordinatorServiceError::Protocol(format!(
+                "admission.active_process_limit: tenant {tenant} already has the maximum of {maximum} active processes"
+            )));
+        }
+        Ok(())
+    }
+
+    fn key(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> MeterKey {
+        MeterKey {
+            scope: ProjectQuotaScope {
+                tenant: tenant.clone(),
+                project: project.clone(),
+            },
+            kind,
+            window: now_epoch_seconds / self.configuration.window_seconds(kind),
+        }
+    }
+
+    fn meter(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> Option<&ResourceMeter> {
+        self.meters
+            .get(&self.key(tenant, project, kind, now_epoch_seconds))
+    }
+
+    fn meter_mut(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> &mut ResourceMeter {
+        let key = self.key(tenant, project, kind, now_epoch_seconds);
+        self.meters.retain(|existing, _| {
+            existing.scope != key.scope || existing.kind != kind || existing.window == key.window
+        });
+        self.meters.entry(key).or_default()
+    }
+
+    fn can_charge(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        amount: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.meter(tenant, project, kind, now_epoch_seconds)
+            .cloned()
+            .unwrap_or_default()
+            .can_charge(&self.configuration.limits, kind, amount)
+    }
+
+    fn charge(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        amount: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        let limits = self.configuration.limits.clone();
+        let meter = self.meter_mut(tenant, project, kind, now_epoch_seconds);
+        meter.charge(&limits, kind, amount)?;
+        Ok(meter.used(&kind))
+    }
+
+    fn used(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        kind: LimitKind,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.meter(tenant, project, kind, now_epoch_seconds)
+            .map_or(0, |meter| meter.used(&kind))
+    }
+
+    pub(super) fn can_charge_workflow_spawn(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
+    }
+
+    pub(super) fn charge_api_call(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
+    }
+
+    pub(super) fn can_charge_log_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(
+            tenant,
+            project,
+            LimitKind::LogBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_log_bytes(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::LogBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_workflow_spawn(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_workflow_spawns(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::Spawn, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_api_calls(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::ApiCall, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn used_log_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(tenant, project, LimitKind::LogBytes, now_epoch_seconds)
+    }
+
+    #[cfg(test)]
+    pub(super) fn active_meter_count(&self) -> usize {
+        self.meters.len()
+    }
+
+    pub(super) fn charge_rendezvous_attempt(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::RendezvousAttempt,
+            1,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn can_charge_download(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), LimitError> {
+        self.can_charge(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_download(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn download_limit(&self) -> u64 {
+        self.configuration
+            .limits
+            .limit(&LimitKind::ArtifactDownloadBytes)
+    }
+
+    pub(super) fn used_download_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(
+            tenant,
+            project,
+            LimitKind::ArtifactDownloadBytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn charge_debug_read(
+        &mut self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<u64, LimitError> {
+        self.charge(
+            tenant,
+            project,
+            LimitKind::DebugReadBytes,
+            bytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn used_debug_read_bytes(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> u64 {
+        self.used(
+            tenant,
+            project,
+            LimitKind::DebugReadBytes,
+            now_epoch_seconds,
+        )
+    }
+
+    pub(super) fn project_status(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+        now_epoch_seconds: u64,
+    ) -> CoordinatorQuotaStatus {
+        let mut usage = BTreeMap::new();
+        let mut window_starts = BTreeMap::new();
+        for kind in LimitKind::ALL {
+            let seconds = self.configuration.window_seconds(kind);
+            usage.insert(kind, self.used(tenant, project, kind, now_epoch_seconds));
+            window_starts.insert(kind, (now_epoch_seconds / seconds).saturating_mul(seconds));
+        }
+        CoordinatorQuotaStatus {
+            policy_label: self.configuration.policy_label.clone(),
+            limits: self.configuration.limits.clone(),
+            window_seconds: self.configuration.window_seconds.clone(),
+            usage,
+            window_started_epoch_seconds: window_starts,
+        }
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_workflow_limits(&mut self, limits: ResourceLimits) {
+        self.configuration
+            .limits
+            .limits
+            .insert(LimitKind::Spawn, limits.limit(&LimitKind::Spawn));
+        self.meters.clear();
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_download_limits(&mut self, limits: ResourceLimits) {
+        self.configuration.limits.limits.insert(
+            LimitKind::ArtifactDownloadBytes,
+            limits.limit(&LimitKind::ArtifactDownloadBytes),
+        );
+        self.meters.clear();
+    }
+
+    #[cfg(test)]
+    pub(super) fn set_rendezvous_limits(&mut self, limits: ResourceLimits) {
+        self.configuration.limits.limits.insert(
+            LimitKind::RendezvousAttempt,
+            limits.limit(&LimitKind::RendezvousAttempt),
+        );
+        self.meters.clear();
+    }
+}

crates/clusterflux-coordinator/src/service/relay.rs

@@ -0,0 +1,590 @@
+use std::collections::BTreeMap;
+
+use clusterflux_core::{ProjectId, TenantId, UserId};
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ArtifactRelayConfiguration {
+    pub enabled: bool,
+    pub max_artifact_bytes: u64,
+    pub max_active_per_project: usize,
+    pub max_active_per_tenant: usize,
+    pub max_active_per_account: usize,
+    pub max_active_global: usize,
+    pub max_period_bytes_per_project: u64,
+    pub max_period_bytes_per_tenant: u64,
+    pub max_period_bytes_per_account: u64,
+    pub period_seconds: u64,
+    pub framing_overhead_bytes: u64,
+    pub max_tracked_scopes: usize,
+}
+
+impl ArtifactRelayConfiguration {
+    pub fn unlimited() -> Self {
+        Self {
+            enabled: true,
+            max_artifact_bytes: u64::MAX,
+            max_active_per_project: usize::MAX,
+            max_active_per_tenant: usize::MAX,
+            max_active_per_account: usize::MAX,
+            max_active_global: usize::MAX,
+            max_period_bytes_per_project: u64::MAX,
+            max_period_bytes_per_tenant: u64::MAX,
+            max_period_bytes_per_account: u64::MAX,
+            period_seconds: u64::MAX,
+            framing_overhead_bytes: 512,
+            max_tracked_scopes: usize::MAX,
+        }
+    }
+
+    pub fn validate(&self) -> Result<(), ArtifactRelayError> {
+        if self.period_seconds == 0
+            || self.max_active_per_project == 0
+            || self.max_active_per_tenant == 0
+            || self.max_active_per_account == 0
+            || self.max_active_global == 0
+            || self.max_tracked_scopes == 0
+        {
+            return Err(ArtifactRelayError::InvalidConfiguration);
+        }
+        Ok(())
+    }
+}
+
+impl Default for ArtifactRelayConfiguration {
+    fn default() -> Self {
+        Self::unlimited()
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct ArtifactRelayUsage {
+    pub active_transfers: usize,
+    pub ingress_bytes: u64,
+    pub egress_bytes: u64,
+    pub abandoned_or_failed_bytes: u64,
+    pub reserved_bytes: u64,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactRelayDurableState {
+    pub reservations: BTreeMap<String, ArtifactRelayReservationState>,
+    pub period: u64,
+    pub project_used: Vec<(TenantId, ProjectId, u64)>,
+    pub tenant_used: Vec<(TenantId, u64)>,
+    pub account_used: Vec<(TenantId, UserId, u64)>,
+    pub ingress_used: u64,
+    pub egress_used: u64,
+    pub abandoned_or_failed_used: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ArtifactRelayReservationState {
+    pub tenant: TenantId,
+    pub project: ProjectId,
+    pub account: UserId,
+    pub remaining_reserved_bytes: u64,
+    pub ingress_bytes: u64,
+    pub egress_bytes: u64,
+    pub expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum RelayFinishReason {
+    Completed,
+    Failed,
+    Cancelled,
+    Expired,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ArtifactRelayError {
+    #[error("artifact relay is disabled by hosted policy")]
+    Disabled,
+    #[error("artifact exceeds the configured relay size limit")]
+    ArtifactTooLarge,
+    #[error("artifact relay concurrency limit reached for {0}")]
+    Concurrency(&'static str),
+    #[error("artifact relay is temporarily at process capacity")]
+    Capacity,
+    #[error("artifact relay period byte budget exhausted for {0}")]
+    PeriodBudget(&'static str),
+    #[error("artifact relay retained scope state limit reached")]
+    StateLimit,
+    #[error("artifact relay reservation is missing or expired")]
+    MissingReservation,
+    #[error("artifact relay configuration contains a zero bound")]
+    InvalidConfiguration,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct RelayScope {
+    tenant: TenantId,
+    project: ProjectId,
+    account: UserId,
+}
+
+#[derive(Clone, Debug)]
+struct RelayReservation {
+    scope: RelayScope,
+    remaining_reserved_bytes: u64,
+    ingress_bytes: u64,
+    egress_bytes: u64,
+    expires_at_epoch_seconds: u64,
+}
+
+#[derive(Clone, Debug)]
+pub(super) struct ArtifactRelayLedger {
+    configuration: ArtifactRelayConfiguration,
+    reservations: BTreeMap<String, RelayReservation>,
+    period: u64,
+    project_used: BTreeMap<(TenantId, ProjectId), u64>,
+    tenant_used: BTreeMap<TenantId, u64>,
+    account_used: BTreeMap<(TenantId, UserId), u64>,
+    ingress_used: u64,
+    egress_used: u64,
+    abandoned_or_failed_used: u64,
+}
+
+impl Default for ArtifactRelayLedger {
+    fn default() -> Self {
+        Self::new(ArtifactRelayConfiguration::default())
+    }
+}
+
+impl ArtifactRelayLedger {
+    pub(super) fn new(configuration: ArtifactRelayConfiguration) -> Self {
+        Self {
+            configuration,
+            reservations: BTreeMap::new(),
+            period: 0,
+            project_used: BTreeMap::new(),
+            tenant_used: BTreeMap::new(),
+            account_used: BTreeMap::new(),
+            ingress_used: 0,
+            egress_used: 0,
+            abandoned_or_failed_used: 0,
+        }
+    }
+
+    pub(super) fn configure(
+        &mut self,
+        configuration: ArtifactRelayConfiguration,
+    ) -> Result<(), ArtifactRelayError> {
+        configuration.validate()?;
+        self.configuration = configuration;
+        Ok(())
+    }
+
+    pub(super) fn from_durable(
+        configuration: ArtifactRelayConfiguration,
+        state: ArtifactRelayDurableState,
+    ) -> Self {
+        Self {
+            configuration,
+            reservations: state
+                .reservations
+                .into_iter()
+                .map(|(key, reservation)| {
+                    (
+                        key,
+                        RelayReservation {
+                            scope: RelayScope {
+                                tenant: reservation.tenant,
+                                project: reservation.project,
+                                account: reservation.account,
+                            },
+                            remaining_reserved_bytes: reservation.remaining_reserved_bytes,
+                            ingress_bytes: reservation.ingress_bytes,
+                            egress_bytes: reservation.egress_bytes,
+                            expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
+                        },
+                    )
+                })
+                .collect(),
+            period: state.period,
+            project_used: state
+                .project_used
+                .into_iter()
+                .map(|(tenant, project, bytes)| ((tenant, project), bytes))
+                .collect(),
+            tenant_used: state.tenant_used.into_iter().collect(),
+            account_used: state
+                .account_used
+                .into_iter()
+                .map(|(tenant, account, bytes)| ((tenant, account), bytes))
+                .collect(),
+            ingress_used: state.ingress_used,
+            egress_used: state.egress_used,
+            abandoned_or_failed_used: state.abandoned_or_failed_used,
+        }
+    }
+
+    pub(super) fn durable_state(&self) -> ArtifactRelayDurableState {
+        ArtifactRelayDurableState {
+            reservations: self
+                .reservations
+                .iter()
+                .map(|(key, reservation)| {
+                    (
+                        key.clone(),
+                        ArtifactRelayReservationState {
+                            tenant: reservation.scope.tenant.clone(),
+                            project: reservation.scope.project.clone(),
+                            account: reservation.scope.account.clone(),
+                            remaining_reserved_bytes: reservation.remaining_reserved_bytes,
+                            ingress_bytes: reservation.ingress_bytes,
+                            egress_bytes: reservation.egress_bytes,
+                            expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
+                        },
+                    )
+                })
+                .collect(),
+            period: self.period,
+            project_used: self
+                .project_used
+                .iter()
+                .map(|((tenant, project), bytes)| (tenant.clone(), project.clone(), *bytes))
+                .collect(),
+            tenant_used: self
+                .tenant_used
+                .iter()
+                .map(|(tenant, bytes)| (tenant.clone(), *bytes))
+                .collect(),
+            account_used: self
+                .account_used
+                .iter()
+                .map(|((tenant, account), bytes)| (tenant.clone(), account.clone(), *bytes))
+                .collect(),
+            ingress_used: self.ingress_used,
+            egress_used: self.egress_used,
+            abandoned_or_failed_used: self.abandoned_or_failed_used,
+        }
+    }
+
+    pub(super) fn reconcile_after_restart(&mut self, now_epoch_seconds: u64) {
+        let reservations = self.reservations.keys().cloned().collect::<Vec<_>>();
+        for key in reservations {
+            self.finish(&key, RelayFinishReason::Failed);
+        }
+        self.prepare_period(now_epoch_seconds);
+    }
+
+    fn prepare_period(&mut self, now_epoch_seconds: u64) {
+        let period = now_epoch_seconds / self.configuration.period_seconds.max(1);
+        if self.period != period {
+            self.period = period;
+            self.project_used.clear();
+            self.tenant_used.clear();
+            self.account_used.clear();
+            self.ingress_used = 0;
+            self.egress_used = 0;
+            self.abandoned_or_failed_used = 0;
+        }
+    }
+
+    pub(super) fn estimated_wire_bytes(&self, artifact_bytes: u64, chunk_bytes: u64) -> u64 {
+        let encoded = artifact_bytes
+            .saturating_add(2)
+            .saturating_div(3)
+            .saturating_mul(4);
+        let chunks = artifact_bytes
+            .saturating_add(chunk_bytes.saturating_sub(1))
+            .saturating_div(chunk_bytes.max(1))
+            .max(1);
+        encoded.saturating_mul(2).saturating_add(
+            chunks
+                .saturating_mul(2)
+                .saturating_mul(self.configuration.framing_overhead_bytes),
+        )
+    }
+
+    pub(super) fn framing_overhead_bytes(&self) -> u64 {
+        self.configuration.framing_overhead_bytes
+    }
+
+    pub(super) fn reserve(
+        &mut self,
+        key: String,
+        tenant: TenantId,
+        project: ProjectId,
+        account: UserId,
+        artifact_bytes: u64,
+        chunk_bytes: u64,
+        expires_at_epoch_seconds: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.expire(now_epoch_seconds);
+        self.prepare_period(now_epoch_seconds);
+        if !self.configuration.enabled {
+            return Err(ArtifactRelayError::Disabled);
+        }
+        if artifact_bytes > self.configuration.max_artifact_bytes {
+            return Err(ArtifactRelayError::ArtifactTooLarge);
+        }
+        let scope = RelayScope {
+            tenant,
+            project,
+            account,
+        };
+        self.check_concurrency(&scope)?;
+        self.check_tracked_scope_capacity(&scope)?;
+        let reserved = self.estimated_wire_bytes(artifact_bytes, chunk_bytes);
+        self.check_budget(&scope, reserved)?;
+        self.reservations.insert(
+            key,
+            RelayReservation {
+                scope,
+                remaining_reserved_bytes: reserved,
+                ingress_bytes: 0,
+                egress_bytes: 0,
+                expires_at_epoch_seconds,
+            },
+        );
+        Ok(())
+    }
+
+    pub(super) fn rekey(&mut self, from: &str, to: String) -> Result<(), ArtifactRelayError> {
+        let reservation = self
+            .reservations
+            .remove(from)
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        self.reservations.insert(to, reservation);
+        Ok(())
+    }
+
+    fn check_concurrency(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
+        if self.reservations.len() >= self.configuration.max_active_global {
+            return Err(ArtifactRelayError::Capacity);
+        }
+        let project = self
+            .reservations
+            .values()
+            .filter(|reservation| {
+                reservation.scope.tenant == scope.tenant
+                    && reservation.scope.project == scope.project
+            })
+            .count();
+        if project >= self.configuration.max_active_per_project {
+            return Err(ArtifactRelayError::Concurrency("project"));
+        }
+        let tenant = self
+            .reservations
+            .values()
+            .filter(|reservation| reservation.scope.tenant == scope.tenant)
+            .count();
+        if tenant >= self.configuration.max_active_per_tenant {
+            return Err(ArtifactRelayError::Concurrency("tenant"));
+        }
+        let account = self
+            .reservations
+            .values()
+            .filter(|reservation| {
+                reservation.scope.tenant == scope.tenant
+                    && reservation.scope.account == scope.account
+            })
+            .count();
+        if account >= self.configuration.max_active_per_account {
+            return Err(ArtifactRelayError::Concurrency("account"));
+        }
+        Ok(())
+    }
+
+    fn check_tracked_scope_capacity(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
+        let project_key = (scope.tenant.clone(), scope.project.clone());
+        let account_key = (scope.tenant.clone(), scope.account.clone());
+        let new_scopes = usize::from(!self.project_used.contains_key(&project_key))
+            + usize::from(!self.tenant_used.contains_key(&scope.tenant))
+            + usize::from(!self.account_used.contains_key(&account_key));
+        let tracked = self.project_used.len() + self.tenant_used.len() + self.account_used.len();
+        if tracked.saturating_add(new_scopes) > self.configuration.max_tracked_scopes {
+            return Err(ArtifactRelayError::StateLimit);
+        }
+        Ok(())
+    }
+
+    fn reserved_for(&self, scope: &RelayScope) -> (u64, u64, u64, u64) {
+        let mut project = 0_u64;
+        let mut tenant = 0_u64;
+        let mut account = 0_u64;
+        let mut global = 0_u64;
+        for reservation in self.reservations.values() {
+            let bytes = reservation.remaining_reserved_bytes;
+            global = global.saturating_add(bytes);
+            if reservation.scope.tenant == scope.tenant {
+                tenant = tenant.saturating_add(bytes);
+                if reservation.scope.project == scope.project {
+                    project = project.saturating_add(bytes);
+                }
+                if reservation.scope.account == scope.account {
+                    account = account.saturating_add(bytes);
+                }
+            }
+        }
+        (project, tenant, account, global)
+    }
+
+    fn check_budget(&self, scope: &RelayScope, additional: u64) -> Result<(), ArtifactRelayError> {
+        let (project_reserved, tenant_reserved, account_reserved, _) = self.reserved_for(scope);
+        let project_used = self
+            .project_used
+            .get(&(scope.tenant.clone(), scope.project.clone()))
+            .copied()
+            .unwrap_or(0);
+        let tenant_used = self.tenant_used.get(&scope.tenant).copied().unwrap_or(0);
+        let account_used = self
+            .account_used
+            .get(&(scope.tenant.clone(), scope.account.clone()))
+            .copied()
+            .unwrap_or(0);
+        for (used, reserved, limit, label) in [
+            (
+                project_used,
+                project_reserved,
+                self.configuration.max_period_bytes_per_project,
+                "project",
+            ),
+            (
+                tenant_used,
+                tenant_reserved,
+                self.configuration.max_period_bytes_per_tenant,
+                "tenant",
+            ),
+            (
+                account_used,
+                account_reserved,
+                self.configuration.max_period_bytes_per_account,
+                "account",
+            ),
+        ] {
+            if used.saturating_add(reserved).saturating_add(additional) > limit {
+                return Err(ArtifactRelayError::PeriodBudget(label));
+            }
+        }
+        Ok(())
+    }
+
+    fn charge(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        ingress: bool,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.prepare_period(now_epoch_seconds);
+        let reservation = self
+            .reservations
+            .get(key)
+            .cloned()
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        let additional = bytes.saturating_sub(reservation.remaining_reserved_bytes);
+        if additional > 0 {
+            self.check_budget(&reservation.scope, additional)?;
+        }
+        let reservation = self
+            .reservations
+            .get_mut(key)
+            .ok_or(ArtifactRelayError::MissingReservation)?;
+        reservation.remaining_reserved_bytes =
+            reservation.remaining_reserved_bytes.saturating_sub(bytes);
+        if ingress {
+            reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
+            self.ingress_used = self.ingress_used.saturating_add(bytes);
+        } else {
+            reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
+            self.egress_used = self.egress_used.saturating_add(bytes);
+        }
+        let project_key = (
+            reservation.scope.tenant.clone(),
+            reservation.scope.project.clone(),
+        );
+        let account_key = (
+            reservation.scope.tenant.clone(),
+            reservation.scope.account.clone(),
+        );
+        *self.project_used.entry(project_key).or_default() = self
+            .project_used
+            .get(&(
+                reservation.scope.tenant.clone(),
+                reservation.scope.project.clone(),
+            ))
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        *self
+            .tenant_used
+            .entry(reservation.scope.tenant.clone())
+            .or_default() = self
+            .tenant_used
+            .get(&reservation.scope.tenant)
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        *self.account_used.entry(account_key).or_default() = self
+            .account_used
+            .get(&(
+                reservation.scope.tenant.clone(),
+                reservation.scope.account.clone(),
+            ))
+            .copied()
+            .unwrap_or(0)
+            .saturating_add(bytes);
+        Ok(())
+    }
+
+    pub(super) fn charge_ingress(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.charge(key, bytes, true, now_epoch_seconds)
+    }
+
+    pub(super) fn charge_egress(
+        &mut self,
+        key: &str,
+        bytes: u64,
+        now_epoch_seconds: u64,
+    ) -> Result<(), ArtifactRelayError> {
+        self.charge(key, bytes, false, now_epoch_seconds)
+    }
+
+    pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
+        if let Some(reservation) = self.reservations.remove(key) {
+            if reason != RelayFinishReason::Completed {
+                self.abandoned_or_failed_used = self
+                    .abandoned_or_failed_used
+                    .saturating_add(reservation.ingress_bytes)
+                    .saturating_add(reservation.egress_bytes);
+            }
+        }
+    }
+
+    pub(super) fn expire(&mut self, now_epoch_seconds: u64) {
+        let expired = self
+            .reservations
+            .iter()
+            .filter(|(_, reservation)| reservation.expires_at_epoch_seconds < now_epoch_seconds)
+            .map(|(key, _)| key.clone())
+            .collect::<Vec<_>>();
+        for key in expired {
+            self.finish(&key, RelayFinishReason::Expired);
+        }
+    }
+
+    pub(super) fn usage(&self) -> ArtifactRelayUsage {
+        ArtifactRelayUsage {
+            active_transfers: self.reservations.len(),
+            ingress_bytes: self.ingress_used,
+            egress_bytes: self.egress_used,
+            abandoned_or_failed_bytes: self.abandoned_or_failed_used,
+            reserved_bytes: self
+                .reservations
+                .values()
+                .map(|reservation| reservation.remaining_reserved_bytes)
+                .fold(0_u64, u64::saturating_add),
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/routing.rs

@@ -0,0 +1,831 @@
+use super::*;
+
+impl CoordinatorService {
+    pub fn handle_request(
+        &mut self,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        self.pump_main_runtime_commands();
+        let request_payload = serde_json::to_value(&request).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "failed to canonicalize coordinator request for authentication: {error}"
+            ))
+        })?;
+        let request_payload_digest =
+            clusterflux_core::signed_request_payload_digest(&request_payload);
+        match request {
+            CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong {
+                epoch: self.coordinator.coordinator_epoch(),
+            }),
+            CoordinatorRequest::Authenticated {
+                session_secret,
+                request,
+            } => self.handle_authenticated_request(session_secret, request),
+            CoordinatorRequest::AuthStatus {
+                tenant,
+                project,
+                actor_user,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(actor_user);
+                let account_state = self.coordinator.account_policy_state(&tenant);
+                Ok(CoordinatorResponse::AuthStatus {
+                    tenant,
+                    project,
+                    actor,
+                    authenticated: true,
+                    account_status: account_state.account_status,
+                    suspended: account_state.suspended,
+                    disabled: account_state.disabled,
+                    deleted: account_state.deleted,
+                    manual_review: account_state.manual_review,
+                    sanitized_reason: account_state.sanitized_reason,
+                    next_actions: account_state.next_actions,
+                    private_moderation_details_exposed: false,
+                    signup_failure_details_exposed: false,
+                })
+            }
+            CoordinatorRequest::AdminStatus {
+                tenant,
+                actor_user,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            } => self.handle_admin_status(
+                tenant,
+                actor_user,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            ),
+            CoordinatorRequest::SuspendTenant {
+                tenant,
+                actor_user,
+                target_tenant,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            } => self.handle_suspend_tenant(
+                tenant,
+                actor_user,
+                target_tenant,
+                admin_proof,
+                admin_nonce,
+                issued_at_epoch_seconds,
+            ),
+            CoordinatorRequest::CreateProject {
+                tenant,
+                actor_user,
+                project,
+                name,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let project = ProjectId::new(project);
+                self.coordinator.ensure_tenant_active(&tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the signed-in tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                if self.coordinator.project(&project).is_none() {
+                    self.quota.ensure_project_admission(
+                        &tenant,
+                        self.coordinator.project_count_for_tenant(&tenant),
+                    )?;
+                }
+                self.coordinator.upsert_tenant(tenant.clone());
+                self.coordinator.upsert_user(
+                    tenant.clone(),
+                    actor.clone(),
+                    CredentialKind::BrowserSession,
+                );
+                self.coordinator
+                    .upsert_project(tenant.clone(), project.clone(), name);
+                self.coordinator.grant_project_debug(
+                    tenant.clone(),
+                    project.clone(),
+                    actor.clone(),
+                );
+                self.persist_durable_state()?;
+                let project = self
+                    .coordinator
+                    .project(&project)
+                    .expect("project was just created")
+                    .clone();
+                Ok(CoordinatorResponse::ProjectCreated { project, actor })
+            }
+            CoordinatorRequest::SelectProject {
+                tenant,
+                actor_user,
+                project,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let project_id = ProjectId::new(project);
+                let project = self
+                    .coordinator
+                    .project(&project_id)
+                    .ok_or_else(|| {
+                        CoordinatorError::Unauthorized(
+                            "project is not visible to the signed-in user".to_owned(),
+                        )
+                    })?
+                    .clone();
+                if project.tenant != tenant {
+                    return Err(CoordinatorError::Unauthorized(
+                        "project is outside the signed-in tenant scope".to_owned(),
+                    )
+                    .into());
+                }
+                self.coordinator
+                    .upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::ProjectSelected { project, actor })
+            }
+            CoordinatorRequest::ListProjects { tenant, actor_user } => {
+                let tenant = TenantId::new(tenant);
+                let actor = UserId::new(actor_user);
+                let context = clusterflux_core::AuthContext {
+                    tenant: tenant.clone(),
+                    project: ProjectId::from("__project_listing__"),
+                    actor: Actor::User(actor.clone()),
+                };
+                self.coordinator
+                    .upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::Projects {
+                    projects: self.coordinator.list_projects(&context),
+                    actor,
+                })
+            }
+            CoordinatorRequest::RegisterAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+                public_key,
+            }
+            | CoordinatorRequest::RotateAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+                public_key,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let agent = AgentId::new(agent);
+                self.coordinator.ensure_tenant_active(&tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the signed-in tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                self.coordinator.upsert_tenant(tenant.clone());
+                self.coordinator.upsert_user(
+                    tenant.clone(),
+                    actor.clone(),
+                    CredentialKind::CliDeviceSession,
+                );
+                self.coordinator
+                    .upsert_project(tenant.clone(), project.clone(), "local");
+                let record = self.coordinator.register_agent_public_key(
+                    tenant,
+                    project,
+                    actor.clone(),
+                    agent,
+                    public_key,
+                );
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::AgentPublicKey { record, actor })
+            }
+            CoordinatorRequest::ListAgentPublicKeys {
+                tenant,
+                project,
+                user,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let context = clusterflux_core::AuthContext {
+                    tenant,
+                    project,
+                    actor: Actor::User(actor.clone()),
+                };
+                Ok(CoordinatorResponse::AgentPublicKeys {
+                    records: self.coordinator.list_agent_public_keys(&context),
+                    actor,
+                })
+            }
+            CoordinatorRequest::RevokeAgentPublicKey {
+                tenant,
+                project,
+                user,
+                agent,
+            } => {
+                let tenant = TenantId::new(tenant);
+                let project = ProjectId::new(project);
+                let actor = UserId::new(user);
+                let agent = AgentId::new(agent);
+                let context = clusterflux_core::AuthContext {
+                    tenant,
+                    project,
+                    actor: Actor::User(actor.clone()),
+                };
+                let record = self.coordinator.revoke_agent_public_key(&context, &agent)?;
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::AgentPublicKey { record, actor })
+            }
+            CoordinatorRequest::AttachNode {
+                tenant,
+                project,
+                node,
+                public_key,
+            } => self.handle_attach_node(tenant, project, node, public_key),
+            CoordinatorRequest::CreateNodeEnrollmentGrant {
+                tenant,
+                project,
+                actor_user,
+                ttl_seconds,
+            } => self.handle_create_node_enrollment_grant(tenant, project, actor_user, ttl_seconds),
+            CoordinatorRequest::ExchangeNodeEnrollmentGrant {
+                tenant,
+                project,
+                node,
+                public_key,
+                enrollment_grant,
+            } => self.handle_exchange_node_enrollment_grant(
+                tenant,
+                project,
+                node,
+                public_key,
+                enrollment_grant,
+            ),
+            CoordinatorRequest::NodeHeartbeat {
+                node,
+                node_signature,
+            } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
+            CoordinatorRequest::SignedNode {
+                node,
+                node_signature,
+                request,
+            } => self.handle_signed_node_request(node, node_signature, *request),
+            CoordinatorRequest::ReportNodeCapabilities { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::ListNodeDescriptors {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_list_node_descriptors(tenant, project, actor_user),
+            CoordinatorRequest::RevokeNodeCredential {
+                tenant,
+                project,
+                actor_user,
+                node,
+            } => self.handle_revoke_node_credential(tenant, project, actor_user, node),
+            CoordinatorRequest::ScheduleTask {
+                tenant,
+                project,
+                environment,
+                environment_digest,
+                required_capabilities,
+                dependency_cache,
+                source_snapshot,
+                required_artifacts,
+                prefer_node,
+            } => self.handle_schedule_task(
+                tenant,
+                project,
+                environment,
+                environment_digest,
+                required_capabilities,
+                dependency_cache,
+                source_snapshot,
+                required_artifacts,
+                prefer_node,
+            ),
+            CoordinatorRequest::LaunchTask {
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            } => self.handle_launch_task(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                Some(&request_payload_digest),
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            ),
+            CoordinatorRequest::LaunchChildTask { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::JoinChildTask { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::PollTaskAssignment { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::RequestRendezvous {
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_request_rendezvous(
+                scope,
+                source,
+                destination,
+                direct_connectivity,
+                failure_reason,
+            ),
+            CoordinatorRequest::RequestSourcePreparation {
+                tenant,
+                project,
+                provider,
+            } => self.handle_request_source_preparation(tenant, project, provider),
+            CoordinatorRequest::CompleteSourcePreparation { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::StartProcess {
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                process,
+                restart,
+            } => self.handle_start_process(
+                tenant,
+                project,
+                actor_user,
+                actor_agent,
+                agent_public_key_fingerprint,
+                agent_signature,
+                Some(&request_payload_digest),
+                process,
+                restart,
+            ),
+            CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::CancelTask {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_cancel_task(tenant, project, process, node, task),
+            CoordinatorRequest::CancelProcess {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_cancel_process(tenant, project, actor_user, process),
+            CoordinatorRequest::AbortProcess {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_abort_process(tenant, project, actor_user, process),
+            CoordinatorRequest::ListProcesses {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_list_processes(tenant, project, actor_user),
+            CoordinatorRequest::QuotaStatus {
+                tenant,
+                project,
+                actor_user,
+            } => self.handle_quota_status(tenant, project, actor_user),
+            CoordinatorRequest::PollTaskControl { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::PollArtifactTransfer { .. }
+            | CoordinatorRequest::UploadArtifactTransferChunk { .. }
+            | CoordinatorRequest::FailArtifactTransfer { .. } => {
+                self.reject_unsigned_node_request()
+            }
+            CoordinatorRequest::RestartTask {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                replacement_bundle,
+            } => self.handle_restart_task(
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                replacement_bundle,
+            ),
+            CoordinatorRequest::ResolveTaskFailure {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+                resolution,
+            } => self.handle_resolve_task_failure(
+                tenant, project, actor_user, process, task, resolution,
+            ),
+            request @ (CoordinatorRequest::DebugAttach { .. }
+            | CoordinatorRequest::SetDebugBreakpoints { .. }
+            | CoordinatorRequest::InspectDebugBreakpoints { .. }
+            | CoordinatorRequest::CreateDebugEpoch { .. }
+            | CoordinatorRequest::ResumeDebugEpoch { .. }
+            | CoordinatorRequest::InspectDebugEpoch { .. }) => self.handle_debug_request(request),
+            CoordinatorRequest::PollDebugCommand { .. }
+            | CoordinatorRequest::ReportDebugState { .. }
+            | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
+            CoordinatorRequest::ListTaskEvents {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_list_task_events(tenant, project, actor_user, process),
+            CoordinatorRequest::ListTaskSnapshots {
+                tenant,
+                project,
+                actor_user,
+                process,
+            } => self.handle_list_task_snapshots(tenant, project, actor_user, process),
+            CoordinatorRequest::JoinTask {
+                tenant,
+                project,
+                actor_user,
+                process,
+                task,
+            } => self.handle_join_task(tenant, project, actor_user, process, task),
+            CoordinatorRequest::RenderOperatorPanel {
+                tenant,
+                project,
+                process,
+                actor_user,
+                max_download_bytes,
+                stopped,
+            } => self.handle_render_operator_panel(
+                tenant,
+                project,
+                actor_user,
+                process,
+                max_download_bytes,
+                stopped,
+            ),
+            CoordinatorRequest::SubmitPanelEvent {
+                tenant,
+                project,
+                process,
+                widget_id,
+                kind,
+                max_events,
+            } => self
+                .handle_submit_panel_event(tenant, project, process, widget_id, kind, max_events),
+            CoordinatorRequest::CreateArtifactDownloadLink {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            } => self.handle_create_artifact_download_link(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            ),
+            CoordinatorRequest::OpenArtifactDownloadStream {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            } => self.handle_open_artifact_download_stream(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            ),
+            CoordinatorRequest::RevokeArtifactDownloadLink {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                token_digest,
+            } => self.handle_revoke_artifact_download_link(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                token_digest,
+            ),
+            CoordinatorRequest::ExportArtifactToNode {
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                receiver_node,
+                direct_connectivity,
+                failure_reason,
+            } => self.handle_export_artifact_to_node(
+                tenant,
+                project,
+                actor_user,
+                artifact,
+                receiver_node,
+                direct_connectivity,
+                failure_reason,
+            ),
+        }
+    }
+
+    fn handle_authenticated_request(
+        &mut self,
+        session_secret: String,
+        request: AuthenticatedCoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let context = if matches!(&request, AuthenticatedCoordinatorRequest::AuthStatus) {
+            self.coordinator
+                .authenticate_cli_session_for_status(&session_secret)?
+        } else {
+            self.coordinator.authenticate_cli_session(&session_secret)?
+        };
+        let authorized = authorize_authenticated_user_operation(&context, &request)?;
+        let now_epoch_seconds = self.current_epoch_seconds()?;
+        self.quota
+            .charge_api_call(&context.tenant, &context.project, now_epoch_seconds)?;
+        let _authorized_operation = authorized.operation;
+        let actor = authorized.actor;
+        match request {
+            AuthenticatedCoordinatorRequest::AuthStatus => {
+                let account_state = self.coordinator.account_policy_state(&context.tenant);
+                Ok(CoordinatorResponse::AuthStatus {
+                    tenant: context.tenant,
+                    project: context.project,
+                    actor,
+                    authenticated: true,
+                    account_status: account_state.account_status,
+                    suspended: account_state.suspended,
+                    disabled: account_state.disabled,
+                    deleted: account_state.deleted,
+                    manual_review: account_state.manual_review,
+                    sanitized_reason: account_state.sanitized_reason,
+                    next_actions: account_state.next_actions,
+                    private_moderation_details_exposed: false,
+                    signup_failure_details_exposed: false,
+                })
+            }
+            AuthenticatedCoordinatorRequest::RevokeCliSession => {
+                self.coordinator.revoke_cli_session(&session_secret)?;
+                self.persist_durable_state()?;
+                Ok(CoordinatorResponse::CliSessionRevoked {
+                    tenant: context.tenant,
+                    project: context.project,
+                    actor,
+                })
+            }
+            AuthenticatedCoordinatorRequest::CreateProject { project, name } => {
+                let project = ProjectId::new(project);
+                self.coordinator.ensure_tenant_active(&context.tenant)?;
+                if let Some(existing) = self.coordinator.project(&project) {
+                    if existing.tenant != context.tenant {
+                        return Err(CoordinatorError::Unauthorized(
+                            "project id is outside the authenticated tenant scope".to_owned(),
+                        )
+                        .into());
+                    }
+                }
+                if self.coordinator.project(&project).is_none() {
+                    self.quota.ensure_project_admission(
+                        &context.tenant,
+                        self.coordinator.project_count_for_tenant(&context.tenant),
+                    )?;
+                }
+                self.coordinator
+                    .upsert_project(context.tenant.clone(), project.clone(), name);
+                self.coordinator.grant_project_debug(
+                    context.tenant.clone(),
+                    project.clone(),
+                    actor.clone(),
+                );
+                self.persist_durable_state()?;
+                let project = self
+                    .coordinator
+                    .project(&project)
+                    .expect("project was just created")
+                    .clone();
+                Ok(CoordinatorResponse::ProjectCreated { project, actor })
+            }
+            AuthenticatedCoordinatorRequest::SelectProject { project } => {
+                let project_id = ProjectId::new(project);
+                let project = self
+                    .coordinator
+                    .project(&project_id)
+                    .ok_or_else(|| {
+                        CoordinatorError::Unauthorized(
+                            "project is not visible to the authenticated user".to_owned(),
+                        )
+                    })?
+                    .clone();
+                if project.tenant != context.tenant {
+                    return Err(CoordinatorError::Unauthorized(
+                        "project is outside the authenticated tenant scope".to_owned(),
+                    )
+                    .into());
+                }
+                Ok(CoordinatorResponse::ProjectSelected { project, actor })
+            }
+            AuthenticatedCoordinatorRequest::ListProjects => Ok(CoordinatorResponse::Projects {
+                projects: self.coordinator.list_projects(&context),
+                actor,
+            }),
+            request @ (AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. }
+            | AuthenticatedCoordinatorRequest::ListAgentPublicKeys
+            | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. }
+            | AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. }) => {
+                self.handle_authenticated_agent_key_request(&context, &actor, request)
+            }
+            AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { ttl_seconds } => self
+                .handle_create_node_enrollment_grant(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    ttl_seconds,
+                ),
+            AuthenticatedCoordinatorRequest::ListNodeDescriptors => self
+                .handle_list_node_descriptors(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                ),
+            AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
+                .handle_revoke_node_credential(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    node,
+                ),
+            AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
+                .handle_start_process(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    Some(actor.as_str().to_owned()),
+                    None,
+                    None,
+                    None,
+                    None,
+                    process,
+                    restart,
+                ),
+            request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
+                self.handle_authenticated_schedule_task(&context, request)
+            }
+            request @ AuthenticatedCoordinatorRequest::LaunchTask { .. } => {
+                self.handle_authenticated_launch_task(&context, &actor, request)
+            }
+            AuthenticatedCoordinatorRequest::CancelProcess { process } => self
+                .handle_cancel_process(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+            ),
+            AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+            ),
+            AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+            ),
+            AuthenticatedCoordinatorRequest::RestartTask {
+                process,
+                task,
+                replacement_bundle,
+            } => self.handle_restart_task(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                replacement_bundle,
+            ),
+            AuthenticatedCoordinatorRequest::ResolveTaskFailure {
+                process,
+                task,
+                resolution,
+            } => self.handle_resolve_task_failure(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+                resolution,
+            ),
+            request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. }
+            | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. }
+            | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. }
+            | AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. }
+            | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. }
+            | AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. }) => self
+                .handle_authenticated_debug_request(
+                    &context.tenant,
+                    &context.project,
+                    &actor,
+                    request,
+                ),
+            AuthenticatedCoordinatorRequest::ListTaskEvents { process } => self
+                .handle_list_task_events(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => self
+                .handle_list_task_snapshots(
+                    context.tenant.as_str().to_owned(),
+                    context.project.as_str().to_owned(),
+                    actor.as_str().to_owned(),
+                    process,
+                ),
+            AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                process,
+                task,
+            ),
+            AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            } => self.handle_create_artifact_download_link(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                max_bytes,
+                ttl_seconds,
+            ),
+            AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            } => self.handle_open_artifact_download_stream(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                max_bytes,
+                token_digest,
+                chunk_bytes,
+            ),
+            AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink {
+                artifact,
+                token_digest,
+            } => self.handle_revoke_artifact_download_link(
+                context.tenant.as_str().to_owned(),
+                context.project.as_str().to_owned(),
+                actor.as_str().to_owned(),
+                artifact,
+                token_digest,
+            ),
+            request @ AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
+                self.handle_authenticated_artifact_export(&context, &actor, request)
+            }
+        }
+    }
+}

crates/clusterflux-coordinator/src/service/signed_nodes.rs

@@ -0,0 +1,365 @@
+use clusterflux_core::{NodeId, NodeSignedRequest};
+
+use crate::CoordinatorError;
+
+use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+impl CoordinatorService {
+    pub(super) fn handle_signed_node_request(
+        &mut self,
+        signed_node: String,
+        node_signature: NodeSignedRequest,
+        request: CoordinatorRequest,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        let request_kind = signed_node_request_kind(&request)?;
+        let request_node = signed_node_request_node(&request)?;
+        let request_payload = serde_json::to_value(&request).map_err(|error| {
+            CoordinatorServiceError::Protocol(format!(
+                "failed to canonicalize signed node request: {error}"
+            ))
+        })?;
+        let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
+        let signed_node = NodeId::new(signed_node);
+        if request_node != signed_node {
+            return Err(CoordinatorError::Unauthorized(
+                "signed node request node does not match the wrapped request node".to_owned(),
+            )
+            .into());
+        }
+        self.authenticate_node_request(
+            &signed_node,
+            Some(node_signature),
+            request_kind,
+            &payload_digest,
+        )?;
+        match request {
+            CoordinatorRequest::ReportNodeCapabilities {
+                tenant,
+                project,
+                node,
+                capabilities,
+                cached_environment_digests,
+                dependency_cache_digests,
+                source_snapshots,
+                artifact_locations,
+                direct_connectivity,
+                online,
+            } => self.handle_report_node_capabilities(
+                tenant,
+                project,
+                node,
+                capabilities,
+                cached_environment_digests,
+                dependency_cache_digests,
+                source_snapshots,
+                artifact_locations,
+                direct_connectivity,
+                online,
+            ),
+            CoordinatorRequest::PollTaskAssignment {
+                tenant,
+                project,
+                node,
+            } => self.handle_poll_task_assignment(tenant, project, node),
+            CoordinatorRequest::PollArtifactTransfer {
+                tenant,
+                project,
+                node,
+            } => self.handle_poll_artifact_transfer(tenant, project, node),
+            CoordinatorRequest::UploadArtifactTransferChunk {
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                offset,
+                content_base64,
+                chunk_digest,
+                eof,
+            } => self.handle_upload_artifact_transfer_chunk(
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                offset,
+                content_base64,
+                chunk_digest,
+                eof,
+            ),
+            CoordinatorRequest::FailArtifactTransfer {
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                message,
+            } => self.handle_fail_artifact_transfer(
+                tenant,
+                project,
+                node,
+                transfer_id,
+                artifact,
+                message,
+            ),
+            CoordinatorRequest::LaunchChildTask {
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            } => self.handle_launch_child_task(
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task_spec,
+                wait_for_node,
+                artifact_path,
+                wasm_module_base64,
+            ),
+            CoordinatorRequest::JoinChildTask {
+                tenant,
+                project,
+                process,
+                node,
+                parent_task,
+                task,
+            } => self.handle_join_child_task(tenant, project, process, node, parent_task, task),
+            CoordinatorRequest::CompleteSourcePreparation {
+                tenant,
+                project,
+                node,
+                provider,
+                source_snapshot,
+            } => self.handle_complete_source_preparation(
+                tenant,
+                project,
+                node,
+                provider,
+                source_snapshot,
+            ),
+            CoordinatorRequest::ReconnectNode {
+                node,
+                process,
+                epoch,
+            } => self.handle_reconnect_node(node, process, epoch),
+            CoordinatorRequest::PollTaskControl {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_poll_task_control(tenant, project, process, node, task),
+            CoordinatorRequest::PollDebugCommand {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+            } => self.handle_poll_debug_command(tenant, project, process, node, task),
+            CoordinatorRequest::ReportDebugState {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                epoch,
+                state,
+                stack_frames,
+                local_values,
+                task_args,
+                handles,
+                command_status,
+                recent_output,
+                message,
+            } => self.handle_report_debug_state(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                epoch,
+                state,
+                stack_frames,
+                local_values,
+                task_args,
+                handles,
+                command_status,
+                recent_output,
+                message,
+            ),
+            CoordinatorRequest::ReportDebugProbeHit {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                probe_symbol,
+            } => self.handle_report_debug_probe_hit(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                probe_symbol,
+            ),
+            CoordinatorRequest::ReportTaskLog {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                backpressured,
+            } => self.handle_report_task_log(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                backpressured,
+            ),
+            CoordinatorRequest::ReportVfsMetadata {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                large_bytes_uploaded,
+            } => self.handle_report_vfs_metadata(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                large_bytes_uploaded,
+            ),
+            CoordinatorRequest::TaskCompleted {
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                terminal_state,
+                status_code,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                result,
+            } => self.handle_task_completed(
+                tenant,
+                project,
+                process,
+                node,
+                task,
+                terminal_state,
+                status_code,
+                stdout_bytes,
+                stderr_bytes,
+                stdout_tail,
+                stderr_tail,
+                stdout_truncated,
+                stderr_truncated,
+                artifact_path,
+                artifact_digest,
+                artifact_size_bytes,
+                result,
+            ),
+            _ => self.reject_unsigned_node_request(),
+        }
+    }
+
+    pub(super) fn reject_unsigned_node_request(
+        &self,
+    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
+        Err(CoordinatorError::Unauthorized(
+            "node-originated request requires signed_node envelope proof".to_owned(),
+        )
+        .into())
+    }
+}
+
+fn signed_node_request_kind(
+    request: &CoordinatorRequest,
+) -> Result<&'static str, CoordinatorServiceError> {
+    match request {
+        CoordinatorRequest::ReportNodeCapabilities { .. } => Ok("report_node_capabilities"),
+        CoordinatorRequest::PollTaskAssignment { .. } => Ok("poll_task_assignment"),
+        CoordinatorRequest::PollArtifactTransfer { .. } => Ok("poll_artifact_transfer"),
+        CoordinatorRequest::UploadArtifactTransferChunk { .. } => {
+            Ok("upload_artifact_transfer_chunk")
+        }
+        CoordinatorRequest::FailArtifactTransfer { .. } => Ok("fail_artifact_transfer"),
+        CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
+        CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
+        CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
+        CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
+        CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
+        CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
+        CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
+        CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
+        CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
+        CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
+        CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
+        _ => Err(CoordinatorError::Unauthorized(
+            "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
+        )
+        .into()),
+    }
+}
+
+fn signed_node_request_node(
+    request: &CoordinatorRequest,
+) -> Result<NodeId, CoordinatorServiceError> {
+    match request {
+        CoordinatorRequest::ReportNodeCapabilities { node, .. }
+        | CoordinatorRequest::PollTaskAssignment { node, .. }
+        | CoordinatorRequest::PollArtifactTransfer { node, .. }
+        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
+        | CoordinatorRequest::FailArtifactTransfer { node, .. }
+        | CoordinatorRequest::LaunchChildTask { node, .. }
+        | CoordinatorRequest::JoinChildTask { node, .. }
+        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
+        | CoordinatorRequest::ReconnectNode { node, .. }
+        | CoordinatorRequest::PollTaskControl { node, .. }
+        | CoordinatorRequest::PollDebugCommand { node, .. }
+        | CoordinatorRequest::ReportDebugState { node, .. }
+        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
+        | CoordinatorRequest::ReportTaskLog { node, .. }
+        | CoordinatorRequest::ReportVfsMetadata { node, .. }
+        | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
+        _ => Err(CoordinatorError::Unauthorized(
+            "signed_node envelope only accepts node-originated coordinator requests".to_owned(),
+        )
+        .into()),
+    }
+}

crates/clusterflux-coordinator/src/service/tcp.rs

@@ -0,0 +1,200 @@
+use std::io::{BufRead, BufReader, Write};
+use std::net::{SocketAddr, TcpListener, TcpStream};
+use std::sync::{Arc, Mutex};
+
+use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ClientAuthorityMode {
+    Strict,
+    LocalTrustedLoopback,
+}
+
+impl CoordinatorService {
+    pub fn serve_tcp(self, listener: TcpListener) -> Result<(), CoordinatorServiceError> {
+        if !listener.local_addr()?.ip().is_loopback() {
+            return Err(CoordinatorServiceError::Protocol(
+                "the native coordinator transport is plaintext and restricted to loopback; expose a remote coordinator only through a secure transport"
+                    .to_owned(),
+            ));
+        }
+        self.serve_tcp_with_authority(listener, ClientAuthorityMode::Strict)
+    }
+
+    pub fn serve_tcp_local_trusted(
+        self,
+        listener: TcpListener,
+    ) -> Result<(), CoordinatorServiceError> {
+        if !listener.local_addr()?.ip().is_loopback() {
+            return Err(CoordinatorServiceError::Protocol(
+                "local trusted request mode is restricted to a loopback listener".to_owned(),
+            ));
+        }
+        self.serve_tcp_with_authority(listener, ClientAuthorityMode::LocalTrustedLoopback)
+    }
+
+    fn serve_tcp_with_authority(
+        self,
+        listener: TcpListener,
+        authority_mode: ClientAuthorityMode,
+    ) -> Result<(), CoordinatorServiceError> {
+        let shared = Arc::new(Mutex::new(self));
+        for stream in listener.incoming() {
+            let stream = stream?;
+            let service = Arc::clone(&shared);
+            std::thread::spawn(move || {
+                if let Err(err) = handle_shared_stream(service, stream, authority_mode) {
+                    eprintln!("coordinator stream failed: {err}");
+                }
+            });
+        }
+        Ok(())
+    }
+
+    pub fn handle_stream(&mut self, stream: TcpStream) -> Result<(), CoordinatorServiceError> {
+        self.handle_stream_with_authority(stream, ClientAuthorityMode::Strict)
+    }
+
+    #[cfg(test)]
+    pub(super) fn handle_stream_local_trusted(
+        &mut self,
+        stream: TcpStream,
+    ) -> Result<(), CoordinatorServiceError> {
+        self.handle_stream_with_authority(stream, ClientAuthorityMode::LocalTrustedLoopback)
+    }
+
+    fn handle_stream_with_authority(
+        &mut self,
+        stream: TcpStream,
+        authority_mode: ClientAuthorityMode,
+    ) -> Result<(), CoordinatorServiceError> {
+        let mut reader = BufReader::new(stream.try_clone()?);
+        let mut writer = stream;
+        loop {
+            let mut line = String::new();
+            if reader.read_line(&mut line)? == 0 {
+                return Ok(());
+            }
+            if line.trim().is_empty() {
+                continue;
+            }
+            let response = match decode_wire_request(&line) {
+                Ok(request) => match authorize_client_request(&request, authority_mode)
+                    .and_then(|()| self.handle_request(request))
+                {
+                    Ok(response) => response,
+                    Err(err) => CoordinatorResponse::Error {
+                        message: err.to_string(),
+                    },
+                },
+                Err(err) => CoordinatorResponse::Error {
+                    message: err.to_string(),
+                },
+            };
+            serde_json::to_writer(&mut writer, &response)?;
+            writer.write_all(b"\n")?;
+            writer.flush()?;
+        }
+    }
+}
+
+fn handle_shared_stream(
+    service: Arc<Mutex<CoordinatorService>>,
+    stream: TcpStream,
+    authority_mode: ClientAuthorityMode,
+) -> Result<(), CoordinatorServiceError> {
+    let mut reader = BufReader::new(stream.try_clone()?);
+    let mut writer = stream;
+    loop {
+        let mut line = String::new();
+        if reader.read_line(&mut line)? == 0 {
+            return Ok(());
+        }
+        if line.trim().is_empty() {
+            continue;
+        }
+        let response = match decode_wire_request(&line) {
+            Ok(request) => match authorize_client_request(&request, authority_mode) {
+                Ok(()) => match service.lock() {
+                    Ok(mut service) => match service.handle_request(request) {
+                        Ok(response) => response,
+                        Err(err) => CoordinatorResponse::Error {
+                            message: err.to_string(),
+                        },
+                    },
+                    Err(_) => CoordinatorResponse::Error {
+                        message: "coordinator service lock poisoned".to_owned(),
+                    },
+                },
+                Err(err) => CoordinatorResponse::Error {
+                    message: err.to_string(),
+                },
+            },
+            Err(err) => CoordinatorResponse::Error {
+                message: err.to_string(),
+            },
+        };
+        serde_json::to_writer(&mut writer, &response)?;
+        writer.write_all(b"\n")?;
+        writer.flush()?;
+    }
+}
+
+pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), CoordinatorServiceError> {
+    let listener = TcpListener::bind(addr)?;
+    let addr = listener.local_addr()?;
+    Ok((listener, addr))
+}
+
+fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
+    serde_json::from_str::<super::CoordinatorWireRequest>(line)?
+        .into_request()
+        .map_err(CoordinatorServiceError::Protocol)
+}
+
+fn authorize_client_request(
+    request: &CoordinatorRequest,
+    authority_mode: ClientAuthorityMode,
+) -> Result<(), CoordinatorServiceError> {
+    if authority_mode == ClientAuthorityMode::LocalTrustedLoopback {
+        return Ok(());
+    }
+    match request {
+        CoordinatorRequest::Ping
+        | CoordinatorRequest::Authenticated { .. }
+        | CoordinatorRequest::ExchangeNodeEnrollmentGrant { .. }
+        | CoordinatorRequest::SignedNode { .. }
+        | CoordinatorRequest::NodeHeartbeat {
+            node_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::StartProcess {
+            actor_agent: Some(_),
+            agent_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::LaunchTask {
+            actor_agent: Some(_),
+            agent_signature: Some(_),
+            ..
+        }
+        | CoordinatorRequest::AdminStatus { .. }
+        | CoordinatorRequest::SuspendTenant { .. } => Ok(()),
+        _ => Err(CoordinatorServiceError::Protocol(
+            "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
+                .to_owned(),
+        )),
+    }
+}
+
+#[cfg(test)]
+mod transport_boundary_tests {
+    use super::*;
+
+    #[test]
+    fn native_plaintext_service_refuses_non_loopback_listener() {
+        let (listener, _) = bind_listener("0.0.0.0:0").unwrap();
+        let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
+        assert!(error.to_string().contains("restricted to loopback"));
+    }
+}

crates/clusterflux-coordinator/src/service/wire_protocol.rs

@@ -0,0 +1,59 @@
+use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use super::CoordinatorRequest;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum CoordinatorWireRequest {
+    Envelope(CoordinatorRequestEnvelope),
+}
+
+impl CoordinatorWireRequest {
+    pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        match self {
+            Self::Envelope(envelope) => envelope.into_request(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct CoordinatorRequestEnvelope {
+    #[serde(rename = "type")]
+    pub envelope_type: String,
+    pub protocol_version: u64,
+    pub request_id: String,
+    pub operation: String,
+    #[serde(default)]
+    pub authentication: Option<Value>,
+    pub payload: CoordinatorRequest,
+}
+
+impl CoordinatorRequestEnvelope {
+    pub fn into_request(self) -> Result<CoordinatorRequest, String> {
+        if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
+            return Err(format!(
+                "unsupported coordinator wire request type {}; expected {}",
+                self.envelope_type, COORDINATOR_WIRE_REQUEST_TYPE
+            ));
+        }
+        if self.protocol_version != COORDINATOR_PROTOCOL_VERSION {
+            return Err(format!(
+                "unsupported coordinator protocol version {}; expected {}",
+                self.protocol_version, COORDINATOR_PROTOCOL_VERSION
+            ));
+        }
+        if self.request_id.trim().is_empty() {
+            return Err("coordinator wire request_id must be non-empty".to_owned());
+        }
+        let payload_operation = self.payload.operation()?;
+        if self.operation != payload_operation {
+            return Err(format!(
+                "coordinator wire operation {} does not match payload operation {}",
+                self.operation, payload_operation
+            ));
+        }
+        Ok(self.payload)
+    }
+}