clusterflux-public/crates/clusterflux-coordinator/src/service/tests.rs

use std::collections::{BTreeMap, BTreeSet};
use std::io::{BufRead, BufReader, Write};
use std::net::TcpStream;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};

use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
    admin_request_proof, agent_ed25519_public_key_from_private_key, coordinator_wire_request,
    derive_ed25519_private_key_from_seed, node_ed25519_public_key_from_private_key,
    sign_agent_workflow_request, sign_node_request, signed_request_payload_digest,
    AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability,
    DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements,
    LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits,
    SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId,
    TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult,
};
use serde_json::json;

use crate::FallibleDurableStore;

use super::keys::{process_control_key, task_control_key};
use super::*;

fn test_admin_request(
    token: &str,
    operation: &str,
    tenant: &str,
    actor_user: &str,
    target_tenant: &str,
    nonce: &str,
) -> (Digest, String, u64) {
    let issued_at_epoch_seconds = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .unwrap_or_default();
    (
        admin_request_proof(
            token,
            operation,
            tenant,
            actor_user,
            target_tenant,
            nonce,
            issued_at_epoch_seconds,
        ),
        nonce.to_owned(),
        issued_at_epoch_seconds,
    )
}

fn enroll_test_node(
    service: &mut CoordinatorService,
    tenant: &str,
    project: &str,
    node: &str,
    public_key: &str,
) {
    let response = service
        .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant {
            tenant: tenant.to_owned(),
            project: project.to_owned(),
            actor_user: "test-user".to_owned(),
            ttl_seconds: 900,
        })
        .unwrap();
    let CoordinatorResponse::NodeEnrollmentGrantCreated { grant, .. } = response else {
        panic!("expected node enrollment grant");
    };
    service
        .handle_request(CoordinatorRequest::ExchangeNodeEnrollmentGrant {
            tenant: tenant.to_owned(),
            project: project.to_owned(),
            node: node.to_owned(),
            public_key: public_key.to_owned(),
            enrollment_grant: grant,
        })
        .unwrap();
}

#[test]
fn runtime_service_uses_memory_only_when_database_url_is_absent() {
    let service = CoordinatorService::new_with_database_url(1, None).unwrap();
    assert_eq!(service.durable_store_kind(), "in_memory");

    let error = CoordinatorService::new_with_database_url(1, Some("not-a-postgres-url"))
        .err()
        .expect("an invalid configured DATABASE_URL must fail closed");
    assert!(error
        .to_string()
        .contains("durable coordinator state failed"));
}

#[test]
fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() {
    let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else {
        return;
    };
    let mut clean_store = crate::PostgresDurableStore::connect(&database_url).unwrap();
    clean_store
        .save_state(&crate::DurableState::default())
        .unwrap();

    let session_secret = "postgres-runtime-service-session";
    let mut first = CoordinatorService::new_with_database_url(41, Some(&database_url)).unwrap();
    assert_eq!(first.durable_store_kind(), "postgres");
    first
        .issue_cli_session(
            TenantId::from("tenant-pg"),
            ProjectId::from("project-pg"),
            UserId::from("user-pg"),
            session_secret,
            None,
        )
        .unwrap();
    enroll_test_node(
        &mut first,
        "tenant-pg",
        "project-pg",
        "node-pg",
        &test_node_public_key("node-pg"),
    );
    let duplicate_private_key = test_node_private_key("node-pg-other-scope");
    enroll_test_node(
        &mut first,
        "tenant-pg-other",
        "project-pg-other",
        "node-pg",
        &node_ed25519_public_key_from_private_key(&duplicate_private_key).unwrap(),
    );
    first
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "process-ephemeral".to_owned(),
                restart: false,
            },
        })
        .unwrap();
    drop(first);

    let mut restarted = CoordinatorService::new_with_database_url(42, Some(&database_url)).unwrap();
    assert_eq!(restarted.durable_store_kind(), "postgres");
    let auth = restarted
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    assert!(matches!(
        auth,
        CoordinatorResponse::AuthStatus {
            authenticated: true,
            ..
        }
    ));
    let heartbeat = restarted
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-pg".to_owned(),
            project: "project-pg".to_owned(),
            node: "node-pg".to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope(
                "tenant-pg",
                "project-pg",
                "node-pg",
                "postgres-restart-heartbeat",
            )),
        })
        .unwrap();
    assert!(matches!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat { epoch: 42, .. }
    ));
    let duplicate_heartbeat = restarted
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-pg-other".to_owned(),
            project: "project-pg-other".to_owned(),
            node: "node-pg".to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                "tenant-pg-other",
                "project-pg-other",
                "node-pg",
                &duplicate_private_key,
                "postgres-restart-heartbeat",
            )),
        })
        .unwrap();
    assert!(matches!(
        duplicate_heartbeat,
        CoordinatorResponse::NodeHeartbeat { epoch: 42, .. }
    ));
    let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcesses,
        })
        .unwrap()
    else {
        panic!("expected process list");
    };
    assert!(processes.is_empty());

    restarted
        .handle_request(CoordinatorRequest::RevokeNodeCredential {
            tenant: "tenant-pg".to_owned(),
            project: "project-pg".to_owned(),
            actor_user: "user-pg".to_owned(),
            node: "node-pg".to_owned(),
        })
        .unwrap();
    drop(restarted);

    let mut after_revocation =
        CoordinatorService::new_with_database_url(43, Some(&database_url)).unwrap();
    assert!(after_revocation
        .coordinator
        .node_identity(
            &TenantId::from("tenant-pg"),
            &ProjectId::from("project-pg"),
            &NodeId::from("node-pg"),
        )
        .is_none());
    assert!(after_revocation
        .coordinator
        .node_identity(
            &TenantId::from("tenant-pg-other"),
            &ProjectId::from("project-pg-other"),
            &NodeId::from("node-pg"),
        )
        .is_some());
    after_revocation
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-pg-other".to_owned(),
            project: "project-pg-other".to_owned(),
            node: "node-pg".to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                "tenant-pg-other",
                "project-pg-other",
                "node-pg",
                &duplicate_private_key,
                "post-revocation-restart",
            )),
        })
        .expect("the duplicate node in the other scope must remain authenticated");
}

fn linux_capabilities() -> NodeCapabilities {
    NodeCapabilities {
        os: Os::Linux,
        arch: "x86_64".to_owned(),
        capabilities: BTreeSet::from([
            Capability::Command,
            Capability::Containers,
            Capability::RootlessPodman,
            Capability::VfsArtifacts,
        ]),
        environment_backends: BTreeSet::from([EnvironmentBackend::Container]),
        source_providers: BTreeSet::from(["filesystem".to_owned()]),
    }
}

fn windows_capabilities() -> NodeCapabilities {
    NodeCapabilities {
        os: Os::Windows,
        arch: "x86_64".to_owned(),
        capabilities: BTreeSet::from([
            Capability::Command,
            Capability::WindowsCommandDev,
            Capability::VfsArtifacts,
        ]),
        environment_backends: BTreeSet::from([EnvironmentBackend::WindowsCommandDev]),
        source_providers: BTreeSet::from(["filesystem".to_owned()]),
    }
}

fn endpoint(name: &str) -> NodeEndpoint {
    NodeEndpoint {
        node: NodeId::from(name),
        advertised_addr: format!("{name}.mesh.invalid:4433"),
        public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
    }
}

fn data_plane_scope(project: &str) -> DataPlaneScope {
    DataPlaneScope {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from(project),
        process: ProcessId::from("process"),
        object: DataPlaneObject::Artifact(ArtifactId::from("artifact")),
        authorization_subject: "node-a-to-node-b".to_owned(),
    }
}

fn assert_agent_workflow_actor(actor: &WorkflowActor, fingerprint: &Digest) {
    assert_eq!(actor.kind, "agent");
    assert_eq!(actor.user, Some(UserId::from("user")));
    assert_eq!(actor.agent, Some(AgentId::from("agent-ci")));
    assert_eq!(actor.credential_kind, CredentialKind::PublicKey);
    assert_eq!(actor.public_key_fingerprint.as_ref(), Some(fingerprint));
    assert!(actor.authenticated_without_browser);
    assert!(actor.scopes.iter().any(|scope| scope == "project:run"));
}

const TEST_AGENT_PRIVATE_KEY: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
static TEST_NODE_REQUEST_NONCE: AtomicU64 = AtomicU64::new(1);
const TEST_WASM_MODULE: &[u8] = b"clusterflux-test-wasm-module";

fn test_wasm_module_base64() -> String {
    BASE64_STANDARD.encode(TEST_WASM_MODULE)
}

fn append_test_custom_section(module: &mut Vec<u8>, name: &str, data: &[u8]) {
    fn leb(mut value: usize, output: &mut Vec<u8>) {
        loop {
            let mut byte = (value & 0x7f) as u8;
            value >>= 7;
            if value != 0 {
                byte |= 0x80;
            }
            output.push(byte);
            if value == 0 {
                break;
            }
        }
    }

    let mut payload = Vec::new();
    leb(name.len(), &mut payload);
    payload.extend_from_slice(name.as_bytes());
    payload.extend_from_slice(data);
    module.push(0);
    leb(payload.len(), module);
    module.extend_from_slice(&payload);
}

fn test_edited_task_bundle(compatibility: &Digest, edit_marker: &str) -> (String, Digest) {
    let descriptor = serde_json::to_vec(&serde_json::json!({
        "kind": "task",
        "name": "compile",
        "function": "compile",
        "export": "compile_export",
        "stable_id": Digest::sha256("compile-stable"),
        "argument_schema": "u32",
        "result_schema": "u32",
        "required_capabilities": [],
        "restart_compatibility_hash": compatibility,
        "abi_version": clusterflux_core::WASM_TASK_ABI_VERSION,
        "probe_symbol": "clusterflux.probe.compile",
    }))
    .unwrap();
    let mut module = b"\0asm\x01\0\0\0".to_vec();
    append_test_custom_section(&mut module, "clusterflux.tasks", &descriptor);
    append_test_custom_section(&mut module, "clusterflux.edit", edit_marker.as_bytes());
    let digest = Digest::sha256(&module);
    (BASE64_STANDARD.encode(module), digest)
}

fn test_task_spec(
    tenant: &str,
    project: &str,
    process: &str,
    task: &str,
    epoch: u64,
    required_capabilities: impl IntoIterator<Item = Capability>,
) -> TaskSpec {
    test_task_spec_instance(
        tenant,
        project,
        process,
        task,
        task,
        epoch,
        required_capabilities,
    )
}

fn test_task_spec_instance(
    tenant: &str,
    project: &str,
    process: &str,
    task_definition: &str,
    task_instance: &str,
    epoch: u64,
    required_capabilities: impl IntoIterator<Item = Capability>,
) -> TaskSpec {
    TaskSpec {
        tenant: TenantId::from(tenant),
        project: ProjectId::from(project),
        process: ProcessId::from(process),
        task_definition: clusterflux_core::TaskDefinitionId::from(task_definition),
        task_instance: clusterflux_core::TaskInstanceId::from(task_instance),
        dispatch: TaskDispatch::CoordinatorNodeWasm {
            export: Some(task_definition.to_owned()),
            abi: WasmExportAbi::TaskV1,
        },
        environment_id: None,
        environment: None,
        environment_digest: None,
        required_capabilities: required_capabilities.into_iter().collect(),
        dependency_cache: None,
        source_snapshot: None,
        required_artifacts: Vec::new(),
        args: Vec::new(),
        vfs_epoch: epoch,
        failure_policy: Default::default(),
        bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
    }
}

trait AuthorizedTestTaskLaunch {
    fn handle_authorized_test_task_launch(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError>;
}

impl AuthorizedTestTaskLaunch for CoordinatorService {
    fn handle_authorized_test_task_launch(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
        let CoordinatorRequest::LaunchTask {
            task_spec,
            wait_for_node,
            artifact_path,
            wasm_module_base64,
            ..
        } = request
        else {
            panic!("authorized test task launch requires LaunchTask");
        };
        let tenant = task_spec.tenant.clone();
        let project = task_spec.project.clone();
        self.handle_launch_task_with_actor(
            tenant,
            project,
            WorkflowActor {
                kind: "task".to_owned(),
                user: None,
                agent: None,
                credential_kind: CredentialKind::TaskCredential,
                public_key_fingerprint: None,
                authenticated_without_browser: true,
                scopes: vec!["process:spawn-child".to_owned()],
            },
            task_spec,
            wait_for_node,
            artifact_path,
            wasm_module_base64,
        )
    }
}

fn register_test_task_assignment(
    service: &mut CoordinatorService,
    tenant: &str,
    project: &str,
    process: &str,
    node: &str,
    task_definition: &str,
    task_instance: &str,
    epoch: u64,
) {
    let task_spec = test_task_spec_instance(
        tenant,
        project,
        process,
        task_definition,
        task_instance,
        epoch,
        [],
    );
    let assignment = TaskAssignment {
        tenant: TenantId::from(tenant),
        project: ProjectId::from(project),
        process: ProcessId::from(process),
        task: TaskInstanceId::from(task_instance),
        node: NodeId::from(node),
        epoch,
        artifact_path: format!("/vfs/artifacts/{task_instance}.bin"),
        task_spec,
        wasm_module_base64: test_wasm_module_base64(),
    };
    service
        .capture_task_restart_checkpoint(&assignment)
        .unwrap();
    service.active_tasks.insert(super::keys::task_control_key(
        &assignment.tenant,
        &assignment.project,
        &assignment.process,
        &assignment.node,
        &assignment.task,
    ));
}

fn service_with_completed_main_and_final_child(
    failure_policy: clusterflux_core::TaskFailurePolicy,
) -> CoordinatorService {
    let mut service = CoordinatorService::new(83);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("terminal-matrix");
    let node = NodeId::from("worker");
    let task = TaskInstanceId::from("final-child");

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: tenant.to_string(),
            project: project.to_string(),
            node: node.to_string(),
            public_key: test_node_public_key(node.as_str()),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: tenant.to_string(),
            project: project.to_string(),
            node: node.to_string(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: process.to_string(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: tenant.to_string(),
            project: project.to_string(),
            node: node.to_string(),
            process: process.to_string(),
            epoch: 83,
        })
        .unwrap();

    let mut task_spec = test_task_spec_instance(
        tenant.as_str(),
        project.as_str(),
        process.as_str(),
        "child-definition",
        task.as_str(),
        83,
        [],
    );
    task_spec.failure_policy = failure_policy;
    let assignment = TaskAssignment {
        tenant: tenant.clone(),
        project: project.clone(),
        process: process.clone(),
        task: task.clone(),
        node: node.clone(),
        epoch: 83,
        artifact_path: "/vfs/artifacts/final-child.bin".to_owned(),
        task_spec: task_spec.clone(),
        wasm_module_base64: test_wasm_module_base64(),
    };
    service
        .capture_task_restart_checkpoint(&assignment)
        .unwrap();
    service
        .begin_task_attempt(
            &task_spec,
            Some(node.clone()),
            Some(&assignment.artifact_path),
            false,
        )
        .unwrap();
    service
        .active_tasks
        .insert(task_control_key(&tenant, &project, &process, &node, &task));
    service
        .task_assignments
        .entry((tenant.clone(), project.clone(), node.clone()))
        .or_default()
        .push_back(assignment);
    service
        .debug_epochs
        .insert(process_control_key(&tenant, &project, &process), 11);
    service.debug_breakpoints.insert(
        process_control_key(&tenant, &project, &process),
        super::debug::DebugBreakpointPlan {
            actor: UserId::from("user"),
            revision: 1,
            probe_symbols: BTreeSet::from(["child-probe".to_owned()]),
            hit_epoch: None,
            hit_task: None,
            hit_probe_symbol: None,
        },
    );
    service.debug_commands.insert(
        task_control_key(&tenant, &project, &process, &node, &task),
        super::debug::DebugPendingCommand {
            epoch: 11,
            command: "continue".to_owned(),
        },
    );
    let panel_key = super::keys::panel_stop_key(&tenant, &project, &process);
    service.panel_snapshots.insert(
        panel_key.clone(),
        PanelState {
            tenant: tenant.clone(),
            project: project.clone(),
            process: process.clone(),
            widgets: BTreeMap::new(),
            program_ui_events_enabled: false,
            control_plane_actions: Vec::new(),
        },
    );
    service.stopped_panels.insert(panel_key);
    service.record_task_completion_event(TaskCompletionEvent {
        tenant: tenant.clone(),
        project: project.clone(),
        process: process.clone(),
        node: NodeId::from("coordinator-main"),
        executor: TaskExecutor::CoordinatorMain,
        task_definition: TaskDefinitionId::from("build"),
        task: TaskInstanceId::from("main"),
        attempt_id: None,
        placement: None,
        terminal_state: TaskTerminalState::Completed,
        status_code: Some(0),
        stdout_bytes: 0,
        stderr_bytes: 0,
        stdout_tail: String::new(),
        stderr_tail: String::new(),
        stdout_truncated: false,
        stderr_truncated: false,
        artifact_path: None,
        artifact_digest: None,
        artifact_size_bytes: None,
        result: None,
    });
    service
        .coordinator
        .grant_project_debug(tenant, project, UserId::from("user"));
    service
}

fn complete_terminal_matrix_child(
    service: &mut CoordinatorService,
    terminal_state: TaskTerminalState,
) {
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "terminal-matrix".to_owned(),
            node: "worker".to_owned(),
            task: "final-child".to_owned(),
            terminal_state: Some(terminal_state),
            status_code: Some(1),
            stdout_bytes: 0,
            stderr_bytes: 4,
            stdout_tail: String::new(),
            stderr_tail: "boom".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();
}

fn test_agent_public_key() -> String {
    agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap()
}

fn signed_agent_workflow_request(
    request: &CoordinatorRequest,
    request_kind: &str,
    process: &str,
    task: Option<&str>,
    nonce: &str,
) -> AgentSignedRequest {
    let task = task.map(TaskInstanceId::from);
    let payload = serde_json::to_value(request).unwrap();
    let payload_digest = signed_request_payload_digest(&payload);
    sign_agent_workflow_request(
        TEST_AGENT_PRIVATE_KEY,
        AgentWorkflowScope {
            tenant: &TenantId::from("tenant"),
            project: &ProjectId::from("project"),
            agent: &AgentId::from("agent-ci"),
            request_kind,
            process: &ProcessId::from(process),
            task: task.as_ref(),
        },
        &payload_digest,
        nonce.to_owned(),
        unix_timestamp_seconds_for_tests(),
    )
    .unwrap()
}

fn with_signed_agent_workflow(
    mut request: CoordinatorRequest,
    request_kind: &str,
    process: &str,
    task: Option<&str>,
    nonce: &str,
) -> CoordinatorRequest {
    let signature = signed_agent_workflow_request(&request, request_kind, process, task, nonce);
    match &mut request {
        CoordinatorRequest::StartProcess {
            launch_attempt: None,
            agent_signature,
            ..
        }
        | CoordinatorRequest::LaunchTask {
            agent_signature, ..
        } => *agent_signature = Some(signature),
        _ => panic!("agent signing helper only accepts agent workflow requests"),
    }
    request
}

fn test_node_private_key(node: &str) -> String {
    derive_ed25519_private_key_from_seed(&format!("test-node-key:{node}"))
}

fn test_node_public_key(node: &str) -> String {
    node_ed25519_public_key_from_private_key(&test_node_private_key(node)).unwrap()
}

fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest {
    signed_node_heartbeat_in_scope("tenant", "project", node, nonce)
}

fn signed_node_heartbeat_in_scope(
    tenant: &str,
    project: &str,
    node: &str,
    nonce: &str,
) -> NodeSignedRequest {
    let payload = json!({
        "type": "node_heartbeat",
        "tenant": tenant,
        "project": project,
        "node": node
    });
    signed_node_request_with_private_key(
        node,
        &test_node_private_key(node),
        "node_heartbeat",
        &signed_request_payload_digest(&payload),
        nonce,
    )
}

fn signed_node_heartbeat_with_private_key(
    node: &str,
    private_key: &str,
    nonce: &str,
) -> NodeSignedRequest {
    signed_node_heartbeat_in_scope_with_private_key("tenant", "project", node, private_key, nonce)
}

fn signed_node_heartbeat_in_scope_with_private_key(
    tenant: &str,
    project: &str,
    node: &str,
    private_key: &str,
    nonce: &str,
) -> NodeSignedRequest {
    let payload = json!({
        "type": "node_heartbeat",
        "tenant": tenant,
        "project": project,
        "node": node
    });
    signed_node_request_with_private_key(
        node,
        private_key,
        "node_heartbeat",
        &signed_request_payload_digest(&payload),
        nonce,
    )
}

fn signed_node_request_with_private_key(
    node: &str,
    private_key: &str,
    request_kind: &str,
    payload_digest: &Digest,
    nonce: &str,
) -> NodeSignedRequest {
    sign_node_request(
        private_key,
        &NodeId::from(node),
        request_kind,
        payload_digest,
        nonce.to_owned(),
        unix_timestamp_seconds_for_tests(),
    )
    .unwrap()
}

fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest {
    let node = match &request {
        CoordinatorRequest::ReportNodeCapabilities { node, .. }
        | CoordinatorRequest::PollTaskAssignment { node, .. }
        | CoordinatorRequest::PollArtifactTransfer { node, .. }
        | CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
        | CoordinatorRequest::FailArtifactTransfer { node, .. }
        | CoordinatorRequest::LaunchChildTask { node, .. }
        | CoordinatorRequest::JoinChildTask { node, .. }
        | CoordinatorRequest::CompleteSourcePreparation { node, .. }
        | CoordinatorRequest::ReconnectNode { node, .. }
        | CoordinatorRequest::PollTaskControl { node, .. }
        | CoordinatorRequest::PollDebugCommand { node, .. }
        | CoordinatorRequest::ReportDebugState { node, .. }
        | CoordinatorRequest::ReportDebugProbeHit { node, .. }
        | CoordinatorRequest::ReportTaskLog { node, .. }
        | CoordinatorRequest::ReportTaskLogChunk { node, .. }
        | CoordinatorRequest::ReportVfsMetadata { node, .. }
        | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(),
        CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(),
        _ => panic!("test helper only signs node-originated requests"),
    };
    signed_node_request_auto_with_private_key(request, &test_node_private_key(&node))
}

fn signed_node_request_auto_with_private_key(
    request: CoordinatorRequest,
    private_key: &str,
) -> CoordinatorRequest {
    let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap());
    let (node, request_kind) = match &request {
        CoordinatorRequest::ReportNodeCapabilities { node, .. } => {
            (node.clone(), "report_node_capabilities")
        }
        CoordinatorRequest::PollTaskAssignment { node, .. } => {
            (node.clone(), "poll_task_assignment")
        }
        CoordinatorRequest::PollArtifactTransfer { node, .. } => {
            (node.clone(), "poll_artifact_transfer")
        }
        CoordinatorRequest::UploadArtifactTransferChunk { node, .. } => {
            (node.clone(), "upload_artifact_transfer_chunk")
        }
        CoordinatorRequest::FailArtifactTransfer { node, .. } => {
            (node.clone(), "fail_artifact_transfer")
        }
        CoordinatorRequest::LaunchChildTask { node, .. } => (node.clone(), "launch_child_task"),
        CoordinatorRequest::JoinChildTask { node, .. } => (node.clone(), "join_child_task"),
        CoordinatorRequest::CompleteSourcePreparation { node, .. } => {
            (node.clone(), "complete_source_preparation")
        }
        CoordinatorRequest::RequestRendezvous { source, .. } => {
            (source.node.to_string(), "request_rendezvous")
        }
        CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"),
        CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"),
        CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"),
        CoordinatorRequest::ReportDebugState { node, .. } => (node.clone(), "report_debug_state"),
        CoordinatorRequest::ReportDebugProbeHit { node, .. } => {
            (node.clone(), "report_debug_probe_hit")
        }
        CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"),
        CoordinatorRequest::ReportTaskLogChunk { node, .. } => {
            (node.clone(), "report_task_log_chunk")
        }
        CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"),
        CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"),
        _ => panic!("test helper only signs node-originated requests"),
    };
    let nonce = TEST_NODE_REQUEST_NONCE
        .fetch_add(1, Ordering::Relaxed)
        .to_string();
    CoordinatorRequest::SignedNode {
        node: node.clone(),
        node_signature: signed_node_request_with_private_key(
            &node,
            private_key,
            request_kind,
            &payload_digest,
            &format!("node-request-{nonce}"),
        ),
        request: Box::new(request),
    }
}

trait SignedNodeRequestTestExt {
    fn handle_signed_node_request_auto(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError>;
}

impl SignedNodeRequestTestExt for CoordinatorService {
    fn handle_signed_node_request_auto(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
        self.handle_request(signed_node_request_auto(request))
    }
}

fn upload_pending_artifact_transfer(
    service: &mut CoordinatorService,
    node: &str,
    bytes: &[u8],
) -> ArtifactTransferAssignment {
    let mut first = None;
    loop {
        let CoordinatorResponse::ArtifactTransferAssignment {
            transfer: Some(transfer),
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected pending artifact reverse transfer");
        };
        assert_eq!(transfer.expected_digest, Digest::sha256(bytes));
        assert_eq!(transfer.expected_size_bytes, bytes.len() as u64);
        let start = transfer.offset as usize;
        let end = start
            .saturating_add(transfer.max_chunk_bytes as usize)
            .min(bytes.len());
        let content = &bytes[start..end];
        let response = service
            .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                transfer_id: transfer.transfer_id.clone(),
                artifact: transfer.artifact.as_str().to_owned(),
                offset: transfer.offset,
                content_base64: BASE64_STANDARD.encode(content),
                chunk_digest: Digest::sha256(content),
                eof: end == bytes.len(),
            })
            .unwrap();
        let complete = matches!(
            response,
            CoordinatorResponse::ArtifactTransferChunkAccepted { complete: true, .. }
        );
        first.get_or_insert_with(|| transfer.clone());
        if complete {
            return first.unwrap();
        }
    }
}

fn unix_timestamp_seconds_for_tests() -> u64 {
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .unwrap_or(0)
}

fn write_coordinator_wire_request(
    stream: &mut TcpStream,
    request: &CoordinatorRequest,
    request_id: &str,
) {
    let payload = serde_json::to_value(request).unwrap();
    let wire_request = coordinator_wire_request(request_id, payload);
    serde_json::to_writer(&mut *stream, &wire_request).unwrap();
    stream.write_all(b"\n").unwrap();
    stream.flush().unwrap();
}

#[test]
fn service_creates_selects_and_lists_signed_in_user_projects() {
    let mut service = CoordinatorService::new(7);

    let CoordinatorResponse::ProjectCreated { project, actor } = service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-a".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project creation");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(project.tenant, TenantId::from("tenant-a"));
    assert_eq!(project.id, ProjectId::from("project-a"));
    assert_eq!(project.name, "Demo");

    let CoordinatorResponse::ProjectSelected { project, actor } = service
        .handle_request(CoordinatorRequest::SelectProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-a".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project selection");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(project.id, ProjectId::from("project-a"));

    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-b".to_owned(),
            actor_user: "user-b".to_owned(),
            project: "project-b".to_owned(),
            name: "Other".to_owned(),
        })
        .unwrap();

    let CoordinatorResponse::Projects { projects, actor } = service
        .handle_request(CoordinatorRequest::ListProjects {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(projects.len(), 1);
    assert_eq!(projects[0].id, ProjectId::from("project-a"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::SelectProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-b".to_owned(),
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("tenant scope"));
}

#[test]
fn authenticated_envelope_derives_user_scope_from_cli_session() {
    let mut service = CoordinatorService::new(7);
    service
        .issue_cli_session(
            TenantId::from("tenant-a"),
            ProjectId::from("project-a"),
            UserId::from("user-a"),
            "cli-session-secret",
            None,
        )
        .unwrap();

    let CoordinatorResponse::AuthStatus {
        tenant,
        project,
        actor,
        authenticated,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap()
    else {
        panic!("expected authenticated auth status");
    };
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));
    assert!(authenticated);

    let CoordinatorResponse::ProjectCreated { project, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CreateProject {
                project: "project-b".to_owned(),
                name: "From Session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated project creation");
    };
    assert_eq!(project.tenant, TenantId::from("tenant-a"));
    assert_eq!(project.id, ProjectId::from("project-b"));
    assert_eq!(actor, UserId::from("user-a"));

    let CoordinatorResponse::Projects { projects, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProjects,
        })
        .unwrap()
    else {
        panic!("expected authenticated project list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(projects.len(), 2);
    assert!(projects
        .iter()
        .any(|project| project.id == ProjectId::from("project-a")));
    assert!(projects
        .iter()
        .any(|project| project.id == ProjectId::from("project-b")));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RegisterAgentPublicKey {
                agent: "agent-session".to_owned(),
                public_key: "agent-session-key-v1".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key registration");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(record.tenant, TenantId::from("tenant-a"));
    assert_eq!(record.project, ProjectId::from("project-a"));
    assert_eq!(record.user, UserId::from("user-a"));
    assert_eq!(record.agent, AgentId::from("agent-session"));

    let CoordinatorResponse::AgentPublicKeys { records, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListAgentPublicKeys,
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(records.len(), 1);
    assert_eq!(records[0].project, ProjectId::from("project-a"));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeAgentPublicKey {
                agent: "agent-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key revocation");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert!(record.revoked);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "session-node".to_owned(),
            public_key: test_node_public_key("session-node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "session-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeDescriptors,
        })
        .unwrap()
    else {
        panic!("expected authenticated node descriptor list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(descriptors.len(), 1);
    assert_eq!(descriptors[0].id, NodeId::from("session-node"));

    let CoordinatorResponse::TaskPlacement { placement } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ScheduleTask {
                environment: None,
                environment_digest: None,
                required_capabilities: vec![Capability::Command],
                dependency_cache: None,
                source_snapshot: None,
                required_artifacts: vec![],
                prefer_node: Some("session-node".to_owned()),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated task placement");
    };
    assert_eq!(placement.node, NodeId::from("session-node"));

    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        process,
        actor,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "vp-session".to_owned(),
                restart: false,
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated process start");
    };
    assert_eq!(process, ProcessId::from("vp-session"));
    assert_eq!(actor.user, Some(UserId::from("user-a")));

    let denied_external_task = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::LaunchTask {
                task_spec: Box::new(test_task_spec(
                    "tenant-a",
                    "project-a",
                    "vp-session",
                    "task-session",
                    7,
                    [Capability::Command],
                )),
                wait_for_node: false,
                artifact_path: "/vfs/artifacts/session.txt".to_owned(),
                wasm_module_base64: test_wasm_module_base64(),
            },
        })
        .unwrap_err();
    assert!(denied_external_task
        .to_string()
        .contains("external callers may launch only EntrypointV1"));

    let CoordinatorResponse::TaskLaunched {
        process,
        task,
        actor,
        assignment,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant-a",
                "project-a",
                "vp-session",
                "task-session",
                7,
                [Capability::Command],
            ),
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/session.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected authenticated task launch");
    };
    assert_eq!(process, ProcessId::from("vp-session"));
    assert_eq!(task, TaskInstanceId::from("task-session"));
    assert_eq!(actor.kind, "task");
    assert_eq!(assignment.tenant, TenantId::from("tenant-a"));
    assert_eq!(assignment.project, ProjectId::from("project-a"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "session-node".to_owned(),
            process: "vp-session".to_owned(),
            epoch: 7,
        })
        .unwrap();
    let artifact_bytes = "session artifact";
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            process: "vp-session".to_owned(),
            node: "session-node".to_owned(),
            task: "task-session".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: artifact_bytes.len() as u64,
            stderr_bytes: 0,
            stdout_tail: artifact_bytes.to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/session.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListTaskEvents {
                process: Some("vp-session".to_owned()),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated task event list");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].tenant, TenantId::from("tenant-a"));
    assert_eq!(events[0].project, ProjectId::from("project-a"));

    service
        .issue_cli_session(
            TenantId::from("tenant-b"),
            ProjectId::from("project-victim"),
            UserId::from("user-b"),
            "victim-cli-session-secret",
            None,
        )
        .unwrap();
    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "victim-cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "vp-victim".to_owned(),
                restart: false,
            },
        })
        .unwrap();
    let cross_tenant_events = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListTaskEvents {
                process: Some("vp-victim".to_owned()),
            },
        })
        .unwrap_err();
    assert!(cross_tenant_events
        .to_string()
        .contains("outside the virtual process tenant/project scope"));

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
                artifact: "session.txt".to_owned(),
                max_bytes: 1024,
                ttl_seconds: 60,
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated artifact link");
    };
    assert_eq!(link.tenant, TenantId::from("tenant-a"));
    assert_eq!(link.project, ProjectId::from("project-a"));
    assert_eq!(link.actor, Actor::User(UserId::from("user-a")));

    let CoordinatorResponse::ProcessCancellationRequested { process, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CancelProcess {
                process: "vp-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated process cancellation");
    };
    assert_eq!(process, ProcessId::from("vp-session"));

    let CoordinatorResponse::DebugAttach { actor, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::DebugAttach {
                process: "vp-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated debug attach");
    };
    assert_eq!(actor, UserId::from("user-a"));

    let CoordinatorResponse::NodeCredentialRevoked {
        node,
        tenant,
        project,
        actor,
        descriptor_removed,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeNodeCredential {
                node: "session-node".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated node credential revocation");
    };
    assert_eq!(node, NodeId::from("session-node"));
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));
    assert!(descriptor_removed);

    let rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "wrong-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(rejected.to_string().contains("not recognized"));

    let expired = service
        .issue_cli_session(
            TenantId::from("tenant-a"),
            ProjectId::from("project-b"),
            UserId::from("user-a"),
            "expired-cli-session-secret",
            Some(1),
        )
        .unwrap();
    assert_eq!(expired.expires_at_epoch_seconds, Some(1));
    let expired_rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "expired-cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(expired_rejected.to_string().contains("expired"));

    let CoordinatorResponse::CliSessionRevoked {
        tenant,
        project,
        actor,
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeCliSession,
        })
        .unwrap()
    else {
        panic!("expected CLI session revocation");
    };
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));

    let revoked_rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(revoked_rejected.to_string().contains("revoked"));
}

#[test]
fn service_reports_and_enforces_public_admin_tenant_suspension() {
    let mut service = CoordinatorService::new_with_admin_token(7, "admin-token");

    let CoordinatorResponse::AuthStatus {
        tenant,
        project,
        actor,
        authenticated,
        account_status,
        suspended,
        disabled,
        sanitized_reason,
        sensitive_moderation_details_exposed,
        signup_failure_details_exposed,
        ..
    } = service
        .handle_request(CoordinatorRequest::AuthStatus {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected auth status");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(project, ProjectId::from("project"));
    assert_eq!(actor, UserId::from("user"));
    assert!(authenticated);
    assert_eq!(account_status, "active");
    assert!(!suspended);
    assert!(!disabled);
    assert!(sanitized_reason.is_none());
    assert!(!sensitive_moderation_details_exposed);
    assert!(!signup_failure_details_exposed);

    for (tenant, policy_name, expected_status, expected_reason) in [
        (
            "manual-tenant",
            "tenant:manual_review",
            "manual_review",
            "account or tenant is pending hosted review",
        ),
        (
            "disabled-tenant",
            "tenant:disabled",
            "disabled",
            "account or tenant is disabled by hosted policy",
        ),
        (
            "deleted-tenant",
            "tenant:deleted",
            "deleted",
            "account or tenant is no longer active",
        ),
    ] {
        service.coordinator.upsert_service_policy_record(
            TenantId::from(tenant),
            policy_name,
            Digest::from_parts([tenant.as_bytes(), policy_name.as_bytes()]),
        );
        let CoordinatorResponse::AuthStatus {
            account_status,
            suspended,
            disabled,
            deleted,
            manual_review,
            sanitized_reason,
            next_actions,
            sensitive_moderation_details_exposed,
            signup_failure_details_exposed,
            ..
        } = service
            .handle_request(CoordinatorRequest::AuthStatus {
                tenant: tenant.to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
            })
            .unwrap()
        else {
            panic!("expected inactive auth status");
        };
        assert_eq!(account_status, expected_status);
        assert_eq!(suspended, expected_status == "suspended");
        assert_eq!(disabled, expected_status == "disabled");
        assert_eq!(deleted, expected_status == "deleted");
        assert_eq!(manual_review, expected_status == "manual_review");
        assert_eq!(sanitized_reason.as_deref(), Some(expected_reason));
        assert!(next_actions
            .iter()
            .any(|action| action.contains("hosted operator")));
        assert!(!sensitive_moderation_details_exposed);
        assert!(!signup_failure_details_exposed);
    }

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-1",
    );
    let admin_status_request = CoordinatorRequest::AdminStatus {
        tenant: "tenant".to_owned(),
        actor_user: "admin".to_owned(),
        admin_proof,
        admin_nonce,
        issued_at_epoch_seconds,
    };
    let CoordinatorResponse::AdminStatus {
        tenant,
        actor,
        suspended,
        safe_default,
    } = service
        .handle_request(admin_status_request.clone())
        .unwrap()
    else {
        panic!("expected admin status");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(actor, UserId::from("admin"));
    assert!(!suspended);
    assert_eq!(safe_default, "read_only");
    let replayed_admin_status = service.handle_request(admin_status_request).unwrap_err();
    assert!(replayed_admin_status
        .to_string()
        .contains("nonce was already used"));

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "suspend_tenant",
        "admin-tenant",
        "admin",
        "tenant",
        "admin-suspend-1",
    );
    let CoordinatorResponse::TenantSuspended {
        tenant,
        actor,
        policy,
    } = service
        .handle_request(CoordinatorRequest::SuspendTenant {
            tenant: "admin-tenant".to_owned(),
            actor_user: "admin".to_owned(),
            target_tenant: "tenant".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap()
    else {
        panic!("expected tenant suspension");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(actor, UserId::from("admin"));
    assert_eq!(policy.name, "tenant:suspended");

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-2",
    );
    let CoordinatorResponse::AdminStatus { suspended, .. } = service
        .handle_request(CoordinatorRequest::AdminStatus {
            tenant: "tenant".to_owned(),
            actor_user: "admin".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap()
    else {
        panic!("expected suspended admin status");
    };
    assert!(suspended);

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-unconfigured",
    );
    let missing_admin_credential = CoordinatorService::new(7)
        .handle_request(CoordinatorRequest::AdminStatus {
            tenant: "tenant".to_owned(),
            actor_user: "admin".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap_err();
    assert!(missing_admin_credential
        .to_string()
        .contains("admin credential is not configured"));

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "wrong-token",
        "suspend_tenant",
        "admin-tenant",
        "admin",
        "other-tenant",
        "admin-suspend-wrong",
    );
    let invalid_admin_credential = service
        .handle_request(CoordinatorRequest::SuspendTenant {
            tenant: "admin-tenant".to_owned(),
            actor_user: "admin".to_owned(),
            target_tenant: "other-tenant".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap_err();
    assert!(invalid_admin_credential
        .to_string()
        .contains("admin request proof is invalid"));

    let CoordinatorResponse::AuthStatus {
        account_status,
        suspended,
        disabled,
        sanitized_reason,
        next_actions,
        sensitive_moderation_details_exposed,
        signup_failure_details_exposed,
        ..
    } = service
        .handle_request(CoordinatorRequest::AuthStatus {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected suspended auth status");
    };
    assert_eq!(account_status, "suspended");
    assert!(suspended);
    assert!(!disabled);
    assert_eq!(
        sanitized_reason.as_deref(),
        Some("account or tenant is suspended by hosted policy")
    );
    assert!(next_actions
        .iter()
        .any(|action| action.contains("hosted operator")));
    assert!(!sensitive_moderation_details_exposed);
    assert!(!signup_failure_details_exposed);

    let create = service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap_err();
    assert!(create.to_string().contains("tenant is suspended"));

    let start = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: Some("attempt-a".to_owned()),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(start.to_string().contains("tenant is suspended"));
}

#[test]
fn service_manages_project_scoped_agent_public_keys() {
    let mut service = CoordinatorService::new(7);

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: "agent-key-v1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key registration");
    };
    assert_eq!(actor, UserId::from("user"));
    assert_eq!(record.tenant, TenantId::from("tenant"));
    assert_eq!(record.project, ProjectId::from("project"));
    assert_eq!(record.user, UserId::from("user"));
    assert_eq!(record.agent, AgentId::from("agent-ci"));
    assert_eq!(record.version, 1);
    assert!(!record.revoked);
    assert_eq!(record.scopes, vec!["project:read", "project:run"]);
    assert!(!record.human_account_creation_privilege);
    assert!(!record.browser_interaction_required_each_run);

    let CoordinatorResponse::AgentPublicKey { record, .. } = service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: "agent-key-v2".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key rotation by re-add");
    };
    assert_eq!(record.version, 2);
    assert_eq!(record.public_key, "agent-key-v2");

    let CoordinatorResponse::AgentPublicKeys { records, actor } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key list");
    };
    assert_eq!(actor, UserId::from("user"));
    assert_eq!(records.len(), 1);
    assert_eq!(records[0].agent, AgentId::from("agent-ci"));

    let CoordinatorResponse::AgentPublicKeys { records, .. } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "other-user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected foreign user agent public key list");
    };
    assert!(records.is_empty());

    let CoordinatorResponse::AgentPublicKeys { records, .. } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected foreign tenant agent public key list");
    };
    assert!(records.is_empty());

    let foreign_user_revoke = service
        .handle_request(CoordinatorRequest::RevokeAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "other-user".to_owned(),
            agent: "agent-ci".to_owned(),
        })
        .unwrap_err();
    assert!(foreign_user_revoke
        .to_string()
        .contains("signed-in user scope"));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::RevokeAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key revocation");
    };
    assert_eq!(actor, UserId::from("user"));
    assert!(record.revoked);
}

#[test]
fn service_runs_agent_workflows_with_scoped_key_attribution() {
    let mut service = CoordinatorService::new(7);
    let agent_public_key = test_agent_public_key();
    let agent_fingerprint = Digest::sha256(&agent_public_key);

    service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: agent_public_key,
        })
        .unwrap();

    let fingerprint_only = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: Some("agent-ci".to_owned()),
            agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
            agent_signature: None,
            process: "vp-agent-fingerprint-only".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(fingerprint_only.to_string().contains("signed request"));

    let wrong_fingerprint = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                launch_attempt: None,
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(Digest::sha256("other-key")),
                agent_signature: None,
                process: "vp-agent-bad-key".to_owned(),
                restart: false,
            },
            "start_process",
            "vp-agent-bad-key",
            None,
            "bad-fingerprint-nonce",
        ))
        .unwrap_err();
    assert!(wrong_fingerprint.to_string().contains("fingerprint"));

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        actor: start_actor,
        ..
    } = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                launch_attempt: None,
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                process: "vp-agent".to_owned(),
                restart: false,
            },
            "start_process",
            "vp-agent",
            None,
            "start-nonce",
        ))
        .unwrap()
    else {
        panic!("expected agent-authenticated process start");
    };
    assert_agent_workflow_actor(&start_actor, &agent_fingerprint);

    let replay = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                launch_attempt: None,
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                process: "vp-agent".to_owned(),
                restart: true,
            },
            "start_process",
            "vp-agent",
            None,
            "start-nonce",
        ))
        .unwrap_err();
    assert!(replay.to_string().contains("nonce"));

    let denied_external_task = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::LaunchTask {
                task_spec: test_task_spec(
                    "tenant",
                    "project",
                    "vp-agent",
                    "compile-linux",
                    7,
                    [Capability::Command],
                ),
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                wait_for_node: false,
                artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
                wasm_module_base64: test_wasm_module_base64(),
            },
            "launch_task",
            "vp-agent",
            Some("compile-linux"),
            "launch-nonce",
        ))
        .unwrap_err();
    assert!(denied_external_task
        .to_string()
        .contains("external callers may launch only EntrypointV1"));
}

#[test]
fn signed_node_and_agent_requests_reject_body_modification() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-bound-body".to_owned(),
            public_key: test_node_public_key("worker-bound-body"),
        })
        .unwrap();

    let original_node_request = CoordinatorRequest::ReportNodeCapabilities {
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        node: "worker-bound-body".to_owned(),
        capabilities: linux_capabilities(),
        cached_environment_digests: Vec::new(),
        dependency_cache_digests: Vec::new(),
        source_snapshots: Vec::new(),
        artifact_locations: Vec::new(),
        direct_connectivity: true,
        online: true,
    };
    let original_node_digest =
        signed_request_payload_digest(&serde_json::to_value(&original_node_request).unwrap());
    let node_signature = signed_node_request_with_private_key(
        "worker-bound-body",
        &test_node_private_key("worker-bound-body"),
        "report_node_capabilities",
        &original_node_digest,
        "node-body-modification",
    );
    let mut modified_node_request = original_node_request;
    let CoordinatorRequest::ReportNodeCapabilities { online, .. } = &mut modified_node_request
    else {
        unreachable!();
    };
    *online = false;
    let modified_node = service
        .handle_request(CoordinatorRequest::SignedNode {
            node: "worker-bound-body".to_owned(),
            node_signature,
            request: Box::new(modified_node_request),
        })
        .unwrap_err();
    assert!(modified_node.to_string().contains("signature"));

    let agent_public_key = test_agent_public_key();
    let agent_fingerprint = Digest::sha256(&agent_public_key);
    service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: agent_public_key,
        })
        .unwrap();
    let original_agent_request = CoordinatorRequest::StartProcess {
        launch_attempt: None,
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        actor_user: None,
        actor_agent: Some("agent-ci".to_owned()),
        agent_public_key_fingerprint: Some(agent_fingerprint),
        agent_signature: None,
        process: "vp-agent-bound-body".to_owned(),
        restart: false,
    };
    let agent_signature = signed_agent_workflow_request(
        &original_agent_request,
        "start_process",
        "vp-agent-bound-body",
        None,
        "agent-body-modification",
    );
    let mut modified_agent_request = original_agent_request;
    let CoordinatorRequest::StartProcess {
        launch_attempt: None,
        restart,
        agent_signature: request_signature,
        ..
    } = &mut modified_agent_request
    else {
        unreachable!();
    };
    *restart = true;
    *request_signature = Some(agent_signature);
    let modified_agent = service.handle_request(modified_agent_request).unwrap_err();
    assert!(modified_agent.to_string().contains("signature"));
}

#[test]
fn service_checks_spawn_quota_before_process_or_task_work_starts() {
    let mut service = CoordinatorService::new(7);
    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 2)]),
    });

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        charged_spawns,
        ..
    } = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-quota".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start within spawn quota");
    };
    assert_eq!(charged_spawns, 1);

    let CoordinatorResponse::TaskLaunched { charged_spawns, .. } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-quota",
                "compile-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected task launch within spawn quota");
    };
    assert_eq!(charged_spawns, 2);

    let denied_task = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-quota",
                "compile-linux-denied",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/denied.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(denied_task.to_string().contains("Spawn"));
    let now_epoch_seconds = service.current_epoch_seconds().unwrap();
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            now_epoch_seconds,
        ),
        2
    );

    let denied_task_key = task_control_key(
        &TenantId::from("tenant"),
        &ProjectId::from("project"),
        &ProcessId::from("vp-quota"),
        &NodeId::from("worker-linux"),
        &TaskInstanceId::from("compile-linux-denied"),
    );
    assert!(!service.active_tasks.contains(&denied_task_key));
    assert!(!service.task_placements.contains_key(&denied_task_key));
    assert_eq!(
        service
            .task_assignments
            .get(&(
                TenantId::from("tenant"),
                ProjectId::from("project"),
                NodeId::from("worker-linux"),
            ))
            .map(VecDeque::len),
        Some(1)
    );

    let other_project_process = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-denied".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        other_project_process,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: None,
            ..
        }
    ));
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("other-project"),
            now_epoch_seconds,
        ),
        1
    );
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            now_epoch_seconds,
        ),
        2
    );
}

#[test]
fn project_quota_resets_at_the_configured_window_boundary() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::Spawn, 1);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::Spawn, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();

    service.set_server_time(59);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: Some("attempt-b".to_owned()),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-window-one".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AbortProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-window-one".to_owned(),
        })
        .unwrap();

    let exhausted = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: Some("attempt-b".to_owned()),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-still-window-one".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(exhausted.to_string().contains("Spawn"));

    service.set_server_time(60);
    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-window-two".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: None,
            charged_spawns: 1,
            ..
        }
    ));
    assert_eq!(service.quota.active_meter_count(), 1);
}

#[test]
fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::ApiCall, 1);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::ApiCall, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();
    service.set_server_time(30);
    for (project, secret) in [("project-a", "session-a"), ("project-b", "session-b")] {
        service
            .issue_cli_session(
                TenantId::from("tenant"),
                ProjectId::from(project),
                UserId::from("user"),
                secret,
                None,
            )
            .unwrap();
    }

    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    let denied = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(denied.to_string().contains("ApiCall"));

    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-b".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    assert_eq!(
        service
            .quota
            .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-a"), 30,),
        1
    );
    assert_eq!(
        service
            .quota
            .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-b"), 30,),
        1
    );
}

#[test]
fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::LogBytes, 4);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();
    service.set_server_time(30);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "task".to_owned(),
            stdout_bytes: 3,
            stderr_bytes: 1,
            stdout_tail: "out".to_owned(),
            stderr_tail: "e".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: false,
        })
        .unwrap();
    let truncated = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "task".to_owned(),
            stdout_bytes: 4,
            stderr_bytes: 1,
            stdout_tail: "outx".to_owned(),
            stderr_tail: "e".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: true,
        })
        .unwrap();
    let CoordinatorResponse::TaskLogRecorded {
        stdout_tail,
        stdout_bytes: 4,
        ..
    } = truncated
    else {
        panic!("expected a successful truncated task-log report");
    };
    assert_eq!(stdout_tail, "[log output truncated at project log quota]");
    assert!(service
        .recent_logs
        .get(&(TenantId::from("tenant"), ProjectId::from("project")))
        .unwrap()
        .iter()
        .any(|entry| entry.text.contains("project log quota") && entry.truncated));
    assert_eq!(
        service
            .quota
            .used_log_bytes(&TenantId::from("tenant"), &ProjectId::from("project"), 30,),
        4
    );
}

#[test]
fn log_quota_exhaustion_cannot_strand_task_completion_or_artifact_publication() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::LogBytes, 4);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();
    service.set_server_time(30);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile",
        "compile-one",
        7,
    );
    service.record_task_completion_event(TaskCompletionEvent {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("process"),
        node: NodeId::from("coordinator-main"),
        executor: TaskExecutor::CoordinatorMain,
        task_definition: TaskDefinitionId::from("build"),
        task: TaskInstanceId::from("main"),
        attempt_id: None,
        placement: None,
        terminal_state: TaskTerminalState::Completed,
        status_code: Some(0),
        stdout_bytes: 0,
        stderr_bytes: 0,
        stdout_tail: String::new(),
        stderr_tail: String::new(),
        stdout_truncated: false,
        stderr_truncated: false,
        artifact_path: None,
        artifact_digest: None,
        artifact_size_bytes: None,
        result: None,
    });

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-one".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 64,
            stderr_bytes: 0,
            stdout_tail: "the-real-final-tail".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: true,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/result.bin".to_owned()),
            artifact_digest: Some(Digest::sha256("artifact bytes")),
            artifact_size_bytes: Some(14),
            result: None,
        })
        .unwrap();

    let event = service
        .task_events
        .iter()
        .find(|event| event.task == TaskInstanceId::from("compile-one"))
        .unwrap();
    assert_eq!(
        event.stdout_tail,
        "[log output truncated at project log quota]"
    );
    assert!(event.stdout_truncated);
    assert!(!service
        .active_tasks
        .iter()
        .any(|key| key.4 == TaskInstanceId::from("compile-one")));
    assert!(service
        .coordinator
        .active_process(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            &ProcessId::from("process"),
        )
        .is_none());
    assert!(matches!(
        service
            .handle_request(CoordinatorRequest::GetArtifact {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
                artifact: "result.bin".to_owned(),
            })
            .unwrap(),
        CoordinatorResponse::Artifact { .. }
    ));
}

#[test]
fn service_attaches_node_starts_process_and_records_scoped_task_event() {
    let mut service = CoordinatorService::new(7);

    let attached = service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    assert!(matches!(attached, CoordinatorResponse::NodeAttached { .. }));

    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    assert_eq!(
        started,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: None,
            process: ProcessId::from("process"),
            epoch: 7,
            actor: WorkflowActor {
                kind: "user".to_owned(),
                user: Some(UserId::from("user")),
                agent: None,
                credential_kind: CredentialKind::BrowserSession,
                public_key_fingerprint: None,
                authenticated_without_browser: false,
                scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
            },
            charged_spawns: 1,
        }
    );

    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")),
        })
        .unwrap();
    assert_eq!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 7,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 12,
            stderr_bytes: 0,
            stdout_tail: "build ok".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("artifact")),
            artifact_size_bytes: Some(12),
            result: None,
        })
        .unwrap();

    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            events_recorded: 1
        }
    );
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].node, NodeId::from("node"));
    assert_eq!(events[0].stdout_tail, "build ok");
    assert_eq!(events[0].stderr_tail, "");
    assert!(!events[0].stdout_truncated);

    let oversized_tail = "x".repeat(MAX_TASK_LOG_TAIL_BYTES + 1);
    let oversized_log = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            stdout_bytes: oversized_tail.len() as u64,
            stderr_bytes: 0,
            stdout_tail: oversized_tail.clone(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: false,
        })
        .unwrap_err();
    assert!(oversized_log.to_string().contains("stdout_tail"));

    let oversized_completion = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: oversized_tail.len() as u64,
            stderr_bytes: 0,
            stdout_tail: oversized_tail,
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap_err();
    assert!(oversized_completion.to_string().contains("stdout_tail"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap_err();
    assert!(cross_tenant
        .to_string()
        .contains("outside the virtual process tenant/project scope"));

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("other-process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert!(events.is_empty());
}

#[test]
fn service_revokes_node_credentials_and_live_descriptors() {
    let mut service = CoordinatorService::new(7);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::NodeCredentialRevoked {
        node,
        tenant,
        project,
        actor,
        descriptor_removed,
        queued_assignments_removed,
    } = service
        .handle_request(CoordinatorRequest::RevokeNodeCredential {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            node: "node".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node credential revocation");
    };
    assert_eq!(node, NodeId::from("node"));
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(project, ProjectId::from("project"));
    assert_eq!(actor, UserId::from("user"));
    assert!(descriptor_removed);
    assert_eq!(queued_assignments_removed, 0);

    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")),
        })
        .unwrap_err();
    assert!(heartbeat.to_string().contains("not enrolled"));

    let CoordinatorResponse::NodeDescriptors { descriptors, .. } = service
        .handle_request(CoordinatorRequest::ListNodeDescriptors {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node descriptors");
    };
    assert!(descriptors.is_empty());
}

#[test]
fn duplicate_node_ids_are_isolated_across_identity_replay_liveness_and_revocation() {
    let mut service = CoordinatorService::new(19);
    let node = "shared-node";
    let private_a = test_node_private_key("tenant-a-shared-node");
    let private_b = test_node_private_key("tenant-b-shared-node");
    let private_c = test_node_private_key("tenant-a-other-project-shared-node");
    let public_a = node_ed25519_public_key_from_private_key(&private_a).unwrap();
    let public_b = node_ed25519_public_key_from_private_key(&private_b).unwrap();
    let public_c = node_ed25519_public_key_from_private_key(&private_c).unwrap();

    for (tenant, project, public_key) in [
        ("tenant-a", "project-a", public_a),
        ("tenant-b", "project-b", public_b),
        ("tenant-a", "project-c", public_c),
    ] {
        service
            .handle_request(CoordinatorRequest::AttachNode {
                tenant: tenant.to_owned(),
                project: project.to_owned(),
                node: node.to_owned(),
                public_key,
            })
            .unwrap();
    }
    assert_eq!(
        service
            .coordinator
            .node_identity_count_for_tenant(&TenantId::from("tenant-a")),
        2
    );
    assert_eq!(
        service
            .coordinator
            .node_identity_count_for_tenant(&TenantId::from("tenant-b")),
        1
    );

    for (index, (tenant, project, private_key)) in [
        ("tenant-a", "project-a", private_a.as_str()),
        ("tenant-b", "project-b", private_b.as_str()),
        ("tenant-a", "project-c", private_c.as_str()),
    ]
    .into_iter()
    .enumerate()
    {
        service.set_server_time(100 + index as u64);
        service
            .handle_request(CoordinatorRequest::NodeHeartbeat {
                tenant: tenant.to_owned(),
                project: project.to_owned(),
                node: node.to_owned(),
                node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                    tenant,
                    project,
                    node,
                    private_key,
                    "same-nonce",
                )),
            })
            .unwrap();
        service
            .handle_request(signed_node_request_auto_with_private_key(
                CoordinatorRequest::ReportNodeCapabilities {
                    tenant: tenant.to_owned(),
                    project: project.to_owned(),
                    node: node.to_owned(),
                    capabilities: linux_capabilities(),
                    cached_environment_digests: vec![Digest::sha256(format!(
                        "cache-{tenant}-{project}"
                    ))],
                    dependency_cache_digests: Vec::new(),
                    source_snapshots: Vec::new(),
                    artifact_locations: Vec::new(),
                    direct_connectivity: index != 1,
                    online: true,
                },
                private_key,
            ))
            .unwrap();
    }

    let replay = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: node.to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                "tenant-a",
                "project-a",
                node,
                &private_a,
                "same-nonce",
            )),
        })
        .unwrap_err();
    assert!(replay.to_string().contains("nonce"));

    let forged = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: node.to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                "tenant-b",
                "project-b",
                node,
                &private_a,
                "forged-cross-scope",
            )),
        })
        .unwrap_err();
    assert!(forged.to_string().contains("signature"));

    let scope_a = crate::NodeScopeKey::new(
        TenantId::from("tenant-a"),
        ProjectId::from("project-a"),
        NodeId::from(node),
    );
    let scope_b = crate::NodeScopeKey::new(
        TenantId::from("tenant-b"),
        ProjectId::from("project-b"),
        NodeId::from(node),
    );
    let scope_c = crate::NodeScopeKey::new(
        TenantId::from("tenant-a"),
        ProjectId::from("project-c"),
        NodeId::from(node),
    );
    assert!(service.node_descriptors.contains_key(&scope_a));
    assert!(service.node_descriptors.contains_key(&scope_b));
    assert!(service.node_descriptors.contains_key(&scope_c));
    assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_a));
    assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_b));
    assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_c));
    assert_eq!(service.node_last_seen_epoch_seconds[&scope_a], 100);
    assert_eq!(service.node_last_seen_epoch_seconds[&scope_b], 101);
    assert_eq!(service.node_last_seen_epoch_seconds[&scope_c], 102);
    assert!(service.node_descriptors[&scope_a].direct_connectivity);
    assert!(!service.node_descriptors[&scope_b].direct_connectivity);
    assert!(service.node_descriptors[&scope_c].direct_connectivity);
    assert!(service
        .node_replay_nonces
        .contains_key(&(scope_a.clone(), "same-nonce".to_owned())));
    assert!(service
        .node_replay_nonces
        .contains_key(&(scope_b.clone(), "same-nonce".to_owned())));
    assert!(service
        .node_replay_nonces
        .contains_key(&(scope_c.clone(), "same-nonce".to_owned())));

    service
        .handle_request(CoordinatorRequest::RevokeNodeCredential {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            actor_user: "user-a".to_owned(),
            node: node.to_owned(),
        })
        .unwrap();
    assert!(service
        .coordinator
        .node_identity(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            &NodeId::from(node),
        )
        .is_none());
    assert!(service
        .coordinator
        .node_identity(
            &TenantId::from("tenant-b"),
            &ProjectId::from("project-b"),
            &NodeId::from(node),
        )
        .is_some());
    assert!(!service.node_descriptors.contains_key(&scope_a));
    assert!(service.node_descriptors.contains_key(&scope_b));
    assert!(service.node_descriptors.contains_key(&scope_c));

    service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: node.to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope_with_private_key(
                "tenant-b",
                "project-b",
                node,
                &private_b,
                "post-other-scope-revocation",
            )),
        })
        .expect("revoking tenant A's duplicate node must not affect tenant B");
}

#[test]
fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() {
    let mut service = CoordinatorService::new(7);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );

    let cancelled = service
        .handle_request(CoordinatorRequest::CancelTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        cancelled,
        CoordinatorResponse::TaskCancellationRequested {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            node: NodeId::from("node"),
        }
    );

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: true,
            abort_requested: false,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: Some(TaskTerminalState::Cancelled),
            status_code: None,
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events[0].terminal_state, TaskTerminalState::Cancelled);

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: false,
            abort_requested: false,
        }
    );
}

#[test]
fn service_authorizes_debug_attach_through_public_api() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let CoordinatorResponse::DebugAttach {
        process,
        actor,
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected debug attach authorization");
    };
    assert_eq!(process, ProcessId::from("process"));
    assert_eq!(actor, UserId::from("user"));
    assert!(authorization.allowed);
    assert_eq!(audit_event.operation, "debug_attach");
    assert_eq!(audit_event.actor, UserId::from("user"));
    assert!(audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    let CoordinatorResponse::DebugAttach {
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected denied debug attach authorization");
    };
    assert!(!authorization.allowed);
    assert!(authorization.reason.contains("explicit project permission"));
    assert!(!audit_event.allowed);
    assert_eq!(audit_event.charged_debug_read_bytes, 0);
    assert_eq!(charged_debug_read_bytes, 0);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    let CoordinatorResponse::DebugAttach {
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected cross-tenant debug attach denial");
    };
    assert!(!authorization.allowed);
    assert!(authorization.reason.contains("tenant or project"));
    assert!(!audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, 0);
    assert_eq!(used_debug_read_bytes, 0);
    assert_eq!(service.debug_audit_events.len(), 3);
}

#[test]
fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "compile-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/out.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();
    service.main_runtime.controls.insert(
        process_control_key(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            &ProcessId::from("process"),
        ),
        super::main_runtime::CoordinatorMainControl {
            task_definition: clusterflux_core::TaskDefinitionId::from("completed-main"),
            task_instance: TaskInstanceId::from("ti:process:main"),
            abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
            debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
            state: "completed".to_owned(),
            stopped_probe_symbol: None,
            handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
            launch_id: 1,
        },
    );
    let CoordinatorResponse::DebugBreakpoints {
        revision,
        probe_symbols,
        hit_epoch,
        ..
    } = service
        .handle_request(CoordinatorRequest::SetDebugBreakpoints {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            revision: 1,
            probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()],
        })
        .unwrap()
    else {
        panic!("expected debug breakpoints response");
    };
    assert_eq!(revision, 1);
    assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
    assert_eq!(hit_epoch, None);

    let CoordinatorResponse::DebugBreakpoints {
        revision,
        probe_symbols,
        ..
    } = service
        .handle_request(CoordinatorRequest::SetDebugBreakpoints {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            revision: 0,
            probe_symbols: vec!["clusterflux.probe.stale".to_owned()],
        })
        .unwrap()
    else {
        panic!("expected stale breakpoint response");
    };
    assert_eq!(revision, 1);
    assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);

    let CoordinatorResponse::DebugProbeHit {
        breakpoint_matched,
        debug_epoch,
        probe_symbol,
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugProbeHit {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            probe_symbol: "clusterflux.probe.compile_linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected executing Wasm probe hit response");
    };
    assert!(breakpoint_matched);
    assert_eq!(debug_epoch, Some(1));
    assert_eq!(probe_symbol, "clusterflux.probe.compile_linux");

    let CoordinatorResponse::DebugBreakpoints {
        hit_epoch,
        hit_task,
        hit_probe_symbol,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugBreakpoints {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected breakpoint hit status");
    };
    assert_eq!(hit_epoch, Some(1));
    assert_eq!(hit_task, Some(TaskInstanceId::from("compile-linux")));
    assert_eq!(
        hit_probe_symbol.as_deref(),
        Some("clusterflux.probe.compile_linux")
    );

    let CoordinatorResponse::DebugCommand {
        epoch: Some(1),
        command: Some(command),
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed node poll to receive freeze command");
    };
    assert_eq!(command, "freeze");

    let CoordinatorResponse::DebugCommand {
        epoch: None,
        command: None,
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("debug commands should be consumed after poll");
    };

    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        acknowledgements,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected pending debug epoch status");
    };
    assert!(!fully_frozen);
    assert!(!fully_resumed);
    assert!(acknowledgements.is_empty());
    let early_resume = service
        .handle_request(CoordinatorRequest::ResumeDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap_err();
    assert!(early_resume
        .to_string()
        .contains("no settled frozen participant set"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            epoch: 1,
            state: DebugAcknowledgementState::Frozen,
            stack_frames: vec!["compile_linux::wasm".to_owned()],
            local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())],
            task_args: vec![("target".to_owned(), "linux".to_owned())],
            handles: vec![("artifact".to_owned(), "pending".to_owned())],
            command_status: Some("frozen at Wasm safepoint".to_owned()),
            recent_output: vec![],
            message: None,
        })
        .unwrap();
    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        acknowledgements,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected frozen debug epoch status");
    };
    assert!(fully_frozen);
    assert!(!fully_resumed);
    assert_eq!(acknowledgements.len(), 1);
    assert_eq!(acknowledgements[0].state, DebugAcknowledgementState::Frozen);

    let CoordinatorResponse::DebugEpoch {
        epoch,
        command,
        affected_tasks,
        all_stop_requested,
        audit_event,
        ..
    } = service
        .handle_request(CoordinatorRequest::ResumeDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected debug epoch resume response");
    };
    assert_eq!(epoch, 1);
    assert_eq!(command, "resume");
    assert!(!all_stop_requested);
    assert_eq!(affected_tasks.len(), 1);
    assert_eq!(audit_event.operation, "resume_debug_epoch");

    let CoordinatorResponse::DebugCommand {
        epoch: Some(1),
        command: Some(command),
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed node poll to receive resume command");
    };
    assert_eq!(command, "resume");

    let CoordinatorResponse::DebugEpochStatus { fully_resumed, .. } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected pending resume status");
    };
    assert!(!fully_resumed);
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            epoch: 1,
            state: DebugAcknowledgementState::Running,
            stack_frames: vec![],
            local_values: vec![],
            task_args: vec![],
            handles: vec![],
            command_status: Some("running".to_owned()),
            recent_output: vec![],
            message: None,
        })
        .unwrap();
    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected resumed debug epoch status");
    };
    assert!(!fully_frozen);
    assert!(fully_resumed);
}

#[test]
fn service_reports_task_restart_boundary_through_public_api() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let denied = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap_err();
    assert!(denied.to_string().contains("task restart denied"));

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        restarted_task_instance,
        message,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected task restart response");
    };
    assert!(!accepted);
    assert!(!clean_boundary_available);
    assert!(!active_task);
    assert!(!completed_event_observed);
    assert!(restarted_task_instance.is_none());
    assert!(requires_whole_process_restart);
    assert!(message.contains("not known"));
    assert_eq!(audit_event.operation, "restart_task");
    assert_eq!(audit_event.task, Some(TaskInstanceId::from("task")));
    assert!(audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "task",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(initial_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected initial task assignment");
    };
    assert_eq!(initial_assignment.task, TaskInstanceId::from("task"));

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        message,
        audit_event,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected active task restart response");
    };
    assert!(!accepted);
    assert!(!clean_boundary_available);
    assert!(active_task);
    assert!(!completed_event_observed);
    assert!(requires_whole_process_restart);
    assert!(message.contains("still active"));
    assert_eq!(
        audit_event.charged_debug_read_bytes,
        DEBUG_CONTROL_READ_BYTES
    );
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 2);

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker-linux".to_owned(),
            task: "task".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        restarted_task_instance,
        restarted_attempt_id,
        message,
        audit_event,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected completed task restart response");
    };
    assert!(accepted);
    assert!(clean_boundary_available);
    assert!(!active_task);
    assert!(completed_event_observed);
    assert!(!requires_whole_process_restart);
    let restarted_task_instance = restarted_task_instance.expect("restart returns logical id");
    let restarted_attempt_id = restarted_attempt_id.expect("restart returns a new attempt id");
    assert!(restarted_attempt_id.starts_with("ta_"));
    assert!(message.contains("from clean VFS entry boundary epoch 7"));
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(restarted_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected restarted task assignment");
    };
    assert_eq!(restarted_assignment.task, restarted_task_instance);
    assert_eq!(restarted_assignment.task, TaskInstanceId::from("task"));
    let mut expected_task_spec = initial_assignment.task_spec.clone();
    expected_task_spec.task_instance = restarted_task_instance;
    assert_eq!(restarted_assignment.task_spec, expected_task_spec);
    assert_eq!(
        restarted_assignment.wasm_module_base64,
        initial_assignment.wasm_module_base64
    );
    assert_eq!(
        audit_event.charged_debug_read_bytes,
        DEBUG_CONTROL_READ_BYTES
    );
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 3);
    assert_eq!(service.debug_audit_events.len(), 4);
}

#[test]
fn service_cancels_whole_process_and_blocks_new_task_launches() {
    let mut service = CoordinatorService::new(7);
    for node in ["node-a", "node-b"] {
        let capabilities = if node == "node-a" {
            linux_capabilities()
        } else {
            windows_capabilities()
        };
        service
            .handle_request(CoordinatorRequest::AttachNode {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                public_key: test_node_public_key(node),
            })
            .unwrap();
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                capabilities,
                cached_environment_digests: vec![],
                dependency_cache_digests: vec![],
                source_snapshots: vec![],
                artifact_locations: vec![],
                direct_connectivity: true,
                online: true,
            })
            .unwrap();
    }
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    for node in ["node-a", "node-b"] {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                process: "process".to_owned(),
                epoch: 7,
            })
            .unwrap();
    }

    for (task, required_capability, preferred_node) in [
        ("compile-linux", Capability::Containers, "node-a"),
        ("link-windows", Capability::WindowsCommandDev, "node-b"),
    ] {
        let response = service
            .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
                task_spec: test_task_spec(
                    "tenant",
                    "project",
                    "process",
                    task,
                    7,
                    [required_capability],
                ),
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: Some("user".to_owned()),
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                wait_for_node: false,
                artifact_path: format!("/vfs/artifacts/{task}.txt"),
                wasm_module_base64: test_wasm_module_base64(),
            })
            .unwrap();
        let CoordinatorResponse::TaskLaunched { assignment, .. } = response else {
            panic!("expected task launch");
        };
        assert_eq!(assignment.node, NodeId::from(preferred_node));
    }

    let CoordinatorResponse::ProcessCancellationRequested {
        process,
        cancelled_tasks,
        affected_nodes,
    } = service
        .handle_request(CoordinatorRequest::CancelProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected process cancellation");
    };
    assert_eq!(process, ProcessId::from("process"));
    assert_eq!(cancelled_tasks.len(), 2);
    assert_eq!(
        affected_nodes,
        vec![NodeId::from("node-a"), NodeId::from("node-b")]
    );

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node-a".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: true,
            abort_requested: false,
        }
    );

    let blocked = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "package-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/package-linux.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(blocked
        .to_string()
        .contains("virtual process is cancelling"));

    service
        .handle_request(CoordinatorRequest::AbortProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap();
    let abort_control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node-a".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        abort_control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: false,
            abort_requested: true,
        }
    );
}

#[test]
fn service_rejects_second_active_process_unless_restarting_same_process() {
    let mut service = CoordinatorService::new(7);
    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: None,
            ..
        }
    ));

    let same_without_restart = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(same_without_restart
        .to_string()
        .contains("already has active virtual process"));

    let other_process = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-b".to_owned(),
            restart: true,
        })
        .unwrap_err();
    assert!(other_process
        .to_string()
        .contains("already has active virtual process"));

    let wrong_attempt_abort = service
        .handle_request(CoordinatorRequest::AbortProcess {
            launch_attempt: Some("attempt-b".to_owned()),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process-a".to_owned(),
        })
        .unwrap_err();
    assert!(wrong_attempt_abort
        .to_string()
        .contains("does not own process process-a"));
    assert!(service
        .coordinator
        .active_process(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            &ProcessId::from("process-a")
        )
        .is_some());

    let retired_main_abort = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
    let process_key = process_control_key(
        &TenantId::from("tenant"),
        &ProjectId::from("project"),
        &ProcessId::from("process-a"),
    );
    service.main_runtime.controls.insert(
        process_key.clone(),
        super::main_runtime::CoordinatorMainControl {
            task_definition: clusterflux_core::TaskDefinitionId::from("build"),
            task_instance: TaskInstanceId::from("ti:process-a:main"),
            abort: std::sync::Arc::clone(&retired_main_abort),
            debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
            state: "running".to_owned(),
            stopped_probe_symbol: None,
            handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
            launch_id: 1,
        },
    );
    let restarted = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: Some("attempt-a-restart".to_owned()),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: true,
        })
        .unwrap();
    assert_eq!(
        restarted,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: Some("attempt-a-restart".to_owned()),
            process: ProcessId::from("process-a"),
            epoch: 7,
            actor: WorkflowActor {
                kind: "user".to_owned(),
                user: Some(UserId::from("user")),
                agent: None,
                credential_kind: CredentialKind::BrowserSession,
                public_key_fingerprint: None,
                authenticated_without_browser: false,
                scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
            },
            charged_spawns: 2,
        }
    );
    assert!(retired_main_abort.load(std::sync::atomic::Ordering::Acquire));
    assert!(!service.main_runtime.controls.contains_key(&process_key));
}

#[test]
fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() {
    let mut service = CoordinatorService::new(31);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("process-main-before-child");
    let child = TaskInstanceId::from("child-active");
    let process_key = process_control_key(&tenant, &project, &process);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: tenant.to_string(),
            project: project.to_string(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: process.to_string(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: tenant.to_string(),
            project: project.to_string(),
            node: "worker".to_owned(),
            process: process.to_string(),
            epoch: 31,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        tenant.as_str(),
        project.as_str(),
        process.as_str(),
        "worker",
        "child-definition",
        child.as_str(),
        31,
    );

    let main = TaskInstanceId::from("main-instance");
    service.main_runtime.controls.insert(
        process_key.clone(),
        super::main_runtime::CoordinatorMainControl {
            task_definition: TaskDefinitionId::from("build"),
            task_instance: main.clone(),
            abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
            debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
            state: "running".to_owned(),
            stopped_probe_symbol: None,
            handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
            launch_id: 1,
        },
    );
    service.debug_epochs.insert(process_key.clone(), 9);

    service.record_coordinator_main_completion(
        super::main_runtime::MainScope {
            tenant: tenant.clone(),
            project: project.clone(),
            process: process.clone(),
            task_definition: TaskDefinitionId::from("build"),
            task_instance: main,
            epoch: 31,
            launch_id: 1,
        },
        Ok(WasmTaskResult::completed(
            TaskInstanceId::from("main-instance"),
            TaskBoundaryValue::SmallJson(json!("main completed")),
        )),
    );

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_some());
    assert!(service.debug_epochs.contains_key(&process_key));
    assert!(service
        .active_tasks
        .iter()
        .any(|(_, _, retained_process, _, task)| {
            retained_process == &process && task == &child
        }));
    let blocked_next = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-too-early".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(blocked_next
        .to_string()
        .contains("already has active virtual process"));

    let artifact_bytes = b"child artifact survives terminal cleanup";
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: tenant.to_string(),
            project: project.to_string(),
            process: process.to_string(),
            node: "worker".to_owned(),
            task: child.to_string(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: artifact_bytes.len() as u64,
            stderr_bytes: 0,
            stdout_tail: "child completed".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/child-output".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))),
        })
        .unwrap();

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_none());
    assert!(!service.debug_epochs.contains_key(&process_key));
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: "user".to_owned(),
            process: Some(process.to_string()),
        })
        .unwrap()
    else {
        panic!("expected retained task events");
    };
    assert_eq!(events.len(), 2);
    assert!(events.iter().any(|event| {
        event.executor == TaskExecutor::CoordinatorMain
            && event.terminal_state == TaskTerminalState::Completed
    }));
    assert!(events.iter().any(|event| {
        event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes))
    }));
    let metadata = service
        .artifact_registry
        .metadata(&tenant, &project, &ArtifactId::from("child-output"))
        .expect("artifact metadata must survive terminal cleanup");
    assert_eq!(metadata.process, process);
    assert_eq!(metadata.digest, Digest::sha256(artifact_bytes));

    let next = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-after-cleanup".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. }));
}

#[test]
fn completed_main_terminal_matrix_retires_after_failed_or_cancelled_final_child() {
    for terminal_state in [TaskTerminalState::Failed, TaskTerminalState::Cancelled] {
        let mut service = service_with_completed_main_and_final_child(
            clusterflux_core::TaskFailurePolicy::FailFast,
        );
        complete_terminal_matrix_child(&mut service, terminal_state.clone());

        let tenant = TenantId::from("tenant");
        let project = ProjectId::from("project");
        let process = ProcessId::from("terminal-matrix");
        assert!(
            service
                .coordinator
                .active_process(&tenant, &project, &process)
                .is_none(),
            "{terminal_state:?} final child left the process slot active"
        );
        assert!(!service
            .debug_epochs
            .contains_key(&process_control_key(&tenant, &project, &process)));
        assert!(!service
            .debug_breakpoints
            .contains_key(&process_control_key(&tenant, &project, &process)));
        assert!(service.debug_commands.keys().all(
            |(task_tenant, task_project, task_process, _, _)| {
                task_tenant != &tenant || task_project != &project || task_process != &process
            }
        ));
        assert!(!service
            .panel_snapshots
            .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process)));
        assert!(service.active_tasks.iter().all(
            |(task_tenant, task_project, task_process, _, _)| {
                task_tenant != &tenant || task_project != &project || task_process != &process
            }
        ));
        assert!(!service
            .main_runtime
            .controls
            .contains_key(&process_control_key(&tenant, &project, &process)));
        let join = service.task_join_result(
            tenant.clone(),
            project.clone(),
            process.clone(),
            TaskInstanceId::from("final-child"),
        );
        assert_eq!(
            join.state,
            match terminal_state {
                TaskTerminalState::Failed => TaskJoinState::Failed,
                TaskTerminalState::Cancelled => TaskJoinState::Cancelled,
                TaskTerminalState::Completed => unreachable!(),
            }
        );
        service
            .handle_request(CoordinatorRequest::StartProcess {
                launch_attempt: None,
                tenant: tenant.to_string(),
                project: project.to_string(),
                actor_user: None,
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                process: "next-after-terminal".to_owned(),
                restart: false,
            })
            .expect("the terminal outcome must release the one-process project slot");
    }
}

#[test]
fn completed_main_unpolled_final_assignment_completion_retires_process() {
    let mut service =
        service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("terminal-matrix");
    let node = NodeId::from("worker");
    let assignment_key = (tenant.clone(), project.clone(), node);
    assert_eq!(
        service
            .task_assignments
            .get(&assignment_key)
            .map_or(0, VecDeque::len),
        1
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: tenant.to_string(),
            project: project.to_string(),
            process: process.to_string(),
            node: "worker".to_owned(),
            task: "final-child".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 2,
            stderr_bytes: 0,
            stdout_tail: "42".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: Some(TaskBoundaryValue::SmallJson(json!(42))),
        })
        .unwrap();

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_none());
    assert!(!service.task_assignments.contains_key(&assignment_key));
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "next-after-unpolled-completion".to_owned(),
            restart: false,
        })
        .expect("unpolled terminal completion must release the one-process slot");
}

#[test]
fn completed_main_retires_from_authoritative_state_after_event_history_rotates() {
    let mut service =
        service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("terminal-matrix");

    for index in 0..=MAX_TASK_EVENTS_PER_PROCESS {
        service.record_task_completion_event(TaskCompletionEvent {
            tenant: tenant.clone(),
            project: project.clone(),
            process: process.clone(),
            node: NodeId::from("worker"),
            executor: TaskExecutor::Node,
            task_definition: TaskDefinitionId::from("historical"),
            task: TaskInstanceId::new(format!("historical-{index}")),
            attempt_id: None,
            placement: None,
            terminal_state: TaskTerminalState::Completed,
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        });
    }
    assert!(
        service.task_events.iter().all(|event| {
            event.process != process || event.executor != TaskExecutor::CoordinatorMain
        }),
        "the regression requires bounded history to have rotated the main event"
    );

    complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed);

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_none());
    let summary = service
        .process_summaries
        .get(&process_control_key(&tenant, &project, &process))
        .expect("the terminal process summary must remain authoritative");
    assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed));
    assert_eq!(
        summary.main_terminal_state,
        Some(TaskTerminalState::Completed)
    );
}

#[test]
fn completed_main_await_operator_blocks_retirement_until_each_resolution() {
    for resolution in [
        TaskFailureResolution::AcceptFailure,
        TaskFailureResolution::Cancel,
    ] {
        let mut service = service_with_completed_main_and_final_child(
            clusterflux_core::TaskFailurePolicy::AwaitOperator,
        );
        complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed);

        let tenant = TenantId::from("tenant");
        let project = ProjectId::from("project");
        let process = ProcessId::from("terminal-matrix");
        let process_key = process_control_key(&tenant, &project, &process);
        assert!(service
            .coordinator
            .active_process(&tenant, &project, &process)
            .is_some());
        assert!(service.debug_epochs.contains_key(&process_key));
        assert!(service.debug_breakpoints.contains_key(&process_key));
        assert!(service
            .panel_snapshots
            .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process)));
        let attempt = service
            .task_attempts
            .get(&super::keys::task_restart_key(
                &tenant,
                &project,
                &process,
                &TaskInstanceId::from("final-child"),
            ))
            .and_then(|attempts| attempts.iter().rev().find(|attempt| attempt.current))
            .unwrap();
        assert_eq!(attempt.state, TaskAttemptState::FailedAwaitingAction);

        service
            .handle_request(CoordinatorRequest::ResolveTaskFailure {
                tenant: tenant.to_string(),
                project: project.to_string(),
                actor_user: "user".to_owned(),
                process: process.to_string(),
                task: "final-child".to_owned(),
                resolution,
            })
            .unwrap();

        assert!(
            service
                .coordinator
                .active_process(&tenant, &project, &process)
                .is_none(),
            "{resolution:?} left the process slot active"
        );
        assert!(!service.debug_epochs.contains_key(&process_key));
        assert!(!service.debug_breakpoints.contains_key(&process_key));
        assert!(!service
            .panel_snapshots
            .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process)));
        assert!(service.debug_commands.keys().all(
            |(task_tenant, task_project, task_process, _, _)| {
                task_tenant != &tenant || task_project != &project || task_process != &process
            }
        ));
        let join = service.task_join_result(
            tenant.clone(),
            project.clone(),
            process.clone(),
            TaskInstanceId::from("final-child"),
        );
        assert_eq!(
            join.state,
            match resolution {
                TaskFailureResolution::AcceptFailure => TaskJoinState::Failed,
                TaskFailureResolution::Cancel => TaskJoinState::Cancelled,
            }
        );
        service
            .handle_request(CoordinatorRequest::StartProcess {
                launch_attempt: None,
                tenant: tenant.to_string(),
                project: project.to_string(),
                actor_user: None,
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                process: "next-after-resolution".to_owned(),
                restart: false,
            })
            .expect("operator resolution must release the one-process project slot");
    }
}

#[test]
fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() {
    let mut service = service_with_completed_main_and_final_child(
        clusterflux_core::TaskFailurePolicy::AwaitOperator,
    );
    complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("terminal-matrix");
    let task = TaskInstanceId::from("final-child");

    let CoordinatorResponse::TaskRestart { accepted, .. } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: tenant.to_string(),
            project: project.to_string(),
            actor_user: "user".to_owned(),
            process: process.to_string(),
            task: task.to_string(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected task restart");
    };
    assert!(accepted);

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: tenant.to_string(),
            project: project.to_string(),
            process: process.to_string(),
            node: "worker".to_owned(),
            task: task.to_string(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 2,
            stderr_bytes: 0,
            stdout_tail: "ok".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: Some(TaskBoundaryValue::SmallJson(json!("ok"))),
        })
        .unwrap();

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_none());
    let attempts = service
        .task_attempts
        .get(&super::keys::task_restart_key(
            &tenant, &project, &process, &task,
        ))
        .unwrap();
    assert!(
        attempts
            .iter()
            .any(|attempt| !attempt.current
                && attempt.state == TaskAttemptState::FailedAwaitingAction)
    );
    assert!(attempts
        .iter()
        .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed));
    assert_eq!(
        service
            .process_summaries
            .get(&process_control_key(&tenant, &project, &process))
            .and_then(|summary| summary.final_result.clone()),
        Some(ProcessFinalResult::Completed),
        "a successful current retry must override the superseded failed attempt"
    );
    assert_eq!(
        service
            .task_join_result(tenant, project, process, task)
            .state,
        TaskJoinState::Completed
    );
}

#[test]
fn completed_main_failed_child_does_not_abort_another_active_child() {
    let mut service =
        service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast);
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "terminal-matrix",
        "worker",
        "other-child-definition",
        "other-child",
        83,
    );

    complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed);
    let tenant = TenantId::from("tenant");
    let project = ProjectId::from("project");
    let process = ProcessId::from("terminal-matrix");
    let other_key = task_control_key(
        &tenant,
        &project,
        &process,
        &NodeId::from("worker"),
        &TaskInstanceId::from("other-child"),
    );
    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_some());
    assert!(service.active_tasks.contains(&other_key));
    assert!(!service.task_aborts.contains(&other_key));
    assert_eq!(
        service
            .task_join_result(
                tenant.clone(),
                project.clone(),
                process.clone(),
                TaskInstanceId::from("final-child"),
            )
            .state,
        TaskJoinState::Failed
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: tenant.to_string(),
            project: project.to_string(),
            process: process.to_string(),
            node: "worker".to_owned(),
            task: "other-child".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();

    assert!(service
        .coordinator
        .active_process(&tenant, &project, &process)
        .is_none());
    assert!(!service.active_tasks.contains(&other_key));
}

#[test]
fn quiescent_cooperative_cancel_releases_slot_immediately() {
    let mut service = CoordinatorService::new(17);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    service.record_task_completion_event(TaskCompletionEvent {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("process-a"),
        node: NodeId::from("node"),
        executor: TaskExecutor::Node,
        task_definition: clusterflux_core::TaskDefinitionId::from("old-task"),
        task: TaskInstanceId::from("ti:process-a:old"),
        attempt_id: None,
        placement: None,
        terminal_state: TaskTerminalState::Completed,
        status_code: Some(0),
        stdout_bytes: 0,
        stderr_bytes: 0,
        stdout_tail: String::new(),
        stderr_tail: String::new(),
        stdout_truncated: false,
        stderr_truncated: false,
        artifact_path: None,
        artifact_digest: None,
        artifact_size_bytes: None,
        result: Some(TaskBoundaryValue::SmallJson(json!("old"))),
    });
    service
        .record_debug_audit_event(
            TenantId::from("tenant"),
            ProjectId::from("project"),
            ProcessId::from("process-a"),
            None,
            UserId::from("user"),
            "old_debug_event",
            false,
            "old process incarnation",
        )
        .unwrap();

    service
        .handle_request(CoordinatorRequest::CancelProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process-a".to_owned(),
        })
        .unwrap();

    let CoordinatorResponse::ProcessStatuses { processes, .. } = service
        .handle_request(CoordinatorRequest::ListProcesses {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected live process statuses");
    };
    assert!(processes.is_empty());

    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted {
            launch_attempt: None,
            ..
        }
    ));
    assert!(service.task_events.iter().all(|event| {
        event.tenant != TenantId::from("tenant")
            || event.project != ProjectId::from("project")
            || event.process != ProcessId::from("process-a")
    }));
    assert!(service.debug_audit_events.iter().all(|event| {
        event.tenant != TenantId::from("tenant")
            || event.project != ProjectId::from("project")
            || event.process != ProcessId::from("process-a")
    }));
}

#[test]
fn aborted_process_accepts_signed_terminal_event_for_issued_task() {
    let mut service = CoordinatorService::new(17);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 17,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile",
        "compile-1",
        17,
    );

    let CoordinatorResponse::ProcessAborted { aborted_tasks, .. } = service
        .handle_request(CoordinatorRequest::AbortProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected process abort");
    };
    assert_eq!(aborted_tasks.len(), 1);

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-1".to_owned(),
            terminal_state: Some(TaskTerminalState::Cancelled),
            status_code: None,
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-1"),
            events_recorded: 1,
        }
    );
    assert_eq!(
        service.task_events[0].task_definition,
        clusterflux_core::TaskDefinitionId::from("compile")
    );
    assert!(service.task_restart_checkpoints.is_empty());
}

#[test]
fn service_download_links_are_scoped_and_streaming_is_metered() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = b"0123456789abcdef0123456789abcdef";
    service.set_server_time(10);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 32,
            stderr_bytes: 0,
            stdout_tail: "artifact bytes".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 31)]),
    });
    let quota_denied_link = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(quota_denied_link
        .to_string()
        .contains("ArtifactDownloadBytes"));
    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 64)]),
    });

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    let private_node_link = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap();
    assert!(matches!(
        private_node_link,
        CoordinatorResponse::ArtifactDownloadLink { .. }
    ));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected download link");
    };

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    let private_node_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap();
    assert!(matches!(
        private_node_open,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    let CoordinatorResponse::ArtifactTransferAssignment {
        transfer: Some(pending_transfer),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected reverse transfer assignment");
    };
    let bad_chunk = service
        .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            transfer_id: pending_transfer.transfer_id,
            artifact: pending_transfer.artifact.as_str().to_owned(),
            offset: 0,
            content_base64: BASE64_STANDARD.encode(artifact_bytes),
            chunk_digest: Digest::sha256("wrong bytes"),
            eof: true,
        })
        .unwrap_err();
    assert!(bad_chunk.to_string().contains("chunk digest mismatch"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    assert_eq!(link.tenant, TenantId::from("tenant"));
    assert_eq!(link.project, ProjectId::from("project"));
    assert_eq!(link.process, ProcessId::from("process"));
    assert_eq!(link.actor, Actor::User(UserId::from("user")));
    assert_eq!(link.max_bytes, 64);
    assert!(link.policy_context_digest.is_valid_sha256());
    assert_eq!(link.expires_at_epoch_seconds, 70);
    assert!(link.url_path.contains("/artifacts/tenant/project/process"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("does not exist"));

    let cross_project = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(cross_project.to_string().contains("does not exist"));

    let cross_tenant_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_tenant_open.to_string().contains("does not exist"));

    let cross_project_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_project_open.to_string().contains("does not exist"));

    let guessed = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: Digest::sha256("guessed"),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(guessed.to_string().contains("token is invalid"));

    let cross_actor_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_actor_open.to_string().contains("token is invalid"));

    service.set_server_time(11);
    upload_pending_artifact_transfer(&mut service, "node", artifact_bytes);
    let CoordinatorResponse::ArtifactDownloadStream {
        streamed_bytes,
        charged_download_bytes,
        content_base64: Some(content_base64),
        content_source: Some(content_source),
        ..
    } = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap()
    else {
        panic!("expected download stream");
    };

    assert_eq!(streamed_bytes, 16);
    assert_eq!(charged_download_bytes, 16);
    assert_eq!(
        BASE64_STANDARD.decode(content_base64).unwrap(),
        &artifact_bytes[..16]
    );
    assert_eq!(content_source, "retaining_node_reverse_stream");

    let CoordinatorResponse::ArtifactDownloadLink {
        link: expiring_link,
    } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 1,
        })
        .unwrap()
    else {
        panic!("expected expiring download link");
    };
    service.set_server_time(13);
    let expired = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: expiring_link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(expired.to_string().contains("expired"));
    service.set_server_time(11);

    let cross_actor_revoke = service
        .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            artifact: "app.txt".to_owned(),
            token_digest: link.scoped_token_digest.clone(),
        })
        .unwrap_err();
    assert!(cross_actor_revoke.to_string().contains("token is invalid"));

    let CoordinatorResponse::ArtifactDownloadLinkRevoked { link: revoked } = service
        .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            token_digest: link.scoped_token_digest.clone(),
        })
        .unwrap()
    else {
        panic!("expected revoked download link");
    };
    assert_eq!(revoked.scoped_token_digest, link.scoped_token_digest);

    let revoked = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(revoked.to_string().contains("revoked"));
}

#[test]
fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = b"0123456789abcdef";
    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 16)]),
    });
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 16,
            stderr_bytes: 0,
            stdout_tail: "artifact bytes".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected initial download link");
    };

    let pending = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap();
    assert!(matches!(
        pending,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    upload_pending_artifact_transfer(&mut service, "node", artifact_bytes);
    let CoordinatorResponse::ArtifactDownloadStream {
        charged_download_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap()
    else {
        panic!("expected initial download stream");
    };
    assert_eq!(charged_download_bytes, 16);

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    let after_reconnect = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(after_reconnect
        .to_string()
        .contains("ArtifactDownloadBytes"));

    let restarted_cli_retry = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(restarted_cli_retry
        .to_string()
        .contains("ArtifactDownloadBytes"));

    let cross_tenant_retry = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_tenant_retry.to_string().contains("does not exist"));

    let cross_project_retry = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_project_retry.to_string().contains("does not exist"));
}

#[test]
fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = vec![0x5a; 600_000];
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "package",
        "package",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "package".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/large.bin".to_owned()),
            artifact_digest: Some(Digest::sha256(&artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: Some(TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("large.bin"),
                digest: Digest::sha256(&artifact_bytes),
                size_bytes: artifact_bytes.len() as u64,
            })),
        })
        .unwrap();
    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "large.bin".to_owned(),
            max_bytes: artifact_bytes.len() as u64,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected download link");
    };
    let pending = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "large.bin".to_owned(),
            max_bytes: artifact_bytes.len() as u64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: artifact_bytes.len() as u64,
        })
        .unwrap();
    assert!(matches!(
        pending,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    upload_pending_artifact_transfer(&mut service, "node", &artifact_bytes);

    let mut downloaded = Vec::new();
    loop {
        let CoordinatorResponse::ArtifactDownloadStream {
            streamed_bytes,
            content_offset: Some(offset),
            content_eof,
            content_base64: Some(content),
            ..
        } = service
            .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
                artifact: "large.bin".to_owned(),
                max_bytes: artifact_bytes.len() as u64,
                token_digest: link.scoped_token_digest.clone(),
                chunk_bytes: artifact_bytes.len() as u64,
            })
            .unwrap()
        else {
            panic!("expected verified reverse-stream content chunk");
        };
        assert_eq!(offset, downloaded.len() as u64);
        assert!(streamed_bytes <= super::artifacts::MAX_ARTIFACT_REVERSE_CHUNK_BYTES);
        downloaded.extend(BASE64_STANDARD.decode(content).unwrap());
        if content_eof {
            break;
        }
    }
    assert_eq!(downloaded, artifact_bytes);
    assert_eq!(
        service.quota.used_download_bytes(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            service.current_epoch_seconds().unwrap(),
        ),
        600_000
    );
}

#[test]
fn windows_task_events_share_the_virtual_process_scope() {
    let mut service = CoordinatorService::new(7);
    service.record_task_completion_event(TaskCompletionEvent {
        tenant: TenantId::from("other-tenant"),
        project: ProjectId::from("other-project"),
        process: ProcessId::from("other-process"),
        node: NodeId::from("other-node"),
        executor: super::TaskExecutor::Node,
        task_definition: TaskDefinitionId::from("other-task"),
        task: TaskInstanceId::from("other-task"),
        attempt_id: None,
        placement: None,
        terminal_state: TaskTerminalState::Completed,
        status_code: Some(0),
        stdout_bytes: 0,
        stderr_bytes: 0,
        stdout_tail: String::new(),
        stderr_tail: String::new(),
        stdout_truncated: false,
        stderr_truncated: false,
        artifact_path: None,
        artifact_digest: None,
        artifact_size_bytes: None,
        result: None,
    });
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "windows-node".to_owned(),
            public_key: test_node_public_key("windows-node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "windows-node".to_owned(),
            capabilities: windows_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env-windows-command-dev")],
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "windows-node",
        "windows-command-dev",
        "windows-command-dev",
        7,
    );

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "windows-node".to_owned(),
            task: "windows-command-dev".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 24,
            stderr_bytes: 0,
            stdout_tail: "windows output".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/windows-output.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("windows-artifact")),
            artifact_size_bytes: Some(24),
            result: None,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("windows-command-dev"),
            events_recorded: 1,
        }
    );

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].process, ProcessId::from("process"));
    assert_eq!(events[0].node, NodeId::from("windows-node"));
    assert_eq!(events[0].task, TaskInstanceId::from("windows-command-dev"));
    assert_eq!(
        events[0].artifact_path,
        Some(VfsPath::new("/vfs/artifacts/windows-output.txt").unwrap())
    );
}

#[test]
fn service_schedules_task_across_reported_node_descriptors() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "other-tenant".to_owned(),
            project: "other-project".to_owned(),
            node: "other-node".to_owned(),
            public_key: test_node_public_key("other-node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "other-tenant".to_owned(),
            project: "other-project".to_owned(),
            node: "other-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    for node in ["cold-node", "warm-node"] {
        service
            .handle_request(CoordinatorRequest::AttachNode {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                public_key: test_node_public_key(node),
            })
            .unwrap();
    }

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "cold-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::NodeCapabilitiesRecorded {
            node: NodeId::from("cold-node"),
            node_descriptors: 1,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "warm-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env")],
            dependency_cache_digests: vec![Digest::sha256("deps")],
            source_snapshots: vec![Digest::sha256("source")],
            artifact_locations: vec!["build-output".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service
        .handle_request(CoordinatorRequest::ListNodeDescriptors {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "operator".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node descriptor inspector state");
    };
    assert_eq!(actor, UserId::from("operator"));
    assert_eq!(descriptors.len(), 2);
    let warm = descriptors
        .iter()
        .find(|descriptor| descriptor.id == NodeId::from("warm-node"))
        .expect("warm node descriptor is visible to inspector state");
    assert!(warm
        .capabilities
        .capabilities
        .contains(&Capability::Command));
    assert!(warm.cached_environments.contains(&Digest::sha256("env")));
    assert!(warm.dependency_caches.contains(&Digest::sha256("deps")));
    assert!(warm.source_snapshots.contains(&Digest::sha256("source")));
    assert!(warm
        .artifact_locations
        .contains(&ArtifactId::from("build-output")));

    let CoordinatorResponse::TaskPlacement { placement } = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: Some(EnvironmentRequirements::linux_container()),
            environment_digest: Some(Digest::sha256("env")),
            required_capabilities: vec![Capability::Command],
            dependency_cache: Some(Digest::sha256("deps")),
            source_snapshot: Some(Digest::sha256("source")),
            required_artifacts: vec!["build-output".to_owned()],
            prefer_node: None,
        })
        .unwrap()
    else {
        panic!("expected task placement");
    };
    assert_eq!(placement.node, NodeId::from("warm-node"));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("environment")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("source")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("dependency")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("artifact")));

    let error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::WindowsCommandDev],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(error.to_string().contains("WindowsCommandDev"));

    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 0)]),
    });
    let quota_error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::Command],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(quota_error
        .to_string()
        .contains("quota unavailable for placement"));

    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 10)]),
    });
    service.admission.workflow_placement_allowed = false;
    let policy_error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::Command],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(policy_error.to_string().contains("policy denied placement"));
}

#[test]
fn coordinator_side_task_launch_queues_worker_assignment() {
    let mut service = CoordinatorService::new(9);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env")],
            dependency_cache_digests: vec![Digest::sha256("deps")],
            source_snapshots: vec![Digest::sha256("source")],
            artifact_locations: vec!["bootstrap-artifact".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        epoch,
        ..
    } = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-control".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected coordinator-side process start");
    };
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            process: "vp-control".to_owned(),
            epoch,
        })
        .unwrap();
    service.artifact_registry.flush_metadata(ArtifactFlush {
        id: ArtifactId::from("bootstrap-artifact"),
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("vp-control"),
        producer_task: TaskInstanceId::from("bootstrap"),
        retaining_node: NodeId::from("worker-linux"),
        digest: Digest::sha256("bootstrap"),
        size: 9,
    });

    let submitted_task_spec = TaskSpec {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("vp-control"),
        task_definition: clusterflux_core::TaskDefinitionId::from("compile-linux"),
        task_instance: clusterflux_core::TaskInstanceId::from("compile-linux-1"),
        dispatch: TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile-linux".to_owned()),
            abi: WasmExportAbi::TaskV1,
        },
        environment_id: Some("linux".to_owned()),
        environment: None,
        environment_digest: Some(Digest::sha256("env")),
        required_capabilities: BTreeSet::from([Capability::Command]),
        dependency_cache: Some(Digest::sha256("deps")),
        source_snapshot: Some(Digest::sha256("source")),
        required_artifacts: vec![ArtifactId::from("bootstrap-artifact")],
        args: vec![
            TaskBoundaryValue::SmallJson(serde_json::json!("test")),
            TaskBoundaryValue::SourceSnapshot(Digest::sha256("source")),
            TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("bootstrap-artifact"),
                digest: Digest::sha256("bootstrap"),
                size_bytes: 9,
            }),
        ],
        vfs_epoch: epoch,
        failure_policy: Default::default(),
        bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
    };

    let CoordinatorResponse::TaskLaunched {
        process,
        task,
        placement,
        assignment,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: submitted_task_spec.clone(),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected launched task");
    };
    assert_eq!(process, ProcessId::from("vp-control"));
    assert_eq!(task, TaskInstanceId::from("compile-linux-1"));
    assert_eq!(placement.node, NodeId::from("worker-linux"));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("environment")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("source")));
    assert_eq!(assignment.node, NodeId::from("worker-linux"));
    assert_eq!(assignment.epoch, epoch);
    assert_eq!(
        assignment.task_spec.bundle_digest,
        Some(Digest::sha256(TEST_WASM_MODULE))
    );
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(assignment.task_spec, submitted_task_spec);
    assert_eq!(
        assignment.task_spec.environment_id.as_deref(),
        Some("linux")
    );
    assert!(matches!(
        assignment.task_spec.dispatch,
        TaskDispatch::CoordinatorNodeWasm {
            export: Some(ref export),
            abi: WasmExportAbi::TaskV1,
        } if export == "compile-linux"
    ));
    assert_eq!(assignment.task_spec.vfs_epoch, epoch);
    assert_eq!(assignment.task_spec.args, submitted_task_spec.args);

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected worker assignment poll");
    };
    let assignment = assignment.expect("worker should receive queued assignment");
    assert_eq!(assignment.process, ProcessId::from("vp-control"));
    assert_eq!(assignment.task, TaskInstanceId::from("compile-linux-1"));
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(
        assignment.task_spec.bundle_digest,
        Some(Digest::sha256(TEST_WASM_MODULE))
    );
    assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64());

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "compile-linux-1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected pending join result");
    };
    assert_eq!(join.state, TaskJoinState::Pending);
    assert!(!join.remote_completion_observed);

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected empty worker assignment poll");
    };
    assert!(assignment.is_none());

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp-control".to_owned(),
            node: "worker-linux".to_owned(),
            task: "compile-linux-1".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 12,
            stderr_bytes: 0,
            stdout_tail: "ok".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/dap-output.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("artifact")),
            artifact_size_bytes: Some(12),
            result: Some(TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("dap-output.txt"),
                digest: Digest::sha256("artifact"),
                size_bytes: 12,
            })),
        })
        .unwrap();
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("vp-control".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    let event_placement = events[0]
        .placement
        .as_ref()
        .expect("task event should retain launch placement explanation");
    assert_eq!(event_placement.node, NodeId::from("worker-linux"));
    assert_eq!(event_placement.reasons, placement.reasons);
    assert_eq!(
        events[0].result,
        Some(TaskBoundaryValue::Artifact(ArtifactHandle {
            id: ArtifactId::from("dap-output.txt"),
            digest: Digest::sha256("artifact"),
            size_bytes: 12,
        }))
    );

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "compile-linux-1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected completed join result");
    };
    assert_eq!(join.state, TaskJoinState::Completed);
    assert!(join.remote_completion_observed);
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::Artifact(ArtifactHandle {
            id: ArtifactId::from("dap-output.txt"),
            digest: Digest::sha256("artifact"),
            size_bytes: 12,
        }))
    );

    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "vp-control",
        "worker-linux",
        "wasm-add",
        "wasm-add",
        epoch,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp-control".to_owned(),
            node: "worker-linux".to_owned(),
            task: "wasm-add".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 3,
            stderr_bytes: 0,
            stdout_tail: "42\n".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(42))),
        })
        .unwrap();
    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "wasm-add".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected serialized join result");
    };
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::SmallJson(serde_json::json!(42)))
    );
}

#[test]
fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order() {
    let mut service = CoordinatorService::new(41);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Duplicate instances".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            process: "vp".to_owned(),
            epoch: 41,
        })
        .unwrap();

    let restart_compatibility = Digest::sha256("compile-u32-to-u32-abi-v1");
    let (initial_module, initial_bundle_digest) =
        test_edited_task_bundle(&restart_compatibility, "initial-body");
    for (instance, argument) in [("compile-1", 1), ("compile-2", 2)] {
        let mut spec =
            test_task_spec_instance("tenant", "project", "vp", "compile", instance, 41, []);
        spec.args = vec![TaskBoundaryValue::SmallJson(serde_json::json!(argument))];
        spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile_export".to_owned()),
            abi: WasmExportAbi::TaskV1,
        };
        spec.bundle_digest = Some(initial_bundle_digest.clone());
        let CoordinatorResponse::TaskLaunched {
            task, assignment, ..
        } = service
            .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: Some("user".to_owned()),
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                task_spec: spec,
                wait_for_node: false,
                artifact_path: format!("/vfs/artifacts/{instance}.json"),
                wasm_module_base64: initial_module.clone(),
            })
            .unwrap()
        else {
            panic!("expected launched task instance");
        };
        assert_eq!(task, TaskInstanceId::from(instance));
        assert_eq!(
            assignment.task_spec.task_definition,
            clusterflux_core::TaskDefinitionId::from("compile")
        );
        assert_eq!(
            assignment.task_spec.task_instance,
            clusterflux_core::TaskInstanceId::from(instance)
        );
    }
    for instance in ["compile-1", "compile-2"] {
        let CoordinatorResponse::TaskAssignment {
            assignment: Some(assignment),
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: "worker".to_owned(),
            })
            .unwrap()
        else {
            panic!("expected queued task assignment");
        };
        assert_eq!(assignment.task, TaskInstanceId::from(instance));
    }

    service
        .handle_request(CoordinatorRequest::CancelTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            task: "compile-1".to_owned(),
        })
        .unwrap();
    for (instance, expected_cancelled) in [("compile-1", true), ("compile-2", false)] {
        let CoordinatorResponse::TaskControl {
            cancel_requested,
            abort_requested,
            ..
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                process: "vp".to_owned(),
                node: "worker".to_owned(),
                task: instance.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected task control response");
        };
        assert_eq!(cancel_requested, expected_cancelled);
        assert!(!abort_requested);
    }

    for (instance, result) in [("compile-2", 22), ("compile-1", 11)] {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                process: "vp".to_owned(),
                node: "worker".to_owned(),
                task: instance.to_owned(),
                terminal_state: Some(TaskTerminalState::Completed),
                status_code: Some(0),
                stdout_bytes: 0,
                stderr_bytes: 0,
                stdout_tail: String::new(),
                stderr_tail: String::new(),
                stdout_truncated: false,
                stderr_truncated: false,
                artifact_path: None,
                artifact_digest: None,
                artifact_size_bytes: None,
                result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(result))),
            })
            .unwrap();
    }

    for (instance, expected) in [("compile-1", 11), ("compile-2", 22)] {
        let CoordinatorResponse::TaskJoined { join } = service
            .handle_request(CoordinatorRequest::JoinTask {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
                process: "vp".to_owned(),
                task: instance.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected task join");
        };
        assert_eq!(
            join.task_instance,
            clusterflux_core::TaskInstanceId::from(instance)
        );
        assert_eq!(
            join.result,
            Some(TaskBoundaryValue::SmallJson(serde_json::json!(expected)))
        );
    }

    let duplicate = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            task_spec: test_task_spec_instance(
                "tenant",
                "project",
                "vp",
                "compile",
                "compile-1",
                41,
                [],
            ),
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/duplicate.json".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(duplicate.to_string().contains("fresh task-instance id"));

    let (replacement_module, replacement_bundle_digest) =
        test_edited_task_bundle(&restart_compatibility, "edited-body");
    assert_ne!(replacement_bundle_digest, initial_bundle_digest);
    let CoordinatorResponse::TaskRestart {
        accepted,
        restarted_task_instance,
        restarted_attempt_id,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-1".to_owned(),
            replacement_bundle: Some(TaskReplacementBundle {
                bundle_digest: replacement_bundle_digest.clone(),
                wasm_module_base64: replacement_module.clone(),
                source_snapshot: None,
            }),
        })
        .unwrap()
    else {
        panic!("expected task restart response");
    };
    assert!(accepted);
    let restarted = restarted_task_instance.expect("restart returns the logical instance");
    let restarted_attempt_id = restarted_attempt_id.expect("restart creates a new attempt");
    assert!(restarted_attempt_id.starts_with("ta_"));
    assert_eq!(restarted, TaskInstanceId::from("compile-1"));
    assert_ne!(restarted, TaskInstanceId::from("compile-2"));
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(restarted_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected restarted assignment");
    };
    assert_eq!(restarted_assignment.task, restarted);
    assert_eq!(
        restarted_assignment.task_spec.task_definition,
        clusterflux_core::TaskDefinitionId::from("compile")
    );
    assert_eq!(
        restarted_assignment.task_spec.args,
        vec![TaskBoundaryValue::SmallJson(serde_json::json!(1))]
    );
    assert_eq!(
        restarted_assignment.task_spec.bundle_digest,
        Some(replacement_bundle_digest)
    );
    assert_eq!(
        restarted_assignment.task_spec.dispatch,
        TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile_export".to_owned()),
            abi: WasmExportAbi::TaskV1,
        }
    );
    assert_eq!(restarted_assignment.wasm_module_base64, replacement_module);

    let incompatible_compatibility = Digest::sha256("compile-changed-signature");
    let (incompatible_module, incompatible_bundle_digest) =
        test_edited_task_bundle(&incompatible_compatibility, "incompatible-body");
    let CoordinatorResponse::TaskRestart {
        accepted,
        requires_whole_process_restart,
        restarted_task_instance,
        message,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-2".to_owned(),
            replacement_bundle: Some(TaskReplacementBundle {
                bundle_digest: incompatible_bundle_digest,
                wasm_module_base64: incompatible_module,
                source_snapshot: None,
            }),
        })
        .unwrap()
    else {
        panic!("expected incompatible task restart response");
    };
    assert!(!accepted);
    assert!(requires_whole_process_restart);
    assert!(restarted_task_instance.is_none());
    assert!(message.contains("task ABI changed"));

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-2".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected unaffected sibling join");
    };
    assert_eq!(join.state, TaskJoinState::Completed);
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::SmallJson(serde_json::json!(22)))
    );
}

#[test]
fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() {
    let mut service = CoordinatorService::new(73);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-soak".to_owned(),
            actor_user: "user-soak".to_owned(),
            project: "project-soak".to_owned(),
            name: "Bounded soak".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            public_key: test_node_public_key("worker-soak"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            actor_user: Some("user-soak".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-soak".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            process: "vp-soak".to_owned(),
            epoch: 73,
        })
        .unwrap();

    let event_waves = 4;
    for index in 0..super::MAX_TASK_EVENTS_PER_PROCESS * event_waves {
        let task = format!("soak-task-{index}");
        register_test_task_assignment(
            &mut service,
            "tenant-soak",
            "project-soak",
            "vp-soak",
            "worker-soak",
            "soak-task",
            &task,
            73,
        );
        service
            .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
                tenant: "tenant-soak".to_owned(),
                project: "project-soak".to_owned(),
                process: "vp-soak".to_owned(),
                node: "worker-soak".to_owned(),
                task,
                terminal_state: Some(TaskTerminalState::Completed),
                status_code: Some(0),
                stdout_bytes: 8,
                stderr_bytes: 0,
                stdout_tail: "soak-log".to_owned(),
                stderr_tail: String::new(),
                stdout_truncated: false,
                stderr_truncated: false,
                artifact_path: None,
                artifact_digest: None,
                artifact_size_bytes: None,
                result: Some(TaskBoundaryValue::SmallJson(json!(index))),
            })
            .unwrap();
    }
    for index in 0..super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS * 3 {
        service
            .record_debug_audit_event(
                TenantId::from("tenant-soak"),
                ProjectId::from("project-soak"),
                ProcessId::from("vp-soak"),
                None,
                UserId::from("user-soak"),
                "soak_inspect",
                true,
                format!("bounded audit {index}"),
            )
            .unwrap();
    }
    for _ in 0..128 {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
                tenant: "tenant-soak".to_owned(),
                project: "project-soak".to_owned(),
                node: "worker-soak".to_owned(),
            })
            .unwrap();
    }

    for index in 0..2_000 {
        service.node_replay_nonces.insert(
            (
                crate::NodeScopeKey::new(
                    TenantId::from("tenant-soak"),
                    ProjectId::from("project-soak"),
                    NodeId::from("worker-soak"),
                ),
                format!("expired-{index}"),
            ),
            0,
        );
    }
    service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            node_signature: Some(signed_node_heartbeat_in_scope(
                "tenant-soak",
                "project-soak",
                "worker-soak",
                "post-expiry-prune",
            )),
        })
        .unwrap();

    for index in 0..3 {
        service.record_task_completion_event(TaskCompletionEvent {
            tenant: TenantId::from("tenant-other"),
            project: ProjectId::from("project-other"),
            process: ProcessId::from("vp-other"),
            node: NodeId::from("worker-other"),
            executor: TaskExecutor::Node,
            task_definition: clusterflux_core::TaskDefinitionId::from("other"),
            task: TaskInstanceId::new(format!("other-{index}")),
            attempt_id: None,
            placement: None,
            terminal_state: TaskTerminalState::Completed,
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        });
    }

    let retained_soak_events = service
        .task_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-soak"))
        .count();
    let retained_other_events = service
        .task_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-other"))
        .count();
    let retained_soak_audit = service
        .debug_audit_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-soak"))
        .count();
    let retained_soak_checkpoints = service
        .task_restart_checkpoints
        .keys()
        .filter(|(_, _, process, _)| process == &ProcessId::from("vp-soak"))
        .count();
    let retained_soak_nonces = service
        .node_replay_nonces
        .keys()
        .filter(|(scope, _)| {
            scope
                == &crate::NodeScopeKey::new(
                    TenantId::from("tenant-soak"),
                    ProjectId::from("project-soak"),
                    NodeId::from("worker-soak"),
                )
        })
        .count();

    assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS);
    assert_eq!(retained_other_events, 3);
    assert_eq!(
        retained_soak_audit,
        super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS
    );
    assert_eq!(
        retained_soak_checkpoints,
        super::MAX_RESTART_CHECKPOINTS_PER_PROCESS
    );
    assert!(retained_soak_nonces <= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY);
    assert!(
        super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
            >= super::NODE_SIGNATURE_WINDOW_SECONDS as usize * 2 * 1_000
                / clusterflux_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS as usize
                + 64,
        "the bounded node replay window must sustain artifact and assignment polls at the protocol minimum with control-message headroom"
    );
    assert!(service
        .coordinator
        .active_process(
            &TenantId::from("tenant-soak"),
            &ProjectId::from("project-soak"),
            &ProcessId::from("vp-soak"),
        )
        .is_some());
}

#[test]
fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() {
    let mut service = CoordinatorService::new(12);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            process: "vp".to_owned(),
            epoch: 12,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            task_spec: test_task_spec("tenant", "project", "vp", "parent", 12, []),
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/parent.json".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();

    let child_request = CoordinatorRequest::LaunchChildTask {
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        process: "vp".to_owned(),
        node: "worker".to_owned(),
        parent_task: "parent".to_owned(),
        task_spec: test_task_spec("tenant", "project", "vp", "child", 12, []),
        wait_for_node: false,
        artifact_path: "/vfs/artifacts/child.json".to_owned(),
        wasm_module_base64: test_wasm_module_base64(),
    };
    let unsigned = service.handle_request(child_request.clone()).unwrap_err();
    assert!(unsigned.to_string().contains("signed_node"));

    let CoordinatorResponse::TaskLaunched { actor, task, .. } = service
        .handle_signed_node_request_auto(child_request)
        .unwrap()
    else {
        panic!("expected signed child launch");
    };
    assert_eq!(task, TaskInstanceId::from("child"));
    assert_eq!(actor.kind, "task");
    assert_eq!(actor.credential_kind, CredentialKind::TaskCredential);
    assert!(actor.scopes.contains(&"process:spawn-child".to_owned()));

    let wrong_parent = service
        .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            parent_task: "not-active".to_owned(),
            task: "child".to_owned(),
        })
        .unwrap_err();
    assert!(wrong_parent.to_string().contains("active parent"));

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            parent_task: "parent".to_owned(),
            task: "child".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed child join");
    };
    assert_eq!(join.state, TaskJoinState::Pending);
}

#[test]
fn coordinator_rejects_named_environment_without_requirements() {
    let mut service = CoordinatorService::new(10);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "uncached-worker".to_owned(),
            public_key: test_node_public_key("uncached-worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "uncached-worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        epoch,
        ..
    } = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-environment".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start");
    };
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "uncached-worker".to_owned(),
            process: "vp-environment".to_owned(),
            epoch,
        })
        .unwrap();
    let mut task_spec = test_task_spec(
        "tenant",
        "project",
        "vp-environment",
        "compile-linux",
        epoch,
        [],
    );
    task_spec.environment_id = Some("missing-environment".to_owned());
    task_spec.environment_digest = Some(Digest::sha256("missing-environment"));

    let error = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/environment-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();

    assert!(error.to_string().contains("named environment cache"));
}

#[test]
fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
    let mut service = CoordinatorService::new(10);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-control".to_owned(),
            restart: false,
        })
        .unwrap();

    let error = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-control",
                "compile-linux",
                10,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();

    assert!(error.to_string().contains("no capable node"));
}

#[test]
fn coordinator_side_task_launch_can_wait_for_capable_worker() {
    let mut service = CoordinatorService::new(11);
    let CoordinatorResponse::ProcessStarted {
        epoch: other_epoch, ..
    } = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "other-tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "other-vp-wait".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected unrelated process start");
    };
    let CoordinatorResponse::TaskQueued {
        queued_tasks: other_queued_tasks,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "other-tenant",
                "other-project",
                "other-vp-wait",
                "other-compile",
                other_epoch,
                [Capability::Command],
            ),
            tenant: "other-tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: Some("other-user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: true,
            artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected unrelated queued task launch");
    };
    assert_eq!(other_queued_tasks, 1);
    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        epoch,
        ..
    } = service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-wait".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start");
    };

    let CoordinatorResponse::TaskQueued {
        process,
        task,
        reason,
        queued_tasks,
        charged_spawns,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-wait",
                "compile-linux",
                epoch,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: true,
            artifact_path: "/vfs/artifacts/wait-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected queued task launch");
    };
    assert_eq!(process, ProcessId::from("vp-wait"));
    assert_eq!(task, TaskInstanceId::from("compile-linux"));
    assert!(reason.contains("waiting for any capable node"));
    assert_eq!(queued_tasks, 1);
    assert_eq!(charged_spawns, 2);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
            public_key: test_node_public_key("late-worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected pending assignment poll");
    };
    let assignment = assignment.expect("late worker should receive pending assignment");
    assert_eq!(assignment.process, ProcessId::from("vp-wait"));
    assert_eq!(assignment.task, TaskInstanceId::from("compile-linux"));
    assert_eq!(assignment.node, NodeId::from("late-worker"));
    assert_eq!(assignment.epoch, epoch);
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64());

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected empty assignment poll");
    };
    assert!(assignment.is_none());
}

#[test]
fn service_rejects_malformed_node_capability_report() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let mut capabilities = linux_capabilities();
    capabilities
        .source_providers
        .insert("../checkout".to_owned());
    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities,
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("source provider id"));
    assert!(service.node_descriptors.is_empty());
}

#[test]
fn service_meters_rendezvous_before_direct_transfer_plan() {
    let mut service = CoordinatorService::new(7);
    service.quota.set_rendezvous_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::RendezvousAttempt, 2)]),
    });

    let CoordinatorResponse::RendezvousPlan {
        plan,
        charged_rendezvous_attempts,
    } = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: true,
            failure_reason: String::new(),
        })
        .unwrap()
    else {
        panic!("expected rendezvous plan");
    };

    assert_eq!(charged_rendezvous_attempts, 1);
    assert_eq!(plan.scope.project, ProjectId::from("project"));
    assert_eq!(plan.source.node, NodeId::from("node-a"));
    assert_eq!(plan.destination.node, NodeId::from("node-b"));
    assert!(plan.coordinator_assisted_rendezvous);
    assert!(!plan.coordinator_bulk_relay_allowed);

    let unavailable = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: false,
            failure_reason: "nat traversal failed".to_owned(),
        })
        .unwrap_err();
    let unavailable = unavailable.to_string();
    assert!(unavailable.contains("nat traversal failed"));
    assert!(unavailable.contains("coordinator bulk relay is disabled"));

    let quota = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: true,
            failure_reason: String::new(),
        })
        .unwrap_err();
    assert!(quota.to_string().contains("RendezvousAttempt"));
}

#[test]
fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node-a".to_owned(),
            public_key: test_node_public_key("node-a"),
        })
        .unwrap();

    let response = service
        .handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: true,
            failure_reason: String::new(),
        })
        .unwrap();

    let CoordinatorResponse::RendezvousPlan { plan, .. } = response else {
        panic!("expected signed rendezvous plan");
    };
    assert_eq!(plan.source.node, NodeId::from("node-a"));
    assert_eq!(plan.destination.node, NodeId::from("node-b"));
}

#[test]
fn signed_hostile_artifact_paths_return_errors_and_the_same_service_stays_healthy() {
    let mut service = CoordinatorService::new(27);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "hostile-path-process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            process: "hostile-path-process".to_owned(),
            epoch: 27,
        })
        .unwrap();

    let invalid_metadata = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "hostile-path-process".to_owned(),
            node: "node".to_owned(),
            task: "metadata-task".to_owned(),
            artifact_path: Some("/vfs/artifacts/bad artifact!".to_owned()),
            artifact_digest: Some(Digest::sha256("bad")),
            artifact_size_bytes: Some(3),
            large_bytes_uploaded: false,
        })
        .unwrap_err();
    assert!(
        invalid_metadata
            .to_string()
            .contains("invalid VFS artifact path"),
        "unexpected error: {invalid_metadata}"
    );

    let valid_metadata = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "hostile-path-process".to_owned(),
            node: "node".to_owned(),
            task: "metadata-task".to_owned(),
            artifact_path: Some("/vfs/artifacts/valid-artifact".to_owned()),
            artifact_digest: Some(Digest::sha256("valid")),
            artifact_size_bytes: Some(5),
            large_bytes_uploaded: false,
        })
        .unwrap();
    assert!(matches!(
        valid_metadata,
        CoordinatorResponse::VfsMetadataRecorded { .. }
    ));

    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "hostile-path-process",
        "node",
        "child",
        "child-instance",
        27,
    );
    let invalid_completion = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "hostile-path-process".to_owned(),
            node: "node".to_owned(),
            task: "child-instance".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/repeated//component".to_owned()),
            artifact_digest: Some(Digest::sha256("bad-completion")),
            artifact_size_bytes: Some(0),
            result: None,
        })
        .unwrap_err();
    assert!(
        invalid_completion
            .to_string()
            .contains("invalid VFS artifact path"),
        "unexpected error: {invalid_completion}"
    );

    let valid_completion = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "hostile-path-process".to_owned(),
            node: "node".to_owned(),
            task: "child-instance".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/valid-completion".to_owned()),
            artifact_digest: Some(Digest::sha256("valid-completion")),
            artifact_size_bytes: Some(0),
            result: None,
        })
        .unwrap();
    assert!(matches!(
        valid_completion,
        CoordinatorResponse::TaskRecorded { .. }
    ));
}

#[test]
fn service_rejects_task_completion_outside_node_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 128,
            stderr_bytes: 64,
            stdout_tail: "foreign stdout".to_owned(),
            stderr_tail: "foreign stderr".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/foreign.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("foreign-artifact")),
            artifact_size_bytes: Some(128),
            result: None,
        })
        .unwrap_err();

    assert!(error.to_string().contains("not enrolled"));
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert!(events.is_empty());
}

#[test]
fn service_rejects_task_event_access_using_retained_process_scope() {
    let mut service = CoordinatorService::new(1);
    service.process_scope_history.push_back((
        TenantId::from("tenant-a"),
        ProjectId::from("project-a"),
        ProcessId::from("process"),
    ));

    let error = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap_err();

    assert!(error
        .to_string()
        .contains("outside the virtual process tenant/project scope"));
}

#[test]
fn service_rejects_node_capability_report_outside_enrollment_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("not enrolled"));
    assert!(service.node_descriptors.is_empty());
}

#[test]
fn service_rejects_source_preparation_completion_outside_node_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::CompleteSourcePreparation {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: "node".to_owned(),
            provider: SourceProviderKind::Filesystem,
            source_snapshot: Digest::sha256("foreign-source"),
        })
        .unwrap_err();

    assert!(error.to_string().contains("not enrolled"));
}

#[test]
fn service_rejects_unknown_node_heartbeat() {
    let mut service = CoordinatorService::new(1);

    let error = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "missing".to_owned(),
            node_signature: None,
        })
        .unwrap_err();

    assert!(error.to_string().contains("not enrolled"));
}

#[test]
fn service_requires_signed_node_heartbeat_from_enrolled_key() {
    let mut service = CoordinatorService::new(5);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let unsigned = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: None,
        })
        .unwrap_err();
    assert!(unsigned.to_string().contains("signed proof"));

    let wrong_private_key = test_node_private_key("other-node");
    let wrong_signature = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat_with_private_key(
                "node",
                &wrong_private_key,
                "wrong-node-key",
            )),
        })
        .unwrap_err();
    assert!(wrong_signature.to_string().contains("signature"));

    let signed = signed_node_heartbeat("node", "fresh-node-heartbeat");
    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed.clone()),
        })
        .unwrap();
    assert_eq!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 5,
        }
    );

    let replay = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed),
        })
        .unwrap_err();
    assert!(replay.to_string().contains("nonce"));
}

#[test]
fn service_rejects_raw_node_originated_requests_without_signed_envelope() {
    let mut service = CoordinatorService::new(5);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_request(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("signed_node envelope proof"));
}

#[test]
fn service_stream_accepts_multiple_requests_on_one_connection() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream_local_trusted(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        },
        "stream-1",
    );

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    assert!(matches!(
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
        CoordinatorResponse::NodeAttached { .. }
    ));

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::NodeHeartbeat {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")),
        },
        "stream-2",
    );

    line.clear();
    reader.read_line(&mut line).unwrap();
    assert_eq!(
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 3,
        }
    );

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        service
            .issue_cli_session(
                TenantId::from("tenant"),
                ProjectId::from("project"),
                UserId::from("user"),
                "strict-stream-session",
                None,
            )
            .unwrap();
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());
    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::StartProcess {
            launch_attempt: None,
            tenant: "victim-tenant".to_owned(),
            project: "victim-project".to_owned(),
            actor_user: Some("forged-user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-forged".to_owned(),
            restart: false,
        },
        "strict-stream-forged",
    );

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    let CoordinatorResponse::Error { error } =
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
    else {
        panic!("expected strict body-authority denial");
    };
    assert!(error
        .message
        .contains("request-body identity fields are not authority"));
    assert_eq!(error.request_id, "strict-stream-forged");
    assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden);

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::Authenticated {
            session_secret: "strict-stream-session".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "vp-authenticated".to_owned(),
                restart: false,
            },
        },
        "strict-stream-authenticated",
    );

    line.clear();
    reader.read_line(&mut line).unwrap();
    let CoordinatorResponse::ProcessStarted {
        launch_attempt: None,
        process,
        actor,
        ..
    } = serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
    else {
        panic!("expected authenticated strict process start");
    };
    assert_eq!(process, ProcessId::from("vp-authenticated"));
    assert_eq!(actor.user, Some(UserId::from("user")));

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn service_stream_rejects_invalid_versioned_envelope_metadata() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());
    let mut wire_request = coordinator_wire_request(
        "bad-operation",
        serde_json::to_value(CoordinatorRequest::Ping).unwrap(),
    );
    wire_request["operation"] = serde_json::Value::String("attach_node".to_owned());
    serde_json::to_writer(&mut stream, &wire_request).unwrap();
    stream.write_all(b"\n").unwrap();
    stream.flush().unwrap();

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    let response = serde_json::from_str::<CoordinatorResponse>(&line).unwrap();
    let CoordinatorResponse::Error { error } = response else {
        panic!("expected invalid wire envelope response");
    };
    assert!(error
        .message
        .contains("operation attach_node does not match payload operation ping"));
    assert_eq!(error.request_id, "bad-operation");
    assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn coordinator_protocol_rejects_client_supplied_authority_fields() {
    for payload in [
        json!({
            "type": "create_node_enrollment_grant",
            "tenant": "tenant",
            "project": "project",
            "actor_user": "user",
            "grant": "attacker-chosen",
            "ttl_seconds": 60,
        }),
        json!({
            "type": "exchange_node_enrollment_grant",
            "tenant": "tenant",
            "project": "project",
            "node": "node",
            "public_key": "key",
            "enrollment_grant": "grant",
            "now_epoch_seconds": 0,
        }),
        json!({
            "type": "schedule_task",
            "tenant": "tenant",
            "project": "project",
            "environment": null,
            "environment_digest": null,
            "required_capabilities": [],
            "dependency_cache": null,
            "source_snapshot": null,
            "required_artifacts": [],
            "quota_available": true,
            "policy_allowed": true,
            "prefer_node": null,
        }),
        json!({
            "type": "create_artifact_download_link",
            "tenant": "tenant",
            "project": "project",
            "actor_user": "user",
            "artifact": "artifact",
            "max_bytes": 1,
            "token_nonce": "attacker-chosen",
            "ttl_seconds": 60,
        }),
    ] {
        let error = serde_json::from_value::<CoordinatorRequest>(payload).unwrap_err();
        assert!(error.to_string().contains("unknown field"));
    }
}

#[test]
fn coordinator_generates_and_bounds_node_enrollment_grants() {
    let mut service = CoordinatorService::new(7);
    service.set_server_time(100);

    let first = service
        .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            ttl_seconds: u64::MAX,
        })
        .unwrap();
    let second = service
        .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            ttl_seconds: u64::MAX,
        })
        .unwrap();

    let CoordinatorResponse::NodeEnrollmentGrantCreated {
        grant: first_grant,
        expires_at_epoch_seconds,
        ..
    } = first
    else {
        panic!("expected enrollment grant");
    };
    let CoordinatorResponse::NodeEnrollmentGrantCreated {
        grant: second_grant,
        ..
    } = second
    else {
        panic!("expected enrollment grant");
    };
    assert!(first_grant.starts_with("node_grant_"));
    assert_ne!(first_grant, second_grant);
    assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60);
}

#[test]
fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() {
    let mut service = CoordinatorService::new(7);
    for (tenant, project, user, secret) in [
        ("tenant-a", "project-a", "user-a", "session-a"),
        ("tenant-b", "project-b", "user-b", "session-b"),
    ] {
        service
            .issue_cli_session(
                TenantId::from(tenant),
                ProjectId::from(project),
                UserId::from(user),
                secret,
                None,
            )
            .unwrap();
    }
    service.set_server_time(100);
    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "process-one".to_owned(),
                restart: false,
            },
        })
        .unwrap();

    let CoordinatorResponse::ProcessSummaries {
        processes,
        next_cursor,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: None,
                limit: 1,
            },
        })
        .unwrap()
    else {
        panic!("expected process summaries");
    };
    assert_eq!(processes.len(), 1);
    assert_eq!(processes[0].process, ProcessId::from("process-one"));
    assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active);
    assert_eq!(processes[0].activity, ProcessActivityState::Running);
    assert_eq!(processes[0].started_at_epoch_seconds, 100);
    assert!(next_cursor.is_none());

    let CoordinatorResponse::ProcessSummaries { processes, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-b".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: None,
                limit: 10,
            },
        })
        .unwrap()
    else {
        panic!("expected scoped process summaries");
    };
    assert!(processes.is_empty());

    service.set_server_time(120);
    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::AbortProcess {
                process: "process-one".to_owned(),
                launch_attempt: None,
            },
        })
        .unwrap();
    let CoordinatorResponse::ProcessSummaries { processes, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: None,
                limit: 10,
            },
        })
        .unwrap()
    else {
        panic!("expected terminal process summary");
    };
    assert_eq!(
        processes[0].lifecycle,
        ProcessLifecycleState::RecentTerminal
    );
    assert_eq!(processes[0].activity, ProcessActivityState::Cancelled);
    assert_eq!(
        processes[0].final_result,
        Some(ProcessFinalResult::Cancelled)
    );
    assert_eq!(processes[0].ended_at_epoch_seconds, Some(120));

    service.set_server_time(130);
    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                launch_attempt: None,
                process: "process-two".to_owned(),
                restart: false,
            },
        })
        .unwrap();
    let CoordinatorResponse::ProcessSummaries {
        processes,
        next_cursor: Some(cursor),
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: None,
                limit: 1,
            },
        })
        .unwrap()
    else {
        panic!("expected first process summary page");
    };
    assert_eq!(processes[0].process, ProcessId::from("process-two"));
    let CoordinatorResponse::ProcessSummaries {
        processes,
        next_cursor: None,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: Some(cursor),
                limit: 1,
            },
        })
        .unwrap()
    else {
        panic!("expected final process summary page");
    };
    assert_eq!(processes[0].process, ProcessId::from("process-one"));

    let oversized = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
                cursor: None,
                limit: 101,
            },
        })
        .unwrap_err();
    assert!(oversized.to_string().contains("limit"));
    assert!(oversized.to_string().contains("100"));
}

#[test]
fn process_summary_eviction_releases_live_log_accounting_state() {
    let mut service = CoordinatorService::new(7);
    let tenant = TenantId::from("tenant-summary-bound");
    let project = ProjectId::from("project-summary-bound");
    let task = TaskInstanceId::from("task");

    for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT {
        let process = ProcessId::new(format!("process-{index:03}"));
        service.record_process_started(&tenant, &project, &process, index as u64);
        service.record_process_terminal(
            &tenant,
            &project,
            &process,
            ProcessFinalResult::Completed,
            index as u64 + 1,
        );
        let key = (
            tenant.clone(),
            project.clone(),
            process,
            task.clone(),
            "stdout".to_owned(),
        );
        service.recent_log_accounted_bytes.insert(key.clone(), 10);
        service.recent_log_truncated_streams.insert(key);
    }

    let evicted = ProcessId::from("process-000");
    service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000);

    assert!(!service.process_summaries.contains_key(&(
        tenant.clone(),
        project.clone(),
        evicted.clone()
    )));
    assert!(!service.recent_log_accounted_bytes.keys().any(
        |(entry_tenant, entry_project, process, _, _)| {
            entry_tenant == &tenant && entry_project == &project && process == &evicted
        }
    ));
    assert!(!service.recent_log_truncated_streams.iter().any(
        |(entry_tenant, entry_project, process, _, _)| {
            entry_tenant == &tenant && entry_project == &project && process == &evicted
        }
    ));
}

#[test]
fn web_node_summaries_are_scoped_paginated_and_hard_bounded() {
    let mut service = CoordinatorService::new(7);
    service
        .issue_cli_session(
            TenantId::from("tenant"),
            ProjectId::from("project"),
            UserId::from("user"),
            "session",
            None,
        )
        .unwrap();
    for node in ["node-a", "node-b", "node-c"] {
        enroll_test_node(
            &mut service,
            "tenant",
            "project",
            node,
            &test_node_public_key(node),
        );
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                capabilities: linux_capabilities(),
                cached_environment_digests: Vec::new(),
                dependency_cache_digests: Vec::new(),
                source_snapshots: Vec::new(),
                artifact_locations: Vec::new(),
                direct_connectivity: true,
                online: true,
            })
            .unwrap();
    }

    let CoordinatorResponse::NodeSummaries {
        nodes,
        next_cursor: Some(cursor),
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
                cursor: None,
                limit: 2,
            },
        })
        .unwrap()
    else {
        panic!("expected first node-summary page");
    };
    assert_eq!(
        nodes
            .iter()
            .map(|node| node.id.as_str())
            .collect::<Vec<_>>(),
        ["node-a", "node-b"]
    );

    let CoordinatorResponse::NodeSummaries {
        nodes,
        next_cursor: None,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
                cursor: Some(cursor),
                limit: 2,
            },
        })
        .unwrap()
    else {
        panic!("expected final node-summary page");
    };
    assert_eq!(nodes.len(), 1);
    assert_eq!(nodes[0].id, NodeId::from("node-c"));

    let oversized = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
                cursor: None,
                limit: 201,
            },
        })
        .unwrap_err();
    assert!(oversized.to_string().contains("limit"));
    assert!(oversized.to_string().contains("200"));
}

#[test]
fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() {
    let mut service = CoordinatorService::new(7);
    for (tenant, project, user, secret, node) in [
        ("tenant-a", "project-a", "user-a", "session-a", "node-a"),
        ("tenant-b", "project-b", "user-b", "session-b", "node-b"),
    ] {
        service
            .issue_cli_session(
                TenantId::from(tenant),
                ProjectId::from(project),
                UserId::from(user),
                secret,
                None,
            )
            .unwrap();
        enroll_test_node(
            &mut service,
            tenant,
            project,
            node,
            &test_node_public_key(node),
        );
    }
    service.set_server_time(100);
    for (tenant, project, node) in [
        ("tenant-a", "project-a", "node-a"),
        ("tenant-b", "project-b", "node-b"),
    ] {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
                tenant: tenant.to_owned(),
                project: project.to_owned(),
                node: node.to_owned(),
                capabilities: linux_capabilities(),
                cached_environment_digests: Vec::new(),
                dependency_cache_digests: Vec::new(),
                source_snapshots: Vec::new(),
                artifact_locations: vec!["shared-artifact".to_owned()],
                direct_connectivity: true,
                online: false,
            })
            .unwrap();
    }
    let CoordinatorResponse::NodeSummaries { nodes, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
                cursor: None,
                limit: 200,
            },
        })
        .unwrap()
    else {
        panic!("expected node summaries");
    };
    assert_eq!(nodes.len(), 1);
    assert_eq!(nodes[0].id, NodeId::from("node-a"));
    assert!(nodes[0].online);
    assert!(!nodes[0].stale);
    assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100));
    assert_eq!(nodes[0].capabilities.os, Os::Linux);
    for (tenant, project, node, digest) in [
        ("tenant-a", "project-a", "node-a", "tenant-a-bytes"),
        ("tenant-b", "project-b", "node-b", "tenant-b-bytes"),
    ] {
        service.artifact_registry.flush_metadata(ArtifactFlush {
            id: ArtifactId::from("shared-artifact"),
            tenant: TenantId::from(tenant),
            project: ProjectId::from(project),
            process: ProcessId::from("process-one"),
            producer_task: TaskInstanceId::from("task-one"),
            retaining_node: NodeId::from(node),
            digest: Digest::sha256(digest),
            size: digest.len() as u64,
        });
    }
    service.artifact_registry.flush_metadata(ArtifactFlush {
        id: ArtifactId::from("second-artifact"),
        tenant: TenantId::from("tenant-a"),
        project: ProjectId::from("project-a"),
        process: ProcessId::from("process-two"),
        producer_task: TaskInstanceId::from("task-two"),
        retaining_node: NodeId::from("node-a"),
        digest: Digest::sha256("second"),
        size: 6,
    });

    let CoordinatorResponse::Artifacts {
        artifacts,
        next_cursor: Some(cursor),
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListArtifacts {
                process: None,
                cursor: None,
                limit: 1,
            },
        })
        .unwrap()
    else {
        panic!("expected first artifact page");
    };
    assert_eq!(artifacts.len(), 1);
    assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact"));
    assert_eq!(artifacts[0].availability, ArtifactAvailability::Available);
    let CoordinatorResponse::Artifacts {
        artifacts,
        next_cursor: None,
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListArtifacts {
                process: None,
                cursor: Some(cursor),
                limit: 1,
            },
        })
        .unwrap()
    else {
        panic!("expected final artifact page");
    };
    assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact"));
    assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::GetArtifact {
                artifact: "tenant-b-only".to_owned(),
            },
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("does not exist"));
    let CoordinatorResponse::Artifact { artifact } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-b".to_owned(),
            request: AuthenticatedCoordinatorRequest::GetArtifact {
                artifact: "shared-artifact".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected tenant-b artifact");
    };
    assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes"));

    service.set_server_time(131);
    let CoordinatorResponse::NodeSummaries { nodes, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
                cursor: None,
                limit: 200,
            },
        })
        .unwrap()
    else {
        panic!("expected stale node summary");
    };
    assert!(!nodes[0].online);
    assert!(nodes[0].stale);
    assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100));
    let CoordinatorResponse::Artifact { artifact } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::GetArtifact {
                artifact: "shared-artifact".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected offline artifact metadata");
    };
    assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline);
    assert!(!artifact.downloadable_now);

    service
        .artifact_registry
        .sync_to_explicit_store(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            &ArtifactId::from("shared-artifact"),
            "store://tenant-a/shared-artifact",
        )
        .unwrap();
    let CoordinatorResponse::Artifact { artifact } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::GetArtifact {
                artifact: "shared-artifact".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected explicitly retained artifact metadata");
    };
    assert_eq!(artifact.availability, ArtifactAvailability::Available);
    assert_eq!(
        artifact.retention_state,
        ArtifactRetentionState::ExplicitStorage
    );
    assert!(artifact.downloadable_now);
}

#[test]
fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
    let mut service = CoordinatorService::new(7);
    for (tenant, project, user, secret, node, process) in [
        (
            "tenant-a",
            "project-a",
            "user-a",
            "session-a",
            "node-a",
            "process-shared",
        ),
        (
            "tenant-b",
            "project-b",
            "user-b",
            "session-b",
            "node-b",
            "process-b",
        ),
    ] {
        service
            .issue_cli_session(
                TenantId::from(tenant),
                ProjectId::from(project),
                UserId::from(user),
                secret,
                None,
            )
            .unwrap();
        enroll_test_node(
            &mut service,
            tenant,
            project,
            node,
            &test_node_public_key(node),
        );
        service
            .handle_request(CoordinatorRequest::Authenticated {
                session_secret: secret.to_owned(),
                request: AuthenticatedCoordinatorRequest::StartProcess {
                    launch_attempt: None,
                    process: process.to_owned(),
                    restart: false,
                },
            })
            .unwrap();
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
                tenant: tenant.to_owned(),
                project: project.to_owned(),
                node: node.to_owned(),
                process: process.to_owned(),
                epoch: 7,
            })
            .unwrap();
    }
    service.set_server_time(100);
    let first = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            process: "process-shared".to_owned(),
            node: "node-a".to_owned(),
            task: "task-one".to_owned(),
            stream: TaskLogStream::Stdout,
            offset: 0,
            source_bytes: 5,
            text: "hello".to_owned(),
            truncated: false,
        })
        .unwrap();
    let CoordinatorResponse::TaskLogChunkRecorded {
        sequence: Some(first_sequence),
        next_offset: 5,
        ..
    } = first
    else {
        panic!("expected first live log sequence");
    };
    assert_eq!(
        service.quota.used_log_bytes(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            100,
        ),
        5,
        "live bytes must be charged when accepted"
    );
    let retry = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            process: "process-shared".to_owned(),
            node: "node-a".to_owned(),
            task: "task-one".to_owned(),
            stream: TaskLogStream::Stdout,
            offset: 0,
            source_bytes: 5,
            text: "hello".to_owned(),
            truncated: false,
        })
        .unwrap();
    assert!(matches!(
        retry,
        CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. }
    ));
    assert_eq!(
        service.quota.used_log_bytes(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            100,
        ),
        5,
        "a retried chunk must not be charged twice"
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            process: "process-shared".to_owned(),
            node: "node-a".to_owned(),
            task: "task-one".to_owned(),
            stream: TaskLogStream::Stdout,
            offset: 8,
            source_bytes: 2,
            text: "ok".to_owned(),
            truncated: false,
        })
        .unwrap();
    assert_eq!(
        service.quota.used_log_bytes(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            100,
        ),
        10,
        "a gap and the delivered bytes must both count toward source-byte usage"
    );

    let CoordinatorResponse::RecentLogs {
        entries,
        next_sequence: Some(cursor),
        history_truncated: false,
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListRecentLogs {
                process: "process-shared".to_owned(),
                task: None,
                after_sequence: None,
                limit: 2,
            },
        })
        .unwrap()
    else {
        panic!("expected first recent-log page");
    };
    assert_eq!(entries.len(), 2);
    assert_eq!(entries[0].sequence, first_sequence);
    assert_eq!(entries[0].text, "hello");
    assert!(entries[1].text.contains("3 bytes"));
    assert!(entries[1].truncated);
    let CoordinatorResponse::RecentLogs { entries, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListRecentLogs {
                process: "process-shared".to_owned(),
                task: None,
                after_sequence: Some(cursor),
                limit: 2,
            },
        })
        .unwrap()
    else {
        panic!("expected second recent-log page");
    };
    assert_eq!(entries.len(), 1);
    assert_eq!(entries[0].text, "ok");

    service
        .handle_report_task_log(
            "tenant-a".to_owned(),
            "project-a".to_owned(),
            "process-shared".to_owned(),
            "node-a".to_owned(),
            "task-one".to_owned(),
            12,
            0,
            "hello???okZZ".to_owned(),
            String::new(),
            false,
            false,
            false,
        )
        .unwrap();
    assert_eq!(
        service.quota.used_log_bytes(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            100,
        ),
        12,
        "the final summary must charge only source bytes not already charged live"
    );
    assert_eq!(
        service
            .recent_logs
            .get(&(TenantId::from("tenant-a"), ProjectId::from("project-a")))
            .unwrap()
            .back()
            .unwrap()
            .text,
        "ZZ",
        "final-tail reconciliation must append only the nonduplicating suffix"
    );
    service
        .handle_report_task_log(
            "tenant-a".to_owned(),
            "project-a".to_owned(),
            "process-shared".to_owned(),
            "node-a".to_owned(),
            "task-one".to_owned(),
            12,
            0,
            "hello???okZZ".to_owned(),
            String::new(),
            false,
            false,
            false,
        )
        .unwrap();
    assert_eq!(
        service.quota.used_log_bytes(
            &TenantId::from("tenant-a"),
            &ProjectId::from("project-a"),
            100,
        ),
        12,
        "replayed final accounting must be idempotent"
    );

    let marker = service
        .handle_report_task_log_chunk(
            "tenant-a".to_owned(),
            "project-a".to_owned(),
            "process-shared".to_owned(),
            "node-a".to_owned(),
            "task-one".to_owned(),
            TaskLogStream::Stdout,
            12,
            0,
            "[log output truncated at node capture limit]".to_owned(),
            true,
        )
        .unwrap();
    assert!(matches!(
        marker,
        CoordinatorResponse::TaskLogChunkRecorded {
            sequence: Some(_),
            next_offset: 12,
            ..
        }
    ));
    let repeated_marker = service
        .handle_report_task_log_chunk(
            "tenant-a".to_owned(),
            "project-a".to_owned(),
            "process-shared".to_owned(),
            "node-a".to_owned(),
            "task-one".to_owned(),
            TaskLogStream::Stdout,
            12,
            0,
            "[log output truncated at node capture limit]".to_owned(),
            true,
        )
        .unwrap();
    assert!(matches!(
        repeated_marker,
        CoordinatorResponse::TaskLogChunkRecorded {
            sequence: None,
            next_offset: 12,
            ..
        }
    ));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-b".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListRecentLogs {
                process: "process-shared".to_owned(),
                task: None,
                after_sequence: None,
                limit: 10,
            },
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("outside"));
    let oversized = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListRecentLogs {
                process: "process-shared".to_owned(),
                task: None,
                after_sequence: None,
                limit: 201,
            },
        })
        .unwrap_err();
    assert!(oversized.to_string().contains("limit"));
    assert!(oversized.to_string().contains("200"));

    service
        .handle_report_task_log_chunk(
            "tenant-b".to_owned(),
            "project-b".to_owned(),
            "process-b".to_owned(),
            "node-b".to_owned(),
            "task-b".to_owned(),
            TaskLogStream::Stderr,
            0,
            1,
            "b".to_owned(),
            false,
        )
        .unwrap();
    for offset in 10..310 {
        service
            .handle_report_task_log_chunk(
                "tenant-a".to_owned(),
                "project-a".to_owned(),
                "process-shared".to_owned(),
                "node-a".to_owned(),
                "task-one".to_owned(),
                TaskLogStream::Stdout,
                offset,
                1,
                "x".to_owned(),
                false,
            )
            .unwrap();
    }
    let tenant_a_logs =
        &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))];
    assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS);
    let tenant_b_logs =
        &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))];
    assert_eq!(tenant_b_logs.len(), 1);
    assert_eq!(tenant_b_logs[0].text, "b");
    let CoordinatorResponse::RecentLogs {
        entries,
        history_truncated,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListRecentLogs {
                process: "process-shared".to_owned(),
                task: None,
                after_sequence: None,
                limit: 200,
            },
        })
        .unwrap()
    else {
        panic!("expected bounded recent-log response");
    };
    assert_eq!(entries.len(), 200);
    assert!(history_truncated);
}