clusterflux-public/crates/clusterflux-coordinator/src/service/tests.rs

use std::collections::{BTreeMap, BTreeSet};
use std::io::{BufRead, BufReader, Write};
use std::net::TcpStream;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};

use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
    admin_request_proof, agent_ed25519_public_key_from_private_key, coordinator_wire_request,
    derive_ed25519_private_key_from_seed, node_ed25519_public_key_from_private_key,
    sign_agent_workflow_request, sign_node_request, signed_request_payload_digest,
    AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability,
    DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements,
    LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits,
    SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec,
    VfsPath, WasmExportAbi,
};
use serde_json::json;

use crate::FallibleDurableStore;

use super::keys::{process_control_key, task_control_key};
use super::*;

fn test_admin_request(
    token: &str,
    operation: &str,
    tenant: &str,
    actor_user: &str,
    target_tenant: &str,
    nonce: &str,
) -> (Digest, String, u64) {
    let issued_at_epoch_seconds = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .unwrap_or_default();
    (
        admin_request_proof(
            token,
            operation,
            tenant,
            actor_user,
            target_tenant,
            nonce,
            issued_at_epoch_seconds,
        ),
        nonce.to_owned(),
        issued_at_epoch_seconds,
    )
}

#[test]
fn runtime_service_uses_memory_only_when_database_url_is_absent() {
    let service = CoordinatorService::new_with_database_url(1, None).unwrap();
    assert_eq!(service.durable_store_kind(), "in_memory");

    let error = CoordinatorService::new_with_database_url(1, Some("not-a-postgres-url"))
        .err()
        .expect("an invalid configured DATABASE_URL must fail closed");
    assert!(error
        .to_string()
        .contains("durable coordinator state failed"));
}

#[test]
fn postgres_backed_runtime_service_survives_restart_without_live_process_state() {
    let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else {
        return;
    };
    let mut clean_store = crate::PostgresDurableStore::connect(&database_url).unwrap();
    clean_store
        .save_state(&crate::DurableState::default())
        .unwrap();

    let session_secret = "postgres-runtime-service-session";
    let mut first = CoordinatorService::new_with_database_url(41, Some(&database_url)).unwrap();
    assert_eq!(first.durable_store_kind(), "postgres");
    first
        .issue_cli_session(
            TenantId::from("tenant-pg"),
            ProjectId::from("project-pg"),
            UserId::from("user-pg"),
            session_secret,
            None,
        )
        .unwrap();
    first
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-pg".to_owned(),
            project: "project-pg".to_owned(),
            node: "node-pg".to_owned(),
            public_key: test_node_public_key("node-pg"),
        })
        .unwrap();
    first
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                process: "process-ephemeral".to_owned(),
                restart: false,
            },
        })
        .unwrap();
    drop(first);

    let mut restarted = CoordinatorService::new_with_database_url(42, Some(&database_url)).unwrap();
    assert_eq!(restarted.durable_store_kind(), "postgres");
    let auth = restarted
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    assert!(matches!(
        auth,
        CoordinatorResponse::AuthStatus {
            authenticated: true,
            ..
        }
    ));
    let heartbeat = restarted
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node-pg".to_owned(),
            node_signature: Some(signed_node_heartbeat(
                "node-pg",
                "postgres-restart-heartbeat",
            )),
        })
        .unwrap();
    assert!(matches!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat { epoch: 42, .. }
    ));
    let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: session_secret.to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProcesses,
        })
        .unwrap()
    else {
        panic!("expected process list");
    };
    assert!(processes.is_empty());
}

fn linux_capabilities() -> NodeCapabilities {
    NodeCapabilities {
        os: Os::Linux,
        arch: "x86_64".to_owned(),
        capabilities: BTreeSet::from([
            Capability::Command,
            Capability::Containers,
            Capability::RootlessPodman,
            Capability::VfsArtifacts,
        ]),
        environment_backends: BTreeSet::from([EnvironmentBackend::Container]),
        source_providers: BTreeSet::from(["filesystem".to_owned()]),
    }
}

fn windows_capabilities() -> NodeCapabilities {
    NodeCapabilities {
        os: Os::Windows,
        arch: "x86_64".to_owned(),
        capabilities: BTreeSet::from([
            Capability::Command,
            Capability::WindowsCommandDev,
            Capability::VfsArtifacts,
        ]),
        environment_backends: BTreeSet::from([EnvironmentBackend::WindowsCommandDev]),
        source_providers: BTreeSet::from(["filesystem".to_owned()]),
    }
}

fn endpoint(name: &str) -> NodeEndpoint {
    NodeEndpoint {
        node: NodeId::from(name),
        advertised_addr: format!("{name}.mesh.invalid:4433"),
        public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
    }
}

fn data_plane_scope(project: &str) -> DataPlaneScope {
    DataPlaneScope {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from(project),
        process: ProcessId::from("process"),
        object: DataPlaneObject::Artifact(ArtifactId::from("artifact")),
        authorization_subject: "node-a-to-node-b".to_owned(),
    }
}

fn assert_agent_workflow_actor(actor: &WorkflowActor, fingerprint: &Digest) {
    assert_eq!(actor.kind, "agent");
    assert_eq!(actor.user, Some(UserId::from("user")));
    assert_eq!(actor.agent, Some(AgentId::from("agent-ci")));
    assert_eq!(actor.credential_kind, CredentialKind::PublicKey);
    assert_eq!(actor.public_key_fingerprint.as_ref(), Some(fingerprint));
    assert!(actor.authenticated_without_browser);
    assert!(actor.scopes.iter().any(|scope| scope == "project:run"));
}

const TEST_AGENT_PRIVATE_KEY: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
static TEST_NODE_REQUEST_NONCE: AtomicU64 = AtomicU64::new(1);
const TEST_WASM_MODULE: &[u8] = b"clusterflux-test-wasm-module";

fn test_wasm_module_base64() -> String {
    BASE64_STANDARD.encode(TEST_WASM_MODULE)
}

fn append_test_custom_section(module: &mut Vec<u8>, name: &str, data: &[u8]) {
    fn leb(mut value: usize, output: &mut Vec<u8>) {
        loop {
            let mut byte = (value & 0x7f) as u8;
            value >>= 7;
            if value != 0 {
                byte |= 0x80;
            }
            output.push(byte);
            if value == 0 {
                break;
            }
        }
    }

    let mut payload = Vec::new();
    leb(name.len(), &mut payload);
    payload.extend_from_slice(name.as_bytes());
    payload.extend_from_slice(data);
    module.push(0);
    leb(payload.len(), module);
    module.extend_from_slice(&payload);
}

fn test_edited_task_bundle(compatibility: &Digest, edit_marker: &str) -> (String, Digest) {
    let descriptor = serde_json::to_vec(&serde_json::json!({
        "kind": "task",
        "name": "compile",
        "function": "compile",
        "export": "compile_export",
        "stable_id": Digest::sha256("compile-stable"),
        "argument_schema": "u32",
        "result_schema": "u32",
        "required_capabilities": [],
        "restart_compatibility_hash": compatibility,
        "abi_version": clusterflux_core::WASM_TASK_ABI_VERSION,
        "probe_symbol": "clusterflux.probe.compile",
    }))
    .unwrap();
    let mut module = b"\0asm\x01\0\0\0".to_vec();
    append_test_custom_section(&mut module, "clusterflux.tasks", &descriptor);
    append_test_custom_section(&mut module, "clusterflux.edit", edit_marker.as_bytes());
    let digest = Digest::sha256(&module);
    (BASE64_STANDARD.encode(module), digest)
}

fn test_task_spec(
    tenant: &str,
    project: &str,
    process: &str,
    task: &str,
    epoch: u64,
    required_capabilities: impl IntoIterator<Item = Capability>,
) -> TaskSpec {
    test_task_spec_instance(
        tenant,
        project,
        process,
        task,
        task,
        epoch,
        required_capabilities,
    )
}

fn test_task_spec_instance(
    tenant: &str,
    project: &str,
    process: &str,
    task_definition: &str,
    task_instance: &str,
    epoch: u64,
    required_capabilities: impl IntoIterator<Item = Capability>,
) -> TaskSpec {
    TaskSpec {
        tenant: TenantId::from(tenant),
        project: ProjectId::from(project),
        process: ProcessId::from(process),
        task_definition: clusterflux_core::TaskDefinitionId::from(task_definition),
        task_instance: clusterflux_core::TaskInstanceId::from(task_instance),
        dispatch: TaskDispatch::CoordinatorNodeWasm {
            export: Some(task_definition.to_owned()),
            abi: WasmExportAbi::TaskV1,
        },
        environment_id: None,
        environment: None,
        environment_digest: None,
        required_capabilities: required_capabilities.into_iter().collect(),
        dependency_cache: None,
        source_snapshot: None,
        required_artifacts: Vec::new(),
        args: Vec::new(),
        vfs_epoch: epoch,
        failure_policy: Default::default(),
        bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
    }
}

trait AuthorizedTestTaskLaunch {
    fn handle_authorized_test_task_launch(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError>;
}

impl AuthorizedTestTaskLaunch for CoordinatorService {
    fn handle_authorized_test_task_launch(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
        let CoordinatorRequest::LaunchTask {
            task_spec,
            wait_for_node,
            artifact_path,
            wasm_module_base64,
            ..
        } = request
        else {
            panic!("authorized test task launch requires LaunchTask");
        };
        let tenant = task_spec.tenant.clone();
        let project = task_spec.project.clone();
        self.handle_launch_task_with_actor(
            tenant,
            project,
            WorkflowActor {
                kind: "task".to_owned(),
                user: None,
                agent: None,
                credential_kind: CredentialKind::TaskCredential,
                public_key_fingerprint: None,
                authenticated_without_browser: true,
                scopes: vec!["process:spawn-child".to_owned()],
            },
            task_spec,
            wait_for_node,
            artifact_path,
            wasm_module_base64,
        )
    }
}

fn register_test_task_assignment(
    service: &mut CoordinatorService,
    tenant: &str,
    project: &str,
    process: &str,
    node: &str,
    task_definition: &str,
    task_instance: &str,
    epoch: u64,
) {
    let task_spec = test_task_spec_instance(
        tenant,
        project,
        process,
        task_definition,
        task_instance,
        epoch,
        [],
    );
    let assignment = TaskAssignment {
        tenant: TenantId::from(tenant),
        project: ProjectId::from(project),
        process: ProcessId::from(process),
        task: TaskInstanceId::from(task_instance),
        node: NodeId::from(node),
        epoch,
        artifact_path: format!("/vfs/artifacts/{task_instance}.bin"),
        task_spec,
        wasm_module_base64: test_wasm_module_base64(),
    };
    service
        .capture_task_restart_checkpoint(&assignment)
        .unwrap();
    service.active_tasks.insert(super::keys::task_control_key(
        &assignment.tenant,
        &assignment.project,
        &assignment.process,
        &assignment.node,
        &assignment.task,
    ));
}

fn test_agent_public_key() -> String {
    agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap()
}

fn signed_agent_workflow_request(
    request: &CoordinatorRequest,
    request_kind: &str,
    process: &str,
    task: Option<&str>,
    nonce: &str,
) -> AgentSignedRequest {
    let task = task.map(TaskInstanceId::from);
    let payload = serde_json::to_value(request).unwrap();
    let payload_digest = signed_request_payload_digest(&payload);
    sign_agent_workflow_request(
        TEST_AGENT_PRIVATE_KEY,
        AgentWorkflowScope {
            tenant: &TenantId::from("tenant"),
            project: &ProjectId::from("project"),
            agent: &AgentId::from("agent-ci"),
            request_kind,
            process: &ProcessId::from(process),
            task: task.as_ref(),
        },
        &payload_digest,
        nonce.to_owned(),
        unix_timestamp_seconds_for_tests(),
    )
    .unwrap()
}

fn with_signed_agent_workflow(
    mut request: CoordinatorRequest,
    request_kind: &str,
    process: &str,
    task: Option<&str>,
    nonce: &str,
) -> CoordinatorRequest {
    let signature = signed_agent_workflow_request(&request, request_kind, process, task, nonce);
    match &mut request {
        CoordinatorRequest::StartProcess {
            agent_signature, ..
        }
        | CoordinatorRequest::LaunchTask {
            agent_signature, ..
        } => *agent_signature = Some(signature),
        _ => panic!("agent signing helper only accepts agent workflow requests"),
    }
    request
}

fn test_node_private_key(node: &str) -> String {
    derive_ed25519_private_key_from_seed(&format!("test-node-key:{node}"))
}

fn test_node_public_key(node: &str) -> String {
    node_ed25519_public_key_from_private_key(&test_node_private_key(node)).unwrap()
}

fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest {
    let payload = json!({"type": "node_heartbeat", "node": node});
    signed_node_request_with_private_key(
        node,
        &test_node_private_key(node),
        "node_heartbeat",
        &signed_request_payload_digest(&payload),
        nonce,
    )
}

fn signed_node_heartbeat_with_private_key(
    node: &str,
    private_key: &str,
    nonce: &str,
) -> NodeSignedRequest {
    let payload = json!({"type": "node_heartbeat", "node": node});
    signed_node_request_with_private_key(
        node,
        private_key,
        "node_heartbeat",
        &signed_request_payload_digest(&payload),
        nonce,
    )
}

fn signed_node_request_with_private_key(
    node: &str,
    private_key: &str,
    request_kind: &str,
    payload_digest: &Digest,
    nonce: &str,
) -> NodeSignedRequest {
    sign_node_request(
        private_key,
        &NodeId::from(node),
        request_kind,
        payload_digest,
        nonce.to_owned(),
        unix_timestamp_seconds_for_tests(),
    )
    .unwrap()
}

fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest {
    let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap());
    let (node, request_kind) = match &request {
        CoordinatorRequest::ReportNodeCapabilities { node, .. } => {
            (node.clone(), "report_node_capabilities")
        }
        CoordinatorRequest::PollTaskAssignment { node, .. } => {
            (node.clone(), "poll_task_assignment")
        }
        CoordinatorRequest::PollArtifactTransfer { node, .. } => {
            (node.clone(), "poll_artifact_transfer")
        }
        CoordinatorRequest::UploadArtifactTransferChunk { node, .. } => {
            (node.clone(), "upload_artifact_transfer_chunk")
        }
        CoordinatorRequest::FailArtifactTransfer { node, .. } => {
            (node.clone(), "fail_artifact_transfer")
        }
        CoordinatorRequest::LaunchChildTask { node, .. } => (node.clone(), "launch_child_task"),
        CoordinatorRequest::JoinChildTask { node, .. } => (node.clone(), "join_child_task"),
        CoordinatorRequest::CompleteSourcePreparation { node, .. } => {
            (node.clone(), "complete_source_preparation")
        }
        CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"),
        CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"),
        CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"),
        CoordinatorRequest::ReportDebugState { node, .. } => (node.clone(), "report_debug_state"),
        CoordinatorRequest::ReportDebugProbeHit { node, .. } => {
            (node.clone(), "report_debug_probe_hit")
        }
        CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"),
        CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"),
        CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"),
        _ => panic!("test helper only signs node-originated requests"),
    };
    let nonce = TEST_NODE_REQUEST_NONCE
        .fetch_add(1, Ordering::Relaxed)
        .to_string();
    CoordinatorRequest::SignedNode {
        node: node.clone(),
        node_signature: signed_node_request_with_private_key(
            &node,
            &test_node_private_key(&node),
            request_kind,
            &payload_digest,
            &format!("node-request-{nonce}"),
        ),
        request: Box::new(request),
    }
}

trait SignedNodeRequestTestExt {
    fn handle_signed_node_request_auto(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError>;
}

impl SignedNodeRequestTestExt for CoordinatorService {
    fn handle_signed_node_request_auto(
        &mut self,
        request: CoordinatorRequest,
    ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
        self.handle_request(signed_node_request_auto(request))
    }
}

fn upload_pending_artifact_transfer(
    service: &mut CoordinatorService,
    node: &str,
    bytes: &[u8],
) -> ArtifactTransferAssignment {
    let mut first = None;
    loop {
        let CoordinatorResponse::ArtifactTransferAssignment {
            transfer: Some(transfer),
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected pending artifact reverse transfer");
        };
        assert_eq!(transfer.expected_digest, Digest::sha256(bytes));
        assert_eq!(transfer.expected_size_bytes, bytes.len() as u64);
        let start = transfer.offset as usize;
        let end = start
            .saturating_add(transfer.max_chunk_bytes as usize)
            .min(bytes.len());
        let content = &bytes[start..end];
        let response = service
            .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                transfer_id: transfer.transfer_id.clone(),
                artifact: transfer.artifact.as_str().to_owned(),
                offset: transfer.offset,
                content_base64: BASE64_STANDARD.encode(content),
                chunk_digest: Digest::sha256(content),
                eof: end == bytes.len(),
            })
            .unwrap();
        let complete = matches!(
            response,
            CoordinatorResponse::ArtifactTransferChunkAccepted { complete: true, .. }
        );
        first.get_or_insert_with(|| transfer.clone());
        if complete {
            return first.unwrap();
        }
    }
}

fn unix_timestamp_seconds_for_tests() -> u64 {
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .unwrap_or(0)
}

fn write_coordinator_wire_request(
    stream: &mut TcpStream,
    request: &CoordinatorRequest,
    request_id: &str,
) {
    let payload = serde_json::to_value(request).unwrap();
    let wire_request = coordinator_wire_request(request_id, payload);
    serde_json::to_writer(&mut *stream, &wire_request).unwrap();
    stream.write_all(b"\n").unwrap();
    stream.flush().unwrap();
}

#[test]
fn service_creates_selects_and_lists_signed_in_user_projects() {
    let mut service = CoordinatorService::new(7);

    let CoordinatorResponse::ProjectCreated { project, actor } = service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-a".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project creation");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(project.tenant, TenantId::from("tenant-a"));
    assert_eq!(project.id, ProjectId::from("project-a"));
    assert_eq!(project.name, "Demo");

    let CoordinatorResponse::ProjectSelected { project, actor } = service
        .handle_request(CoordinatorRequest::SelectProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-a".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project selection");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(project.id, ProjectId::from("project-a"));

    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-b".to_owned(),
            actor_user: "user-b".to_owned(),
            project: "project-b".to_owned(),
            name: "Other".to_owned(),
        })
        .unwrap();

    let CoordinatorResponse::Projects { projects, actor } = service
        .handle_request(CoordinatorRequest::ListProjects {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected project list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(projects.len(), 1);
    assert_eq!(projects[0].id, ProjectId::from("project-a"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::SelectProject {
            tenant: "tenant-a".to_owned(),
            actor_user: "user-a".to_owned(),
            project: "project-b".to_owned(),
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("tenant scope"));
}

#[test]
fn authenticated_envelope_derives_user_scope_from_cli_session() {
    let mut service = CoordinatorService::new(7);
    service
        .issue_cli_session(
            TenantId::from("tenant-a"),
            ProjectId::from("project-a"),
            UserId::from("user-a"),
            "cli-session-secret",
            None,
        )
        .unwrap();

    let CoordinatorResponse::AuthStatus {
        tenant,
        project,
        actor,
        authenticated,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap()
    else {
        panic!("expected authenticated auth status");
    };
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));
    assert!(authenticated);

    let CoordinatorResponse::ProjectCreated { project, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CreateProject {
                project: "project-b".to_owned(),
                name: "From Session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated project creation");
    };
    assert_eq!(project.tenant, TenantId::from("tenant-a"));
    assert_eq!(project.id, ProjectId::from("project-b"));
    assert_eq!(actor, UserId::from("user-a"));

    let CoordinatorResponse::Projects { projects, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListProjects,
        })
        .unwrap()
    else {
        panic!("expected authenticated project list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(projects.len(), 2);
    assert!(projects
        .iter()
        .any(|project| project.id == ProjectId::from("project-a")));
    assert!(projects
        .iter()
        .any(|project| project.id == ProjectId::from("project-b")));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RegisterAgentPublicKey {
                agent: "agent-session".to_owned(),
                public_key: "agent-session-key-v1".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key registration");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(record.tenant, TenantId::from("tenant-a"));
    assert_eq!(record.project, ProjectId::from("project-a"));
    assert_eq!(record.user, UserId::from("user-a"));
    assert_eq!(record.agent, AgentId::from("agent-session"));

    let CoordinatorResponse::AgentPublicKeys { records, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListAgentPublicKeys,
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(records.len(), 1);
    assert_eq!(records[0].project, ProjectId::from("project-a"));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeAgentPublicKey {
                agent: "agent-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated agent key revocation");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert!(record.revoked);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "session-node".to_owned(),
            public_key: test_node_public_key("session-node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "session-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListNodeDescriptors,
        })
        .unwrap()
    else {
        panic!("expected authenticated node descriptor list");
    };
    assert_eq!(actor, UserId::from("user-a"));
    assert_eq!(descriptors.len(), 1);
    assert_eq!(descriptors[0].id, NodeId::from("session-node"));

    let CoordinatorResponse::TaskPlacement { placement } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ScheduleTask {
                environment: None,
                environment_digest: None,
                required_capabilities: vec![Capability::Command],
                dependency_cache: None,
                source_snapshot: None,
                required_artifacts: vec![],
                prefer_node: Some("session-node".to_owned()),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated task placement");
    };
    assert_eq!(placement.node, NodeId::from("session-node"));

    let CoordinatorResponse::ProcessStarted { process, actor, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                process: "vp-session".to_owned(),
                restart: false,
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated process start");
    };
    assert_eq!(process, ProcessId::from("vp-session"));
    assert_eq!(actor.user, Some(UserId::from("user-a")));

    let denied_external_task = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::LaunchTask {
                task_spec: Box::new(test_task_spec(
                    "tenant-a",
                    "project-a",
                    "vp-session",
                    "task-session",
                    7,
                    [Capability::Command],
                )),
                wait_for_node: false,
                artifact_path: "/vfs/artifacts/session.txt".to_owned(),
                wasm_module_base64: test_wasm_module_base64(),
            },
        })
        .unwrap_err();
    assert!(denied_external_task
        .to_string()
        .contains("external callers may launch only EntrypointV1"));

    let CoordinatorResponse::TaskLaunched {
        process,
        task,
        actor,
        assignment,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant-a",
                "project-a",
                "vp-session",
                "task-session",
                7,
                [Capability::Command],
            ),
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/session.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected authenticated task launch");
    };
    assert_eq!(process, ProcessId::from("vp-session"));
    assert_eq!(task, TaskInstanceId::from("task-session"));
    assert_eq!(actor.kind, "task");
    assert_eq!(assignment.tenant, TenantId::from("tenant-a"));
    assert_eq!(assignment.project, ProjectId::from("project-a"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "session-node".to_owned(),
            process: "vp-session".to_owned(),
            epoch: 7,
        })
        .unwrap();
    let artifact_bytes = "session artifact";
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            process: "vp-session".to_owned(),
            node: "session-node".to_owned(),
            task: "task-session".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: artifact_bytes.len() as u64,
            stderr_bytes: 0,
            stdout_tail: artifact_bytes.to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/session.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListTaskEvents {
                process: Some("vp-session".to_owned()),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated task event list");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].tenant, TenantId::from("tenant-a"));
    assert_eq!(events[0].project, ProjectId::from("project-a"));

    service
        .issue_cli_session(
            TenantId::from("tenant-b"),
            ProjectId::from("project-victim"),
            UserId::from("user-b"),
            "victim-cli-session-secret",
            None,
        )
        .unwrap();
    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "victim-cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                process: "vp-victim".to_owned(),
                restart: false,
            },
        })
        .unwrap();
    let cross_tenant_events = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::ListTaskEvents {
                process: Some("vp-victim".to_owned()),
            },
        })
        .unwrap_err();
    assert!(cross_tenant_events
        .to_string()
        .contains("outside the virtual process tenant/project scope"));

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
                artifact: "session.txt".to_owned(),
                max_bytes: 1024,
                ttl_seconds: 60,
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated artifact link");
    };
    assert_eq!(link.tenant, TenantId::from("tenant-a"));
    assert_eq!(link.project, ProjectId::from("project-a"));
    assert_eq!(link.actor, Actor::User(UserId::from("user-a")));

    let CoordinatorResponse::ProcessCancellationRequested { process, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::CancelProcess {
                process: "vp-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated process cancellation");
    };
    assert_eq!(process, ProcessId::from("vp-session"));

    let CoordinatorResponse::DebugAttach { actor, .. } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::DebugAttach {
                process: "vp-session".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated debug attach");
    };
    assert_eq!(actor, UserId::from("user-a"));

    let CoordinatorResponse::NodeCredentialRevoked {
        node,
        tenant,
        project,
        actor,
        descriptor_removed,
        ..
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeNodeCredential {
                node: "session-node".to_owned(),
            },
        })
        .unwrap()
    else {
        panic!("expected authenticated node credential revocation");
    };
    assert_eq!(node, NodeId::from("session-node"));
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));
    assert!(descriptor_removed);

    let rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "wrong-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(rejected.to_string().contains("not recognized"));

    let expired = service
        .issue_cli_session(
            TenantId::from("tenant-a"),
            ProjectId::from("project-b"),
            UserId::from("user-a"),
            "expired-cli-session-secret",
            Some(1),
        )
        .unwrap();
    assert_eq!(expired.expires_at_epoch_seconds, Some(1));
    let expired_rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "expired-cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(expired_rejected.to_string().contains("expired"));

    let CoordinatorResponse::CliSessionRevoked {
        tenant,
        project,
        actor,
    } = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::RevokeCliSession,
        })
        .unwrap()
    else {
        panic!("expected CLI session revocation");
    };
    assert_eq!(tenant, TenantId::from("tenant-a"));
    assert_eq!(project, ProjectId::from("project-a"));
    assert_eq!(actor, UserId::from("user-a"));

    let revoked_rejected = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "cli-session-secret".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(revoked_rejected.to_string().contains("revoked"));
}

#[test]
fn service_reports_and_enforces_public_admin_tenant_suspension() {
    let mut service = CoordinatorService::new_with_admin_token(7, "admin-token");

    let CoordinatorResponse::AuthStatus {
        tenant,
        project,
        actor,
        authenticated,
        account_status,
        suspended,
        disabled,
        sanitized_reason,
        private_moderation_details_exposed,
        signup_failure_details_exposed,
        ..
    } = service
        .handle_request(CoordinatorRequest::AuthStatus {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected auth status");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(project, ProjectId::from("project"));
    assert_eq!(actor, UserId::from("user"));
    assert!(authenticated);
    assert_eq!(account_status, "active");
    assert!(!suspended);
    assert!(!disabled);
    assert!(sanitized_reason.is_none());
    assert!(!private_moderation_details_exposed);
    assert!(!signup_failure_details_exposed);

    for (tenant, policy_name, expected_status, expected_reason) in [
        (
            "manual-tenant",
            "tenant:manual_review",
            "manual_review",
            "account or tenant is pending hosted review",
        ),
        (
            "disabled-tenant",
            "tenant:disabled",
            "disabled",
            "account or tenant is disabled by hosted policy",
        ),
        (
            "deleted-tenant",
            "tenant:deleted",
            "deleted",
            "account or tenant is no longer active",
        ),
    ] {
        service.coordinator.upsert_service_policy_record(
            TenantId::from(tenant),
            policy_name,
            Digest::from_parts([tenant.as_bytes(), policy_name.as_bytes()]),
        );
        let CoordinatorResponse::AuthStatus {
            account_status,
            suspended,
            disabled,
            deleted,
            manual_review,
            sanitized_reason,
            next_actions,
            private_moderation_details_exposed,
            signup_failure_details_exposed,
            ..
        } = service
            .handle_request(CoordinatorRequest::AuthStatus {
                tenant: tenant.to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
            })
            .unwrap()
        else {
            panic!("expected inactive auth status");
        };
        assert_eq!(account_status, expected_status);
        assert_eq!(suspended, expected_status == "suspended");
        assert_eq!(disabled, expected_status == "disabled");
        assert_eq!(deleted, expected_status == "deleted");
        assert_eq!(manual_review, expected_status == "manual_review");
        assert_eq!(sanitized_reason.as_deref(), Some(expected_reason));
        assert!(next_actions
            .iter()
            .any(|action| action.contains("hosted operator")));
        assert!(!private_moderation_details_exposed);
        assert!(!signup_failure_details_exposed);
    }

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-1",
    );
    let admin_status_request = CoordinatorRequest::AdminStatus {
        tenant: "tenant".to_owned(),
        actor_user: "admin".to_owned(),
        admin_proof,
        admin_nonce,
        issued_at_epoch_seconds,
    };
    let CoordinatorResponse::AdminStatus {
        tenant,
        actor,
        suspended,
        safe_default,
    } = service
        .handle_request(admin_status_request.clone())
        .unwrap()
    else {
        panic!("expected admin status");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(actor, UserId::from("admin"));
    assert!(!suspended);
    assert_eq!(safe_default, "read_only");
    let replayed_admin_status = service.handle_request(admin_status_request).unwrap_err();
    assert!(replayed_admin_status
        .to_string()
        .contains("nonce was already used"));

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "suspend_tenant",
        "admin-tenant",
        "admin",
        "tenant",
        "admin-suspend-1",
    );
    let CoordinatorResponse::TenantSuspended {
        tenant,
        actor,
        policy,
    } = service
        .handle_request(CoordinatorRequest::SuspendTenant {
            tenant: "admin-tenant".to_owned(),
            actor_user: "admin".to_owned(),
            target_tenant: "tenant".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap()
    else {
        panic!("expected tenant suspension");
    };
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(actor, UserId::from("admin"));
    assert_eq!(policy.name, "tenant:suspended");

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-2",
    );
    let CoordinatorResponse::AdminStatus { suspended, .. } = service
        .handle_request(CoordinatorRequest::AdminStatus {
            tenant: "tenant".to_owned(),
            actor_user: "admin".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap()
    else {
        panic!("expected suspended admin status");
    };
    assert!(suspended);

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "admin-token",
        "admin_status",
        "tenant",
        "admin",
        "tenant",
        "admin-status-unconfigured",
    );
    let missing_admin_credential = CoordinatorService::new(7)
        .handle_request(CoordinatorRequest::AdminStatus {
            tenant: "tenant".to_owned(),
            actor_user: "admin".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap_err();
    assert!(missing_admin_credential
        .to_string()
        .contains("admin credential is not configured"));

    let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request(
        "wrong-token",
        "suspend_tenant",
        "admin-tenant",
        "admin",
        "other-tenant",
        "admin-suspend-wrong",
    );
    let invalid_admin_credential = service
        .handle_request(CoordinatorRequest::SuspendTenant {
            tenant: "admin-tenant".to_owned(),
            actor_user: "admin".to_owned(),
            target_tenant: "other-tenant".to_owned(),
            admin_proof,
            admin_nonce,
            issued_at_epoch_seconds,
        })
        .unwrap_err();
    assert!(invalid_admin_credential
        .to_string()
        .contains("admin request proof is invalid"));

    let CoordinatorResponse::AuthStatus {
        account_status,
        suspended,
        disabled,
        sanitized_reason,
        next_actions,
        private_moderation_details_exposed,
        signup_failure_details_exposed,
        ..
    } = service
        .handle_request(CoordinatorRequest::AuthStatus {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected suspended auth status");
    };
    assert_eq!(account_status, "suspended");
    assert!(suspended);
    assert!(!disabled);
    assert_eq!(
        sanitized_reason.as_deref(),
        Some("account or tenant is suspended by hosted policy")
    );
    assert!(next_actions
        .iter()
        .any(|action| action.contains("hosted operator")));
    assert!(!private_moderation_details_exposed);
    assert!(!signup_failure_details_exposed);

    let create = service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap_err();
    assert!(create.to_string().contains("tenant is suspended"));

    let start = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(start.to_string().contains("tenant is suspended"));
}

#[test]
fn service_manages_project_scoped_agent_public_keys() {
    let mut service = CoordinatorService::new(7);

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: "agent-key-v1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key registration");
    };
    assert_eq!(actor, UserId::from("user"));
    assert_eq!(record.tenant, TenantId::from("tenant"));
    assert_eq!(record.project, ProjectId::from("project"));
    assert_eq!(record.user, UserId::from("user"));
    assert_eq!(record.agent, AgentId::from("agent-ci"));
    assert_eq!(record.version, 1);
    assert!(!record.revoked);
    assert_eq!(record.scopes, vec!["project:read", "project:run"]);
    assert!(!record.human_account_creation_privilege);
    assert!(!record.browser_interaction_required_each_run);

    let CoordinatorResponse::AgentPublicKey { record, .. } = service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: "agent-key-v2".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key rotation by re-add");
    };
    assert_eq!(record.version, 2);
    assert_eq!(record.public_key, "agent-key-v2");

    let CoordinatorResponse::AgentPublicKeys { records, actor } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key list");
    };
    assert_eq!(actor, UserId::from("user"));
    assert_eq!(records.len(), 1);
    assert_eq!(records[0].agent, AgentId::from("agent-ci"));

    let CoordinatorResponse::AgentPublicKeys { records, .. } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "other-user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected foreign user agent public key list");
    };
    assert!(records.is_empty());

    let CoordinatorResponse::AgentPublicKeys { records, .. } = service
        .handle_request(CoordinatorRequest::ListAgentPublicKeys {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected foreign tenant agent public key list");
    };
    assert!(records.is_empty());

    let foreign_user_revoke = service
        .handle_request(CoordinatorRequest::RevokeAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "other-user".to_owned(),
            agent: "agent-ci".to_owned(),
        })
        .unwrap_err();
    assert!(foreign_user_revoke
        .to_string()
        .contains("signed-in user scope"));

    let CoordinatorResponse::AgentPublicKey { record, actor } = service
        .handle_request(CoordinatorRequest::RevokeAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected agent public key revocation");
    };
    assert_eq!(actor, UserId::from("user"));
    assert!(record.revoked);
}

#[test]
fn service_runs_agent_workflows_with_scoped_key_attribution() {
    let mut service = CoordinatorService::new(7);
    let agent_public_key = test_agent_public_key();
    let agent_fingerprint = Digest::sha256(&agent_public_key);

    service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: agent_public_key,
        })
        .unwrap();

    let fingerprint_only = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: Some("agent-ci".to_owned()),
            agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
            agent_signature: None,
            process: "vp-agent-fingerprint-only".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(fingerprint_only.to_string().contains("signed request"));

    let wrong_fingerprint = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(Digest::sha256("other-key")),
                agent_signature: None,
                process: "vp-agent-bad-key".to_owned(),
                restart: false,
            },
            "start_process",
            "vp-agent-bad-key",
            None,
            "bad-fingerprint-nonce",
        ))
        .unwrap_err();
    assert!(wrong_fingerprint.to_string().contains("fingerprint"));

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ProcessStarted {
        actor: start_actor, ..
    } = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                process: "vp-agent".to_owned(),
                restart: false,
            },
            "start_process",
            "vp-agent",
            None,
            "start-nonce",
        ))
        .unwrap()
    else {
        panic!("expected agent-authenticated process start");
    };
    assert_agent_workflow_actor(&start_actor, &agent_fingerprint);

    let replay = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::StartProcess {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                process: "vp-agent".to_owned(),
                restart: true,
            },
            "start_process",
            "vp-agent",
            None,
            "start-nonce",
        ))
        .unwrap_err();
    assert!(replay.to_string().contains("nonce"));

    let denied_external_task = service
        .handle_request(with_signed_agent_workflow(
            CoordinatorRequest::LaunchTask {
                task_spec: test_task_spec(
                    "tenant",
                    "project",
                    "vp-agent",
                    "compile-linux",
                    7,
                    [Capability::Command],
                ),
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: None,
                actor_agent: Some("agent-ci".to_owned()),
                agent_public_key_fingerprint: Some(agent_fingerprint.clone()),
                agent_signature: None,
                wait_for_node: false,
                artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
                wasm_module_base64: test_wasm_module_base64(),
            },
            "launch_task",
            "vp-agent",
            Some("compile-linux"),
            "launch-nonce",
        ))
        .unwrap_err();
    assert!(denied_external_task
        .to_string()
        .contains("external callers may launch only EntrypointV1"));
}

#[test]
fn signed_node_and_agent_requests_reject_body_modification() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-bound-body".to_owned(),
            public_key: test_node_public_key("worker-bound-body"),
        })
        .unwrap();

    let original_node_request = CoordinatorRequest::ReportNodeCapabilities {
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        node: "worker-bound-body".to_owned(),
        capabilities: linux_capabilities(),
        cached_environment_digests: Vec::new(),
        dependency_cache_digests: Vec::new(),
        source_snapshots: Vec::new(),
        artifact_locations: Vec::new(),
        direct_connectivity: true,
        online: true,
    };
    let original_node_digest =
        signed_request_payload_digest(&serde_json::to_value(&original_node_request).unwrap());
    let node_signature = signed_node_request_with_private_key(
        "worker-bound-body",
        &test_node_private_key("worker-bound-body"),
        "report_node_capabilities",
        &original_node_digest,
        "node-body-modification",
    );
    let mut modified_node_request = original_node_request;
    let CoordinatorRequest::ReportNodeCapabilities { online, .. } = &mut modified_node_request
    else {
        unreachable!();
    };
    *online = false;
    let modified_node = service
        .handle_request(CoordinatorRequest::SignedNode {
            node: "worker-bound-body".to_owned(),
            node_signature,
            request: Box::new(modified_node_request),
        })
        .unwrap_err();
    assert!(modified_node.to_string().contains("signature"));

    let agent_public_key = test_agent_public_key();
    let agent_fingerprint = Digest::sha256(&agent_public_key);
    service
        .handle_request(CoordinatorRequest::RegisterAgentPublicKey {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            user: "user".to_owned(),
            agent: "agent-ci".to_owned(),
            public_key: agent_public_key,
        })
        .unwrap();
    let original_agent_request = CoordinatorRequest::StartProcess {
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        actor_user: None,
        actor_agent: Some("agent-ci".to_owned()),
        agent_public_key_fingerprint: Some(agent_fingerprint),
        agent_signature: None,
        process: "vp-agent-bound-body".to_owned(),
        restart: false,
    };
    let agent_signature = signed_agent_workflow_request(
        &original_agent_request,
        "start_process",
        "vp-agent-bound-body",
        None,
        "agent-body-modification",
    );
    let mut modified_agent_request = original_agent_request;
    let CoordinatorRequest::StartProcess {
        restart,
        agent_signature: request_signature,
        ..
    } = &mut modified_agent_request
    else {
        unreachable!();
    };
    *restart = true;
    *request_signature = Some(agent_signature);
    let modified_agent = service.handle_request(modified_agent_request).unwrap_err();
    assert!(modified_agent.to_string().contains("signature"));
}

#[test]
fn service_checks_spawn_quota_before_process_or_task_work_starts() {
    let mut service = CoordinatorService::new(7);
    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 2)]),
    });

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ProcessStarted { charged_spawns, .. } = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-quota".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start within spawn quota");
    };
    assert_eq!(charged_spawns, 1);

    let CoordinatorResponse::TaskLaunched { charged_spawns, .. } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-quota",
                "compile-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected task launch within spawn quota");
    };
    assert_eq!(charged_spawns, 2);

    let denied_task = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-quota",
                "compile-linux-denied",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/denied.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(denied_task.to_string().contains("Spawn"));
    let now_epoch_seconds = service.current_epoch_seconds().unwrap();
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            now_epoch_seconds,
        ),
        2
    );

    let denied_task_key = task_control_key(
        &TenantId::from("tenant"),
        &ProjectId::from("project"),
        &ProcessId::from("vp-quota"),
        &NodeId::from("worker-linux"),
        &TaskInstanceId::from("compile-linux-denied"),
    );
    assert!(!service.active_tasks.contains(&denied_task_key));
    assert!(!service.task_placements.contains_key(&denied_task_key));
    assert_eq!(
        service
            .task_assignments
            .get(&(
                TenantId::from("tenant"),
                ProjectId::from("project"),
                NodeId::from("worker-linux"),
            ))
            .map(VecDeque::len),
        Some(1)
    );

    let other_project_process = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-denied".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        other_project_process,
        CoordinatorResponse::ProcessStarted { .. }
    ));
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("other-project"),
            now_epoch_seconds,
        ),
        1
    );
    assert_eq!(
        service.quota.used_workflow_spawns(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            now_epoch_seconds,
        ),
        2
    );
}

#[test]
fn project_quota_resets_at_the_configured_window_boundary() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::Spawn, 1);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::Spawn, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();

    service.set_server_time(59);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-window-one".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AbortProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-window-one".to_owned(),
        })
        .unwrap();

    let exhausted = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-still-window-one".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(exhausted.to_string().contains("Spawn"));

    service.set_server_time(60);
    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-window-two".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted {
            charged_spawns: 1,
            ..
        }
    ));
    assert_eq!(service.quota.active_meter_count(), 1);
}

#[test]
fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::ApiCall, 1);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::ApiCall, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();
    service.set_server_time(30);
    for (project, secret) in [("project-a", "session-a"), ("project-b", "session-b")] {
        service
            .issue_cli_session(
                TenantId::from("tenant"),
                ProjectId::from(project),
                UserId::from("user"),
                secret,
                None,
            )
            .unwrap();
    }

    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    let denied = service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-a".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap_err();
    assert!(denied.to_string().contains("ApiCall"));

    service
        .handle_request(CoordinatorRequest::Authenticated {
            session_secret: "session-b".to_owned(),
            request: AuthenticatedCoordinatorRequest::AuthStatus,
        })
        .unwrap();
    assert_eq!(
        service
            .quota
            .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-a"), 30,),
        1
    );
    assert_eq!(
        service
            .quota
            .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-b"), 30,),
        1
    );
}

#[test]
fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() {
    let mut limits = ResourceLimits::unlimited();
    limits.limits.insert(LimitKind::LogBytes, 4);
    let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap();
    let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota(
        7,
        "test-admin-token",
        None,
        quota,
    )
    .unwrap();
    service.set_server_time(30);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "task".to_owned(),
            stdout_bytes: 3,
            stderr_bytes: 1,
            stdout_tail: "out".to_owned(),
            stderr_tail: "e".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: false,
        })
        .unwrap();
    let denied = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "task".to_owned(),
            stdout_bytes: 1,
            stderr_bytes: 0,
            stdout_tail: "x".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: true,
        })
        .unwrap_err();
    assert!(denied.to_string().contains("LogBytes"));
    assert_eq!(
        service
            .quota
            .used_log_bytes(&TenantId::from("tenant"), &ProjectId::from("project"), 30,),
        4
    );
}

#[test]
fn service_attaches_node_starts_process_and_records_scoped_task_event() {
    let mut service = CoordinatorService::new(7);

    let attached = service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    assert!(matches!(attached, CoordinatorResponse::NodeAttached { .. }));

    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    assert_eq!(
        started,
        CoordinatorResponse::ProcessStarted {
            process: ProcessId::from("process"),
            epoch: 7,
            actor: WorkflowActor {
                kind: "user".to_owned(),
                user: Some(UserId::from("user")),
                agent: None,
                credential_kind: CredentialKind::BrowserSession,
                public_key_fingerprint: None,
                authenticated_without_browser: false,
                scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
            },
            charged_spawns: 1,
        }
    );

    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")),
        })
        .unwrap();
    assert_eq!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 7,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 12,
            stderr_bytes: 0,
            stdout_tail: "build ok".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("artifact")),
            artifact_size_bytes: Some(12),
            result: None,
        })
        .unwrap();

    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            events_recorded: 1
        }
    );
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].node, NodeId::from("node"));
    assert_eq!(events[0].stdout_tail, "build ok");
    assert_eq!(events[0].stderr_tail, "");
    assert!(!events[0].stdout_truncated);

    let oversized_tail = "x".repeat(MAX_TASK_LOG_TAIL_BYTES + 1);
    let oversized_log = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            stdout_bytes: oversized_tail.len() as u64,
            stderr_bytes: 0,
            stdout_tail: oversized_tail.clone(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            backpressured: false,
        })
        .unwrap_err();
    assert!(oversized_log.to_string().contains("stdout_tail"));

    let oversized_completion = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: oversized_tail.len() as u64,
            stderr_bytes: 0,
            stdout_tail: oversized_tail,
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap_err();
    assert!(oversized_completion.to_string().contains("stdout_tail"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap_err();
    assert!(cross_tenant
        .to_string()
        .contains("outside the virtual process tenant/project scope"));

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("other-process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert!(events.is_empty());
}

#[test]
fn service_revokes_node_credentials_and_live_descriptors() {
    let mut service = CoordinatorService::new(7);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::NodeCredentialRevoked {
        node,
        tenant,
        project,
        actor,
        descriptor_removed,
        queued_assignments_removed,
    } = service
        .handle_request(CoordinatorRequest::RevokeNodeCredential {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            node: "node".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node credential revocation");
    };
    assert_eq!(node, NodeId::from("node"));
    assert_eq!(tenant, TenantId::from("tenant"));
    assert_eq!(project, ProjectId::from("project"));
    assert_eq!(actor, UserId::from("user"));
    assert!(descriptor_removed);
    assert_eq!(queued_assignments_removed, 0);

    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")),
        })
        .unwrap_err();
    assert!(heartbeat.to_string().contains("not enrolled"));

    let CoordinatorResponse::NodeDescriptors { descriptors, .. } = service
        .handle_request(CoordinatorRequest::ListNodeDescriptors {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node descriptors");
    };
    assert!(descriptors.is_empty());
}

#[test]
fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() {
    let mut service = CoordinatorService::new(7);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );

    let cancelled = service
        .handle_request(CoordinatorRequest::CancelTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        cancelled,
        CoordinatorResponse::TaskCancellationRequested {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            node: NodeId::from("node"),
        }
    );

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: true,
            abort_requested: false,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: Some(TaskTerminalState::Cancelled),
            status_code: None,
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events[0].terminal_state, TaskTerminalState::Cancelled);

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: false,
            abort_requested: false,
        }
    );
}

#[test]
fn service_authorizes_debug_attach_through_public_api() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let CoordinatorResponse::DebugAttach {
        process,
        actor,
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected debug attach authorization");
    };
    assert_eq!(process, ProcessId::from("process"));
    assert_eq!(actor, UserId::from("user"));
    assert!(authorization.allowed);
    assert_eq!(audit_event.operation, "debug_attach");
    assert_eq!(audit_event.actor, UserId::from("user"));
    assert!(audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    let CoordinatorResponse::DebugAttach {
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected denied debug attach authorization");
    };
    assert!(!authorization.allowed);
    assert!(authorization.reason.contains("explicit project permission"));
    assert!(!audit_event.allowed);
    assert_eq!(audit_event.charged_debug_read_bytes, 0);
    assert_eq!(charged_debug_read_bytes, 0);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    let CoordinatorResponse::DebugAttach {
        authorization,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::DebugAttach {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected cross-tenant debug attach denial");
    };
    assert!(!authorization.allowed);
    assert!(authorization.reason.contains("tenant or project"));
    assert!(!audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, 0);
    assert_eq!(used_debug_read_bytes, 0);
    assert_eq!(service.debug_audit_events.len(), 3);
}

#[test]
fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![],
            dependency_cache_digests: vec![],
            source_snapshots: vec![],
            artifact_locations: vec![],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "compile-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/out.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();
    service.main_runtime.controls.insert(
        process_control_key(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            &ProcessId::from("process"),
        ),
        super::main_runtime::CoordinatorMainControl {
            task_definition: clusterflux_core::TaskDefinitionId::from("completed-main"),
            task_instance: TaskInstanceId::from("ti:process:main"),
            abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
            debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
            state: "completed".to_owned(),
            stopped_probe_symbol: None,
            handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
            launch_id: 1,
        },
    );
    let CoordinatorResponse::DebugBreakpoints {
        probe_symbols,
        hit_epoch,
        ..
    } = service
        .handle_request(CoordinatorRequest::SetDebugBreakpoints {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()],
        })
        .unwrap()
    else {
        panic!("expected debug breakpoints response");
    };
    assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
    assert_eq!(hit_epoch, None);

    let CoordinatorResponse::DebugProbeHit {
        breakpoint_matched,
        debug_epoch,
        probe_symbol,
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugProbeHit {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            probe_symbol: "clusterflux.probe.compile_linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected executing Wasm probe hit response");
    };
    assert!(breakpoint_matched);
    assert_eq!(debug_epoch, Some(1));
    assert_eq!(probe_symbol, "clusterflux.probe.compile_linux");

    let CoordinatorResponse::DebugBreakpoints {
        hit_epoch,
        hit_task,
        hit_probe_symbol,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugBreakpoints {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected breakpoint hit status");
    };
    assert_eq!(hit_epoch, Some(1));
    assert_eq!(hit_task, Some(TaskInstanceId::from("compile-linux")));
    assert_eq!(
        hit_probe_symbol.as_deref(),
        Some("clusterflux.probe.compile_linux")
    );

    let CoordinatorResponse::DebugCommand {
        epoch: Some(1),
        command: Some(command),
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed node poll to receive freeze command");
    };
    assert_eq!(command, "freeze");

    let CoordinatorResponse::DebugCommand {
        epoch: None,
        command: None,
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("debug commands should be consumed after poll");
    };

    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        acknowledgements,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected pending debug epoch status");
    };
    assert!(!fully_frozen);
    assert!(!fully_resumed);
    assert!(acknowledgements.is_empty());
    let early_resume = service
        .handle_request(CoordinatorRequest::ResumeDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap_err();
    assert!(early_resume
        .to_string()
        .contains("no settled frozen participant set"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            epoch: 1,
            state: DebugAcknowledgementState::Frozen,
            stack_frames: vec!["compile_linux::wasm".to_owned()],
            local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())],
            task_args: vec![("target".to_owned(), "linux".to_owned())],
            handles: vec![("artifact".to_owned(), "pending".to_owned())],
            command_status: Some("frozen at Wasm safepoint".to_owned()),
            recent_output: vec![],
            message: None,
        })
        .unwrap();
    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        acknowledgements,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected frozen debug epoch status");
    };
    assert!(fully_frozen);
    assert!(!fully_resumed);
    assert_eq!(acknowledgements.len(), 1);
    assert_eq!(acknowledgements[0].state, DebugAcknowledgementState::Frozen);

    let CoordinatorResponse::DebugEpoch {
        epoch,
        command,
        affected_tasks,
        all_stop_requested,
        audit_event,
        ..
    } = service
        .handle_request(CoordinatorRequest::ResumeDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected debug epoch resume response");
    };
    assert_eq!(epoch, 1);
    assert_eq!(command, "resume");
    assert!(!all_stop_requested);
    assert_eq!(affected_tasks.len(), 1);
    assert_eq!(audit_event.operation, "resume_debug_epoch");

    let CoordinatorResponse::DebugCommand {
        epoch: Some(1),
        command: Some(command),
        ..
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed node poll to receive resume command");
    };
    assert_eq!(command, "resume");

    let CoordinatorResponse::DebugEpochStatus { fully_resumed, .. } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected pending resume status");
    };
    assert!(!fully_resumed);
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker".to_owned(),
            task: "compile-linux".to_owned(),
            epoch: 1,
            state: DebugAcknowledgementState::Running,
            stack_frames: vec![],
            local_values: vec![],
            task_args: vec![],
            handles: vec![],
            command_status: Some("running".to_owned()),
            recent_output: vec![],
            message: None,
        })
        .unwrap();
    let CoordinatorResponse::DebugEpochStatus {
        fully_frozen,
        fully_resumed,
        ..
    } = service
        .handle_request(CoordinatorRequest::InspectDebugEpoch {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            epoch: 1,
        })
        .unwrap()
    else {
        panic!("expected resumed debug epoch status");
    };
    assert!(!fully_frozen);
    assert!(fully_resumed);
}

#[test]
fn service_reports_task_restart_boundary_through_public_api() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Demo".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let denied = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap_err();
    assert!(denied.to_string().contains("task restart denied"));

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        restarted_task_instance,
        message,
        audit_event,
        charged_debug_read_bytes,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected task restart response");
    };
    assert!(!accepted);
    assert!(!clean_boundary_available);
    assert!(!active_task);
    assert!(!completed_event_observed);
    assert!(restarted_task_instance.is_none());
    assert!(requires_whole_process_restart);
    assert!(message.contains("not known"));
    assert_eq!(audit_event.operation, "restart_task");
    assert_eq!(audit_event.task, Some(TaskInstanceId::from("task")));
    assert!(audit_event.allowed);
    assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "task",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(initial_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected initial task assignment");
    };
    assert_eq!(initial_assignment.task, TaskInstanceId::from("task"));

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        message,
        audit_event,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected active task restart response");
    };
    assert!(!accepted);
    assert!(!clean_boundary_available);
    assert!(active_task);
    assert!(!completed_event_observed);
    assert!(requires_whole_process_restart);
    assert!(message.contains("still active"));
    assert_eq!(
        audit_event.charged_debug_read_bytes,
        DEBUG_CONTROL_READ_BYTES
    );
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 2);

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "worker-linux".to_owned(),
            task: "task".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::TaskRestart {
        accepted,
        clean_boundary_available,
        active_task,
        completed_event_observed,
        requires_whole_process_restart,
        restarted_task_instance,
        restarted_attempt_id,
        message,
        audit_event,
        used_debug_read_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
            task: "task".to_owned(),
            replacement_bundle: None,
        })
        .unwrap()
    else {
        panic!("expected completed task restart response");
    };
    assert!(accepted);
    assert!(clean_boundary_available);
    assert!(!active_task);
    assert!(completed_event_observed);
    assert!(!requires_whole_process_restart);
    let restarted_task_instance = restarted_task_instance.expect("restart returns logical id");
    let restarted_attempt_id = restarted_attempt_id.expect("restart returns a new attempt id");
    assert!(restarted_attempt_id.starts_with("ta_"));
    assert!(message.contains("from clean VFS entry boundary epoch 7"));
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(restarted_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected restarted task assignment");
    };
    assert_eq!(restarted_assignment.task, restarted_task_instance);
    assert_eq!(restarted_assignment.task, TaskInstanceId::from("task"));
    let mut expected_task_spec = initial_assignment.task_spec.clone();
    expected_task_spec.task_instance = restarted_task_instance;
    assert_eq!(restarted_assignment.task_spec, expected_task_spec);
    assert_eq!(
        restarted_assignment.wasm_module_base64,
        initial_assignment.wasm_module_base64
    );
    assert_eq!(
        audit_event.charged_debug_read_bytes,
        DEBUG_CONTROL_READ_BYTES
    );
    assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 3);
    assert_eq!(service.debug_audit_events.len(), 4);
}

#[test]
fn service_cancels_whole_process_and_blocks_new_task_launches() {
    let mut service = CoordinatorService::new(7);
    for node in ["node-a", "node-b"] {
        let capabilities = if node == "node-a" {
            linux_capabilities()
        } else {
            windows_capabilities()
        };
        service
            .handle_request(CoordinatorRequest::AttachNode {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                public_key: test_node_public_key(node),
            })
            .unwrap();
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                capabilities,
                cached_environment_digests: vec![],
                dependency_cache_digests: vec![],
                source_snapshots: vec![],
                artifact_locations: vec![],
                direct_connectivity: true,
                online: true,
            })
            .unwrap();
    }
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    for node in ["node-a", "node-b"] {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
                node: node.to_owned(),
                process: "process".to_owned(),
                epoch: 7,
            })
            .unwrap();
    }

    for (task, required_capability, preferred_node) in [
        ("compile-linux", Capability::Containers, "node-a"),
        ("link-windows", Capability::WindowsCommandDev, "node-b"),
    ] {
        let response = service
            .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
                task_spec: test_task_spec(
                    "tenant",
                    "project",
                    "process",
                    task,
                    7,
                    [required_capability],
                ),
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: Some("user".to_owned()),
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                wait_for_node: false,
                artifact_path: format!("/vfs/artifacts/{task}.txt"),
                wasm_module_base64: test_wasm_module_base64(),
            })
            .unwrap();
        let CoordinatorResponse::TaskLaunched { assignment, .. } = response else {
            panic!("expected task launch");
        };
        assert_eq!(assignment.node, NodeId::from(preferred_node));
    }

    let CoordinatorResponse::ProcessCancellationRequested {
        process,
        cancelled_tasks,
        affected_nodes,
    } = service
        .handle_request(CoordinatorRequest::CancelProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected process cancellation");
    };
    assert_eq!(process, ProcessId::from("process"));
    assert_eq!(cancelled_tasks.len(), 2);
    assert_eq!(
        affected_nodes,
        vec![NodeId::from("node-a"), NodeId::from("node-b")]
    );

    let control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node-a".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: true,
            abort_requested: false,
        }
    );

    let blocked = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "process",
                "package-linux",
                7,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/package-linux.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(blocked
        .to_string()
        .contains("virtual process is cancelling"));

    service
        .handle_request(CoordinatorRequest::AbortProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap();
    let abort_control = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node-a".to_owned(),
            task: "compile-linux".to_owned(),
        })
        .unwrap();
    assert_eq!(
        abort_control,
        CoordinatorResponse::TaskControl {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-linux"),
            cancel_requested: false,
            abort_requested: true,
        }
    );
}

#[test]
fn service_rejects_second_active_process_unless_restarting_same_process() {
    let mut service = CoordinatorService::new(7);
    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted { .. }
    ));

    let same_without_restart = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap_err();
    assert!(same_without_restart
        .to_string()
        .contains("already has active virtual process"));

    let other_process = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-b".to_owned(),
            restart: true,
        })
        .unwrap_err();
    assert!(other_process
        .to_string()
        .contains("already has active virtual process"));

    let retired_main_abort = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
    let process_key = process_control_key(
        &TenantId::from("tenant"),
        &ProjectId::from("project"),
        &ProcessId::from("process-a"),
    );
    service.main_runtime.controls.insert(
        process_key.clone(),
        super::main_runtime::CoordinatorMainControl {
            task_definition: clusterflux_core::TaskDefinitionId::from("build"),
            task_instance: TaskInstanceId::from("ti:process-a:main"),
            abort: std::sync::Arc::clone(&retired_main_abort),
            debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
            state: "running".to_owned(),
            stopped_probe_symbol: None,
            handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
            launch_id: 1,
        },
    );
    let restarted = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: true,
        })
        .unwrap();
    assert_eq!(
        restarted,
        CoordinatorResponse::ProcessStarted {
            process: ProcessId::from("process-a"),
            epoch: 7,
            actor: WorkflowActor {
                kind: "user".to_owned(),
                user: Some(UserId::from("user")),
                agent: None,
                credential_kind: CredentialKind::BrowserSession,
                public_key_fingerprint: None,
                authenticated_without_browser: false,
                scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
            },
            charged_spawns: 2,
        }
    );
    assert!(retired_main_abort.load(std::sync::atomic::Ordering::Acquire));
    assert!(!service.main_runtime.controls.contains_key(&process_key));
}

#[test]
fn quiescent_cooperative_cancel_releases_slot_immediately() {
    let mut service = CoordinatorService::new(17);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    service.record_task_completion_event(TaskCompletionEvent {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("process-a"),
        node: NodeId::from("node"),
        executor: TaskExecutor::Node,
        task_definition: clusterflux_core::TaskDefinitionId::from("old-task"),
        task: TaskInstanceId::from("ti:process-a:old"),
        attempt_id: None,
        placement: None,
        terminal_state: TaskTerminalState::Completed,
        status_code: Some(0),
        stdout_bytes: 0,
        stderr_bytes: 0,
        stdout_tail: String::new(),
        stderr_tail: String::new(),
        stdout_truncated: false,
        stderr_truncated: false,
        artifact_path: None,
        artifact_digest: None,
        artifact_size_bytes: None,
        result: Some(TaskBoundaryValue::SmallJson(json!("old"))),
    });
    service
        .record_debug_audit_event(
            TenantId::from("tenant"),
            ProjectId::from("project"),
            ProcessId::from("process-a"),
            None,
            UserId::from("user"),
            "old_debug_event",
            false,
            "old process incarnation",
        )
        .unwrap();

    service
        .handle_request(CoordinatorRequest::CancelProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process-a".to_owned(),
        })
        .unwrap();

    let CoordinatorResponse::ProcessStatuses { processes, .. } = service
        .handle_request(CoordinatorRequest::ListProcesses {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected live process statuses");
    };
    assert!(processes.is_empty());

    let started = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process-a".to_owned(),
            restart: false,
        })
        .unwrap();
    assert!(matches!(
        started,
        CoordinatorResponse::ProcessStarted { .. }
    ));
    assert!(service.task_events.iter().all(|event| {
        event.tenant != TenantId::from("tenant")
            || event.project != ProjectId::from("project")
            || event.process != ProcessId::from("process-a")
    }));
    assert!(service.debug_audit_events.iter().all(|event| {
        event.tenant != TenantId::from("tenant")
            || event.project != ProjectId::from("project")
            || event.process != ProcessId::from("process-a")
    }));
}

#[test]
fn aborted_process_accepts_signed_terminal_event_for_issued_task() {
    let mut service = CoordinatorService::new(17);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 17,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile",
        "compile-1",
        17,
    );

    let CoordinatorResponse::ProcessAborted { aborted_tasks, .. } = service
        .handle_request(CoordinatorRequest::AbortProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "process".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected process abort");
    };
    assert_eq!(aborted_tasks.len(), 1);

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-1".to_owned(),
            terminal_state: Some(TaskTerminalState::Cancelled),
            status_code: None,
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("compile-1"),
            events_recorded: 1,
        }
    );
    assert_eq!(
        service.task_events[0].task_definition,
        clusterflux_core::TaskDefinitionId::from("compile")
    );
    assert!(service.task_restart_checkpoints.is_empty());
}

#[test]
fn service_download_links_are_scoped_and_streaming_is_metered() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = b"0123456789abcdef0123456789abcdef";
    service.set_server_time(10);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 32,
            stderr_bytes: 0,
            stdout_tail: "artifact bytes".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 31)]),
    });
    let quota_denied_link = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(quota_denied_link
        .to_string()
        .contains("ArtifactDownloadBytes"));
    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 64)]),
    });

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    let private_node_link = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap();
    assert!(matches!(
        private_node_link,
        CoordinatorResponse::ArtifactDownloadLink { .. }
    ));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected download link");
    };

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    let private_node_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap();
    assert!(matches!(
        private_node_open,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    let CoordinatorResponse::ArtifactTransferAssignment {
        transfer: Some(pending_transfer),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected reverse transfer assignment");
    };
    let bad_chunk = service
        .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            transfer_id: pending_transfer.transfer_id,
            artifact: pending_transfer.artifact.as_str().to_owned(),
            offset: 0,
            content_base64: BASE64_STANDARD.encode(artifact_bytes),
            chunk_digest: Digest::sha256("wrong bytes"),
            eof: true,
        })
        .unwrap_err();
    assert!(bad_chunk.to_string().contains("chunk digest mismatch"));

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: vec!["app.txt".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    assert_eq!(link.tenant, TenantId::from("tenant"));
    assert_eq!(link.project, ProjectId::from("project"));
    assert_eq!(link.process, ProcessId::from("process"));
    assert_eq!(link.actor, Actor::User(UserId::from("user")));
    assert_eq!(link.max_bytes, 64);
    assert!(link.policy_context_digest.is_valid_sha256());
    assert_eq!(link.expires_at_epoch_seconds, 70);
    assert!(link.url_path.contains("/artifacts/tenant/project/process"));

    let cross_tenant = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(cross_tenant.to_string().contains("tenant mismatch"));

    let cross_project = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(cross_project.to_string().contains("project mismatch"));

    let cross_tenant_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "other".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_tenant_open.to_string().contains("tenant mismatch"));

    let cross_project_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_project_open.to_string().contains("project mismatch"));

    let guessed = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: Digest::sha256("guessed"),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(guessed.to_string().contains("token is invalid"));

    let cross_actor_open = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_actor_open.to_string().contains("token is invalid"));

    service.set_server_time(11);
    upload_pending_artifact_transfer(&mut service, "node", artifact_bytes);
    let CoordinatorResponse::ArtifactDownloadStream {
        streamed_bytes,
        charged_download_bytes,
        content_base64: Some(content_base64),
        content_source: Some(content_source),
        ..
    } = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap()
    else {
        panic!("expected download stream");
    };

    assert_eq!(streamed_bytes, 16);
    assert_eq!(charged_download_bytes, 16);
    assert_eq!(
        BASE64_STANDARD.decode(content_base64).unwrap(),
        &artifact_bytes[..16]
    );
    assert_eq!(content_source, "retaining_node_reverse_stream");

    let CoordinatorResponse::ArtifactDownloadLink {
        link: expiring_link,
    } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            ttl_seconds: 1,
        })
        .unwrap()
    else {
        panic!("expected expiring download link");
    };
    service.set_server_time(13);
    let expired = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: expiring_link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(expired.to_string().contains("expired"));
    service.set_server_time(11);

    let cross_actor_revoke = service
        .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "other-user".to_owned(),
            artifact: "app.txt".to_owned(),
            token_digest: link.scoped_token_digest.clone(),
        })
        .unwrap_err();
    assert!(cross_actor_revoke.to_string().contains("token is invalid"));

    let CoordinatorResponse::ArtifactDownloadLinkRevoked { link: revoked } = service
        .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            token_digest: link.scoped_token_digest.clone(),
        })
        .unwrap()
    else {
        panic!("expected revoked download link");
    };
    assert_eq!(revoked.scoped_token_digest, link.scoped_token_digest);

    let revoked = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 64,
            token_digest: link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(revoked.to_string().contains("revoked"));
}

#[test]
fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = b"0123456789abcdef";
    service.quota.set_download_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 16)]),
    });
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "compile-linux",
        "compile-linux",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 16,
            stderr_bytes: 0,
            stdout_tail: "artifact bytes".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/app.txt".to_owned()),
            artifact_digest: Some(Digest::sha256(artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: None,
        })
        .unwrap();

    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected initial download link");
    };

    let pending = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap();
    assert!(matches!(
        pending,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    upload_pending_artifact_transfer(&mut service, "node", artifact_bytes);
    let CoordinatorResponse::ArtifactDownloadStream {
        charged_download_bytes,
        ..
    } = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 16,
        })
        .unwrap()
    else {
        panic!("expected initial download stream");
    };
    assert_eq!(charged_download_bytes, 16);

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "node".to_owned(),
            process: "process".to_owned(),
            epoch: 7,
        })
        .unwrap();
    let after_reconnect = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(after_reconnect
        .to_string()
        .contains("ArtifactDownloadBytes"));

    let restarted_cli_retry = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            ttl_seconds: 60,
        })
        .unwrap_err();
    assert!(restarted_cli_retry
        .to_string()
        .contains("ArtifactDownloadBytes"));

    let cross_tenant_retry = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "other-tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_tenant_retry.to_string().contains("tenant mismatch"));

    let cross_project_retry = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "other-project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "app.txt".to_owned(),
            max_bytes: 16,
            token_digest: link.scoped_token_digest,
            chunk_bytes: 1,
        })
        .unwrap_err();
    assert!(cross_project_retry.to_string().contains("project mismatch"));
}

#[test]
fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() {
    let mut service = CoordinatorService::new(7);
    let artifact_bytes = vec![0x5a; 600_000];
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "node",
        "package",
        "package",
        7,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "package".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/large.bin".to_owned()),
            artifact_digest: Some(Digest::sha256(&artifact_bytes)),
            artifact_size_bytes: Some(artifact_bytes.len() as u64),
            result: Some(TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("large.bin"),
                digest: Digest::sha256(&artifact_bytes),
                size_bytes: artifact_bytes.len() as u64,
            })),
        })
        .unwrap();
    let CoordinatorResponse::ArtifactDownloadLink { link } = service
        .handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "large.bin".to_owned(),
            max_bytes: artifact_bytes.len() as u64,
            ttl_seconds: 60,
        })
        .unwrap()
    else {
        panic!("expected download link");
    };
    let pending = service
        .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            artifact: "large.bin".to_owned(),
            max_bytes: artifact_bytes.len() as u64,
            token_digest: link.scoped_token_digest.clone(),
            chunk_bytes: artifact_bytes.len() as u64,
        })
        .unwrap();
    assert!(matches!(
        pending,
        CoordinatorResponse::ArtifactDownloadStream {
            content_bytes_available: false,
            ..
        }
    ));
    upload_pending_artifact_transfer(&mut service, "node", &artifact_bytes);

    let mut downloaded = Vec::new();
    loop {
        let CoordinatorResponse::ArtifactDownloadStream {
            streamed_bytes,
            content_offset: Some(offset),
            content_eof,
            content_base64: Some(content),
            ..
        } = service
            .handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
                artifact: "large.bin".to_owned(),
                max_bytes: artifact_bytes.len() as u64,
                token_digest: link.scoped_token_digest.clone(),
                chunk_bytes: artifact_bytes.len() as u64,
            })
            .unwrap()
        else {
            panic!("expected verified reverse-stream content chunk");
        };
        assert_eq!(offset, downloaded.len() as u64);
        assert!(streamed_bytes <= super::artifacts::MAX_ARTIFACT_REVERSE_CHUNK_BYTES);
        downloaded.extend(BASE64_STANDARD.decode(content).unwrap());
        if content_eof {
            break;
        }
    }
    assert_eq!(downloaded, artifact_bytes);
    assert_eq!(
        service.quota.used_download_bytes(
            &TenantId::from("tenant"),
            &ProjectId::from("project"),
            service.current_epoch_seconds().unwrap(),
        ),
        600_000
    );
}

#[test]
fn windows_task_events_share_the_virtual_process_scope() {
    let mut service = CoordinatorService::new(7);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "windows-node".to_owned(),
            public_key: test_node_public_key("windows-node"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "windows-node".to_owned(),
            capabilities: windows_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env-windows-command-dev")],
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();
    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "process",
        "windows-node",
        "windows-command-dev",
        "windows-command-dev",
        7,
    );

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "process".to_owned(),
            node: "windows-node".to_owned(),
            task: "windows-command-dev".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 24,
            stderr_bytes: 0,
            stdout_tail: "windows output".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/windows-output.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("windows-artifact")),
            artifact_size_bytes: Some(24),
            result: None,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::TaskRecorded {
            process: ProcessId::from("process"),
            task: TaskInstanceId::from("windows-command-dev"),
            events_recorded: 1,
        }
    );

    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert_eq!(events.len(), 1);
    assert_eq!(events[0].process, ProcessId::from("process"));
    assert_eq!(events[0].node, NodeId::from("windows-node"));
    assert_eq!(events[0].task, TaskInstanceId::from("windows-command-dev"));
    assert_eq!(
        events[0].artifact_path,
        Some(VfsPath::new("/vfs/artifacts/windows-output.txt").unwrap())
    );
}

#[test]
fn service_schedules_task_across_reported_node_descriptors() {
    let mut service = CoordinatorService::new(7);
    for node in ["cold-node", "warm-node"] {
        service
            .handle_request(CoordinatorRequest::AttachNode {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: node.to_owned(),
                public_key: test_node_public_key(node),
            })
            .unwrap();
    }

    let recorded = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "cold-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: false,
            online: true,
        })
        .unwrap();
    assert_eq!(
        recorded,
        CoordinatorResponse::NodeCapabilitiesRecorded {
            node: NodeId::from("cold-node"),
            node_descriptors: 1,
        }
    );

    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "warm-node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env")],
            dependency_cache_digests: vec![Digest::sha256("deps")],
            source_snapshots: vec![Digest::sha256("source")],
            artifact_locations: vec!["build-output".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service
        .handle_request(CoordinatorRequest::ListNodeDescriptors {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "operator".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected node descriptor inspector state");
    };
    assert_eq!(actor, UserId::from("operator"));
    assert_eq!(descriptors.len(), 2);
    let warm = descriptors
        .iter()
        .find(|descriptor| descriptor.id == NodeId::from("warm-node"))
        .expect("warm node descriptor is visible to inspector state");
    assert!(warm
        .capabilities
        .capabilities
        .contains(&Capability::Command));
    assert!(warm.cached_environments.contains(&Digest::sha256("env")));
    assert!(warm.dependency_caches.contains(&Digest::sha256("deps")));
    assert!(warm.source_snapshots.contains(&Digest::sha256("source")));
    assert!(warm
        .artifact_locations
        .contains(&ArtifactId::from("build-output")));

    let CoordinatorResponse::TaskPlacement { placement } = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: Some(EnvironmentRequirements::linux_container()),
            environment_digest: Some(Digest::sha256("env")),
            required_capabilities: vec![Capability::Command],
            dependency_cache: Some(Digest::sha256("deps")),
            source_snapshot: Some(Digest::sha256("source")),
            required_artifacts: vec!["build-output".to_owned()],
            prefer_node: None,
        })
        .unwrap()
    else {
        panic!("expected task placement");
    };
    assert_eq!(placement.node, NodeId::from("warm-node"));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("environment")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("source")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("dependency")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("artifact")));

    let error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::WindowsCommandDev],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(error.to_string().contains("WindowsCommandDev"));

    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 0)]),
    });
    let quota_error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::Command],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(quota_error
        .to_string()
        .contains("quota unavailable for placement"));

    service.quota.set_workflow_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::Spawn, 10)]),
    });
    service.admission.workflow_placement_allowed = false;
    let policy_error = service
        .handle_request(CoordinatorRequest::ScheduleTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            environment: None,
            environment_digest: None,
            required_capabilities: vec![Capability::Command],
            dependency_cache: None,
            source_snapshot: None,
            required_artifacts: Vec::new(),
            prefer_node: None,
        })
        .unwrap_err();
    assert!(policy_error.to_string().contains("policy denied placement"));
}

#[test]
fn coordinator_side_task_launch_queues_worker_assignment() {
    let mut service = CoordinatorService::new(9);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            public_key: test_node_public_key("worker-linux"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: vec![Digest::sha256("env")],
            dependency_cache_digests: vec![Digest::sha256("deps")],
            source_snapshots: vec![Digest::sha256("source")],
            artifact_locations: vec!["bootstrap-artifact".to_owned()],
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::ProcessStarted { epoch, .. } = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-control".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected coordinator-side process start");
    };
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "worker-linux".to_owned(),
            process: "vp-control".to_owned(),
            epoch,
        })
        .unwrap();
    service.artifact_registry.flush_metadata(ArtifactFlush {
        id: ArtifactId::from("bootstrap-artifact"),
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("vp-control"),
        producer_task: TaskInstanceId::from("bootstrap"),
        retaining_node: NodeId::from("worker-linux"),
        digest: Digest::sha256("bootstrap"),
        size: 9,
    });

    let submitted_task_spec = TaskSpec {
        tenant: TenantId::from("tenant"),
        project: ProjectId::from("project"),
        process: ProcessId::from("vp-control"),
        task_definition: clusterflux_core::TaskDefinitionId::from("compile-linux"),
        task_instance: clusterflux_core::TaskInstanceId::from("compile-linux-1"),
        dispatch: TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile-linux".to_owned()),
            abi: WasmExportAbi::TaskV1,
        },
        environment_id: Some("linux".to_owned()),
        environment: None,
        environment_digest: Some(Digest::sha256("env")),
        required_capabilities: BTreeSet::from([Capability::Command]),
        dependency_cache: Some(Digest::sha256("deps")),
        source_snapshot: Some(Digest::sha256("source")),
        required_artifacts: vec![ArtifactId::from("bootstrap-artifact")],
        args: vec![
            TaskBoundaryValue::SmallJson(serde_json::json!("test")),
            TaskBoundaryValue::SourceSnapshot(Digest::sha256("source")),
            TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("bootstrap-artifact"),
                digest: Digest::sha256("bootstrap"),
                size_bytes: 9,
            }),
        ],
        vfs_epoch: epoch,
        failure_policy: Default::default(),
        bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
    };

    let CoordinatorResponse::TaskLaunched {
        process,
        task,
        placement,
        assignment,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: submitted_task_spec.clone(),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected launched task");
    };
    assert_eq!(process, ProcessId::from("vp-control"));
    assert_eq!(task, TaskInstanceId::from("compile-linux-1"));
    assert_eq!(placement.node, NodeId::from("worker-linux"));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("environment")));
    assert!(placement
        .reasons
        .iter()
        .any(|reason| reason.contains("source")));
    assert_eq!(assignment.node, NodeId::from("worker-linux"));
    assert_eq!(assignment.epoch, epoch);
    assert_eq!(
        assignment.task_spec.bundle_digest,
        Some(Digest::sha256(TEST_WASM_MODULE))
    );
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(assignment.task_spec, submitted_task_spec);
    assert_eq!(
        assignment.task_spec.environment_id.as_deref(),
        Some("linux")
    );
    assert!(matches!(
        assignment.task_spec.dispatch,
        TaskDispatch::CoordinatorNodeWasm {
            export: Some(ref export),
            abi: WasmExportAbi::TaskV1,
        } if export == "compile-linux"
    ));
    assert_eq!(assignment.task_spec.vfs_epoch, epoch);
    assert_eq!(assignment.task_spec.args, submitted_task_spec.args);

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected worker assignment poll");
    };
    let assignment = assignment.expect("worker should receive queued assignment");
    assert_eq!(assignment.process, ProcessId::from("vp-control"));
    assert_eq!(assignment.task, TaskInstanceId::from("compile-linux-1"));
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(
        assignment.task_spec.bundle_digest,
        Some(Digest::sha256(TEST_WASM_MODULE))
    );
    assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64());

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "compile-linux-1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected pending join result");
    };
    assert_eq!(join.state, TaskJoinState::Pending);
    assert!(!join.remote_completion_observed);

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker-linux".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected empty worker assignment poll");
    };
    assert!(assignment.is_none());

    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp-control".to_owned(),
            node: "worker-linux".to_owned(),
            task: "compile-linux-1".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 12,
            stderr_bytes: 0,
            stdout_tail: "ok".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/dap-output.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("artifact")),
            artifact_size_bytes: Some(12),
            result: Some(TaskBoundaryValue::Artifact(ArtifactHandle {
                id: ArtifactId::from("dap-output.txt"),
                digest: Digest::sha256("artifact"),
                size_bytes: 12,
            })),
        })
        .unwrap();
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("vp-control".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    let event_placement = events[0]
        .placement
        .as_ref()
        .expect("task event should retain launch placement explanation");
    assert_eq!(event_placement.node, NodeId::from("worker-linux"));
    assert_eq!(event_placement.reasons, placement.reasons);
    assert_eq!(
        events[0].result,
        Some(TaskBoundaryValue::Artifact(ArtifactHandle {
            id: ArtifactId::from("dap-output.txt"),
            digest: Digest::sha256("artifact"),
            size_bytes: 12,
        }))
    );

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "compile-linux-1".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected completed join result");
    };
    assert_eq!(join.state, TaskJoinState::Completed);
    assert!(join.remote_completion_observed);
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::Artifact(ArtifactHandle {
            id: ArtifactId::from("dap-output.txt"),
            digest: Digest::sha256("artifact"),
            size_bytes: 12,
        }))
    );

    register_test_task_assignment(
        &mut service,
        "tenant",
        "project",
        "vp-control",
        "worker-linux",
        "wasm-add",
        "wasm-add",
        epoch,
    );
    service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp-control".to_owned(),
            node: "worker-linux".to_owned(),
            task: "wasm-add".to_owned(),
            terminal_state: Some(TaskTerminalState::Completed),
            status_code: Some(0),
            stdout_bytes: 3,
            stderr_bytes: 0,
            stdout_tail: "42\n".to_owned(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(42))),
        })
        .unwrap();
    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp-control".to_owned(),
            task: "wasm-add".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected serialized join result");
    };
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::SmallJson(serde_json::json!(42)))
    );
}

#[test]
fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order() {
    let mut service = CoordinatorService::new(41);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant".to_owned(),
            actor_user: "user".to_owned(),
            project: "project".to_owned(),
            name: "Duplicate instances".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "worker".to_owned(),
            process: "vp".to_owned(),
            epoch: 41,
        })
        .unwrap();

    let restart_compatibility = Digest::sha256("compile-u32-to-u32-abi-v1");
    let (initial_module, initial_bundle_digest) =
        test_edited_task_bundle(&restart_compatibility, "initial-body");
    for (instance, argument) in [("compile-1", 1), ("compile-2", 2)] {
        let mut spec =
            test_task_spec_instance("tenant", "project", "vp", "compile", instance, 41, []);
        spec.args = vec![TaskBoundaryValue::SmallJson(serde_json::json!(argument))];
        spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile_export".to_owned()),
            abi: WasmExportAbi::TaskV1,
        };
        spec.bundle_digest = Some(initial_bundle_digest.clone());
        let CoordinatorResponse::TaskLaunched {
            task, assignment, ..
        } = service
            .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: Some("user".to_owned()),
                actor_agent: None,
                agent_public_key_fingerprint: None,
                agent_signature: None,
                task_spec: spec,
                wait_for_node: false,
                artifact_path: format!("/vfs/artifacts/{instance}.json"),
                wasm_module_base64: initial_module.clone(),
            })
            .unwrap()
        else {
            panic!("expected launched task instance");
        };
        assert_eq!(task, TaskInstanceId::from(instance));
        assert_eq!(
            assignment.task_spec.task_definition,
            clusterflux_core::TaskDefinitionId::from("compile")
        );
        assert_eq!(
            assignment.task_spec.task_instance,
            clusterflux_core::TaskInstanceId::from(instance)
        );
    }
    for instance in ["compile-1", "compile-2"] {
        let CoordinatorResponse::TaskAssignment {
            assignment: Some(assignment),
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                node: "worker".to_owned(),
            })
            .unwrap()
        else {
            panic!("expected queued task assignment");
        };
        assert_eq!(assignment.task, TaskInstanceId::from(instance));
    }

    service
        .handle_request(CoordinatorRequest::CancelTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            task: "compile-1".to_owned(),
        })
        .unwrap();
    for (instance, expected_cancelled) in [("compile-1", true), ("compile-2", false)] {
        let CoordinatorResponse::TaskControl {
            cancel_requested,
            abort_requested,
            ..
        } = service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                process: "vp".to_owned(),
                node: "worker".to_owned(),
                task: instance.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected task control response");
        };
        assert_eq!(cancel_requested, expected_cancelled);
        assert!(!abort_requested);
    }

    for (instance, result) in [("compile-2", 22), ("compile-1", 11)] {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                process: "vp".to_owned(),
                node: "worker".to_owned(),
                task: instance.to_owned(),
                terminal_state: Some(TaskTerminalState::Completed),
                status_code: Some(0),
                stdout_bytes: 0,
                stderr_bytes: 0,
                stdout_tail: String::new(),
                stderr_tail: String::new(),
                stdout_truncated: false,
                stderr_truncated: false,
                artifact_path: None,
                artifact_digest: None,
                artifact_size_bytes: None,
                result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(result))),
            })
            .unwrap();
    }

    for (instance, expected) in [("compile-1", 11), ("compile-2", 22)] {
        let CoordinatorResponse::TaskJoined { join } = service
            .handle_request(CoordinatorRequest::JoinTask {
                tenant: "tenant".to_owned(),
                project: "project".to_owned(),
                actor_user: "user".to_owned(),
                process: "vp".to_owned(),
                task: instance.to_owned(),
            })
            .unwrap()
        else {
            panic!("expected task join");
        };
        assert_eq!(
            join.task_instance,
            clusterflux_core::TaskInstanceId::from(instance)
        );
        assert_eq!(
            join.result,
            Some(TaskBoundaryValue::SmallJson(serde_json::json!(expected)))
        );
    }

    let duplicate = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            task_spec: test_task_spec_instance(
                "tenant",
                "project",
                "vp",
                "compile",
                "compile-1",
                41,
                [],
            ),
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/duplicate.json".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();
    assert!(duplicate.to_string().contains("fresh task-instance id"));

    let (replacement_module, replacement_bundle_digest) =
        test_edited_task_bundle(&restart_compatibility, "edited-body");
    assert_ne!(replacement_bundle_digest, initial_bundle_digest);
    let CoordinatorResponse::TaskRestart {
        accepted,
        restarted_task_instance,
        restarted_attempt_id,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-1".to_owned(),
            replacement_bundle: Some(TaskReplacementBundle {
                bundle_digest: replacement_bundle_digest.clone(),
                wasm_module_base64: replacement_module.clone(),
            }),
        })
        .unwrap()
    else {
        panic!("expected task restart response");
    };
    assert!(accepted);
    let restarted = restarted_task_instance.expect("restart returns the logical instance");
    let restarted_attempt_id = restarted_attempt_id.expect("restart creates a new attempt");
    assert!(restarted_attempt_id.starts_with("ta_"));
    assert_eq!(restarted, TaskInstanceId::from("compile-1"));
    assert_ne!(restarted, TaskInstanceId::from("compile-2"));
    let CoordinatorResponse::TaskAssignment {
        assignment: Some(restarted_assignment),
    } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected restarted assignment");
    };
    assert_eq!(restarted_assignment.task, restarted);
    assert_eq!(
        restarted_assignment.task_spec.task_definition,
        clusterflux_core::TaskDefinitionId::from("compile")
    );
    assert_eq!(
        restarted_assignment.task_spec.args,
        vec![TaskBoundaryValue::SmallJson(serde_json::json!(1))]
    );
    assert_eq!(
        restarted_assignment.task_spec.bundle_digest,
        Some(replacement_bundle_digest)
    );
    assert_eq!(
        restarted_assignment.task_spec.dispatch,
        TaskDispatch::CoordinatorNodeWasm {
            export: Some("compile_export".to_owned()),
            abi: WasmExportAbi::TaskV1,
        }
    );
    assert_eq!(restarted_assignment.wasm_module_base64, replacement_module);

    let incompatible_compatibility = Digest::sha256("compile-changed-signature");
    let (incompatible_module, incompatible_bundle_digest) =
        test_edited_task_bundle(&incompatible_compatibility, "incompatible-body");
    let CoordinatorResponse::TaskRestart {
        accepted,
        requires_whole_process_restart,
        restarted_task_instance,
        message,
        ..
    } = service
        .handle_request(CoordinatorRequest::RestartTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-2".to_owned(),
            replacement_bundle: Some(TaskReplacementBundle {
                bundle_digest: incompatible_bundle_digest,
                wasm_module_base64: incompatible_module,
            }),
        })
        .unwrap()
    else {
        panic!("expected incompatible task restart response");
    };
    assert!(!accepted);
    assert!(requires_whole_process_restart);
    assert!(restarted_task_instance.is_none());
    assert!(message.contains("task ABI changed"));

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_request(CoordinatorRequest::JoinTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            process: "vp".to_owned(),
            task: "compile-2".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected unaffected sibling join");
    };
    assert_eq!(join.state, TaskJoinState::Completed);
    assert_eq!(
        join.result,
        Some(TaskBoundaryValue::SmallJson(serde_json::json!(22)))
    );
}

#[test]
fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() {
    let mut service = CoordinatorService::new(73);
    service
        .handle_request(CoordinatorRequest::CreateProject {
            tenant: "tenant-soak".to_owned(),
            actor_user: "user-soak".to_owned(),
            project: "project-soak".to_owned(),
            name: "Bounded soak".to_owned(),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            public_key: test_node_public_key("worker-soak"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            node: "worker-soak".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant-soak".to_owned(),
            project: "project-soak".to_owned(),
            actor_user: Some("user-soak".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-soak".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "worker-soak".to_owned(),
            process: "vp-soak".to_owned(),
            epoch: 73,
        })
        .unwrap();

    let event_waves = 4;
    for index in 0..super::MAX_TASK_EVENTS_PER_PROCESS * event_waves {
        let task = format!("soak-task-{index}");
        register_test_task_assignment(
            &mut service,
            "tenant-soak",
            "project-soak",
            "vp-soak",
            "worker-soak",
            "soak-task",
            &task,
            73,
        );
        service
            .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
                tenant: "tenant-soak".to_owned(),
                project: "project-soak".to_owned(),
                process: "vp-soak".to_owned(),
                node: "worker-soak".to_owned(),
                task,
                terminal_state: Some(TaskTerminalState::Completed),
                status_code: Some(0),
                stdout_bytes: 8,
                stderr_bytes: 0,
                stdout_tail: "soak-log".to_owned(),
                stderr_tail: String::new(),
                stdout_truncated: false,
                stderr_truncated: false,
                artifact_path: None,
                artifact_digest: None,
                artifact_size_bytes: None,
                result: Some(TaskBoundaryValue::SmallJson(json!(index))),
            })
            .unwrap();
    }
    for index in 0..super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS * 3 {
        service
            .record_debug_audit_event(
                TenantId::from("tenant-soak"),
                ProjectId::from("project-soak"),
                ProcessId::from("vp-soak"),
                None,
                UserId::from("user-soak"),
                "soak_inspect",
                true,
                format!("bounded audit {index}"),
            )
            .unwrap();
    }
    for _ in 0..128 {
        service
            .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
                tenant: "tenant-soak".to_owned(),
                project: "project-soak".to_owned(),
                node: "worker-soak".to_owned(),
            })
            .unwrap();
    }

    for index in 0..2_000 {
        service
            .node_replay_nonces
            .insert((NodeId::from("worker-soak"), format!("expired-{index}")), 0);
    }
    service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "worker-soak".to_owned(),
            node_signature: Some(signed_node_heartbeat("worker-soak", "post-expiry-prune")),
        })
        .unwrap();

    for index in 0..3 {
        service.record_task_completion_event(TaskCompletionEvent {
            tenant: TenantId::from("tenant-other"),
            project: ProjectId::from("project-other"),
            process: ProcessId::from("vp-other"),
            node: NodeId::from("worker-other"),
            executor: TaskExecutor::Node,
            task_definition: clusterflux_core::TaskDefinitionId::from("other"),
            task: TaskInstanceId::new(format!("other-{index}")),
            attempt_id: None,
            placement: None,
            terminal_state: TaskTerminalState::Completed,
            status_code: Some(0),
            stdout_bytes: 0,
            stderr_bytes: 0,
            stdout_tail: String::new(),
            stderr_tail: String::new(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: None,
            artifact_digest: None,
            artifact_size_bytes: None,
            result: None,
        });
    }

    let retained_soak_events = service
        .task_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-soak"))
        .count();
    let retained_other_events = service
        .task_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-other"))
        .count();
    let retained_soak_audit = service
        .debug_audit_events
        .iter()
        .filter(|event| event.process == ProcessId::from("vp-soak"))
        .count();
    let retained_soak_checkpoints = service
        .task_restart_checkpoints
        .keys()
        .filter(|(_, _, process, _)| process == &ProcessId::from("vp-soak"))
        .count();
    let retained_soak_nonces = service
        .node_replay_nonces
        .keys()
        .filter(|(node, _)| node == &NodeId::from("worker-soak"))
        .count();

    assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS);
    assert_eq!(retained_other_events, 3);
    assert_eq!(
        retained_soak_audit,
        super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS
    );
    assert_eq!(
        retained_soak_checkpoints,
        super::MAX_RESTART_CHECKPOINTS_PER_PROCESS
    );
    assert!(retained_soak_nonces <= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY);
    assert!(
        super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
            >= super::NODE_SIGNATURE_WINDOW_SECONDS as usize * 2 * 1_000
                / clusterflux_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS as usize
                + 64,
        "the bounded node replay window must sustain artifact and assignment polls at the protocol minimum with control-message headroom"
    );
    assert!(service
        .coordinator
        .active_process(
            &TenantId::from("tenant-soak"),
            &ProjectId::from("project-soak"),
            &ProcessId::from("vp-soak"),
        )
        .is_some());
}

#[test]
fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() {
    let mut service = CoordinatorService::new(12);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            public_key: test_node_public_key("worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp".to_owned(),
            restart: false,
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "worker".to_owned(),
            process: "vp".to_owned(),
            epoch: 12,
        })
        .unwrap();
    service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            task_spec: test_task_spec("tenant", "project", "vp", "parent", 12, []),
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/parent.json".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap();

    let child_request = CoordinatorRequest::LaunchChildTask {
        tenant: "tenant".to_owned(),
        project: "project".to_owned(),
        process: "vp".to_owned(),
        node: "worker".to_owned(),
        parent_task: "parent".to_owned(),
        task_spec: test_task_spec("tenant", "project", "vp", "child", 12, []),
        wait_for_node: false,
        artifact_path: "/vfs/artifacts/child.json".to_owned(),
        wasm_module_base64: test_wasm_module_base64(),
    };
    let unsigned = service.handle_request(child_request.clone()).unwrap_err();
    assert!(unsigned.to_string().contains("signed_node"));

    let CoordinatorResponse::TaskLaunched { actor, task, .. } = service
        .handle_signed_node_request_auto(child_request)
        .unwrap()
    else {
        panic!("expected signed child launch");
    };
    assert_eq!(task, TaskInstanceId::from("child"));
    assert_eq!(actor.kind, "task");
    assert_eq!(actor.credential_kind, CredentialKind::TaskCredential);
    assert!(actor.scopes.contains(&"process:spawn-child".to_owned()));

    let wrong_parent = service
        .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            parent_task: "not-active".to_owned(),
            task: "child".to_owned(),
        })
        .unwrap_err();
    assert!(wrong_parent.to_string().contains("active parent"));

    let CoordinatorResponse::TaskJoined { join } = service
        .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            process: "vp".to_owned(),
            node: "worker".to_owned(),
            parent_task: "parent".to_owned(),
            task: "child".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected signed child join");
    };
    assert_eq!(join.state, TaskJoinState::Pending);
}

#[test]
fn coordinator_rejects_named_environment_without_requirements() {
    let mut service = CoordinatorService::new(10);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "uncached-worker".to_owned(),
            public_key: test_node_public_key("uncached-worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "uncached-worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();
    let CoordinatorResponse::ProcessStarted { epoch, .. } = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-environment".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start");
    };
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
            node: "uncached-worker".to_owned(),
            process: "vp-environment".to_owned(),
            epoch,
        })
        .unwrap();
    let mut task_spec = test_task_spec(
        "tenant",
        "project",
        "vp-environment",
        "compile-linux",
        epoch,
        [],
    );
    task_spec.environment_id = Some("missing-environment".to_owned());
    task_spec.environment_digest = Some(Digest::sha256("missing-environment"));

    let error = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec,
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/environment-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();

    assert!(error.to_string().contains("named environment cache"));
}

#[test]
fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
    let mut service = CoordinatorService::new(10);
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-control".to_owned(),
            restart: false,
        })
        .unwrap();

    let error = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-control",
                "compile-linux",
                10,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: false,
            artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap_err();

    assert!(error.to_string().contains("no capable node"));
}

#[test]
fn coordinator_side_task_launch_can_wait_for_capable_worker() {
    let mut service = CoordinatorService::new(11);
    let CoordinatorResponse::ProcessStarted { epoch, .. } = service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-wait".to_owned(),
            restart: false,
        })
        .unwrap()
    else {
        panic!("expected process start");
    };

    let CoordinatorResponse::TaskQueued {
        process,
        task,
        reason,
        queued_tasks,
        charged_spawns,
        ..
    } = service
        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
            task_spec: test_task_spec(
                "tenant",
                "project",
                "vp-wait",
                "compile-linux",
                epoch,
                [Capability::Command],
            ),
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: Some("user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            wait_for_node: true,
            artifact_path: "/vfs/artifacts/wait-output.txt".to_owned(),
            wasm_module_base64: test_wasm_module_base64(),
        })
        .unwrap()
    else {
        panic!("expected queued task launch");
    };
    assert_eq!(process, ProcessId::from("vp-wait"));
    assert_eq!(task, TaskInstanceId::from("compile-linux"));
    assert!(reason.contains("waiting for any capable node"));
    assert_eq!(queued_tasks, 1);
    assert_eq!(charged_spawns, 2);

    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
            public_key: test_node_public_key("late-worker"),
        })
        .unwrap();
    service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap();

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected pending assignment poll");
    };
    let assignment = assignment.expect("late worker should receive pending assignment");
    assert_eq!(assignment.process, ProcessId::from("vp-wait"));
    assert_eq!(assignment.task, TaskInstanceId::from("compile-linux"));
    assert_eq!(assignment.node, NodeId::from("late-worker"));
    assert_eq!(assignment.epoch, epoch);
    assert!(assignment.task_spec.product_mode_uses_remote_dispatch());
    assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64());

    let CoordinatorResponse::TaskAssignment { assignment } = service
        .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "late-worker".to_owned(),
        })
        .unwrap()
    else {
        panic!("expected empty assignment poll");
    };
    assert!(assignment.is_none());
}

#[test]
fn service_rejects_malformed_node_capability_report() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let mut capabilities = linux_capabilities();
    capabilities
        .source_providers
        .insert("../checkout".to_owned());
    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities,
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("source provider id"));
    assert!(service.node_descriptors.is_empty());
}

#[test]
fn service_meters_rendezvous_before_direct_transfer_plan() {
    let mut service = CoordinatorService::new(7);
    service.quota.set_rendezvous_limits(ResourceLimits {
        limits: BTreeMap::from([(LimitKind::RendezvousAttempt, 2)]),
    });

    let CoordinatorResponse::RendezvousPlan {
        plan,
        charged_rendezvous_attempts,
    } = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: true,
            failure_reason: String::new(),
        })
        .unwrap()
    else {
        panic!("expected rendezvous plan");
    };

    assert_eq!(charged_rendezvous_attempts, 1);
    assert_eq!(plan.scope.project, ProjectId::from("project"));
    assert_eq!(plan.source.node, NodeId::from("node-a"));
    assert_eq!(plan.destination.node, NodeId::from("node-b"));
    assert!(plan.coordinator_assisted_rendezvous);
    assert!(!plan.coordinator_bulk_relay_allowed);

    let unavailable = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: false,
            failure_reason: "nat traversal failed".to_owned(),
        })
        .unwrap_err();
    let unavailable = unavailable.to_string();
    assert!(unavailable.contains("nat traversal failed"));
    assert!(unavailable.contains("coordinator bulk relay is disabled"));

    let quota = service
        .handle_request(CoordinatorRequest::RequestRendezvous {
            scope: data_plane_scope("project"),
            source: endpoint("node-a"),
            destination: endpoint("node-b"),
            direct_connectivity: true,
            failure_reason: String::new(),
        })
        .unwrap_err();
    assert!(quota.to_string().contains("RendezvousAttempt"));
}

#[test]
fn service_rejects_task_completion_outside_node_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();
    service
        .handle_request(CoordinatorRequest::StartProcess {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: None,
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "process".to_owned(),
            restart: false,
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            process: "process".to_owned(),
            node: "node".to_owned(),
            task: "compile-linux".to_owned(),
            terminal_state: None,
            status_code: Some(0),
            stdout_bytes: 128,
            stderr_bytes: 64,
            stdout_tail: "foreign stdout".to_owned(),
            stderr_tail: "foreign stderr".to_owned(),
            stdout_truncated: false,
            stderr_truncated: false,
            artifact_path: Some("/vfs/artifacts/foreign.txt".to_owned()),
            artifact_digest: Some(Digest::sha256("foreign-artifact")),
            artifact_size_bytes: Some(128),
            result: None,
        })
        .unwrap_err();

    assert!(error.to_string().contains("outside"));
    let CoordinatorResponse::TaskEvents { events } = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap()
    else {
        panic!("expected task events");
    };
    assert!(events.is_empty());
}

#[test]
fn service_rejects_task_event_access_using_retained_process_scope() {
    let mut service = CoordinatorService::new(1);
    service.process_scope_history.push_back((
        TenantId::from("tenant-a"),
        ProjectId::from("project-a"),
        ProcessId::from("process"),
    ));

    let error = service
        .handle_request(CoordinatorRequest::ListTaskEvents {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            actor_user: "user".to_owned(),
            process: Some("process".to_owned()),
        })
        .unwrap_err();

    assert!(error
        .to_string()
        .contains("outside the virtual process tenant/project scope"));
}

#[test]
fn service_rejects_node_capability_report_outside_enrollment_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("tenant/project scope"));
    assert!(service.node_descriptors.is_empty());
}

#[test]
fn service_rejects_source_preparation_completion_outside_node_scope() {
    let mut service = CoordinatorService::new(1);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant-a".to_owned(),
            project: "project-a".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_signed_node_request_auto(CoordinatorRequest::CompleteSourcePreparation {
            tenant: "tenant-b".to_owned(),
            project: "project-b".to_owned(),
            node: "node".to_owned(),
            provider: SourceProviderKind::Filesystem,
            source_snapshot: Digest::sha256("foreign-source"),
        })
        .unwrap_err();

    assert!(error.to_string().contains("tenant/project scope"));
}

#[test]
fn service_rejects_unknown_node_heartbeat() {
    let mut service = CoordinatorService::new(1);

    let error = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "missing".to_owned(),
            node_signature: None,
        })
        .unwrap_err();

    assert!(error.to_string().contains("not enrolled"));
}

#[test]
fn service_requires_signed_node_heartbeat_from_enrolled_key() {
    let mut service = CoordinatorService::new(5);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let unsigned = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: None,
        })
        .unwrap_err();
    assert!(unsigned.to_string().contains("signed proof"));

    let wrong_private_key = test_node_private_key("other-node");
    let wrong_signature = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat_with_private_key(
                "node",
                &wrong_private_key,
                "wrong-node-key",
            )),
        })
        .unwrap_err();
    assert!(wrong_signature.to_string().contains("signature"));

    let signed = signed_node_heartbeat("node", "fresh-node-heartbeat");
    let heartbeat = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed.clone()),
        })
        .unwrap();
    assert_eq!(
        heartbeat,
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 5,
        }
    );

    let replay = service
        .handle_request(CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed),
        })
        .unwrap_err();
    assert!(replay.to_string().contains("nonce"));
}

#[test]
fn service_rejects_raw_node_originated_requests_without_signed_envelope() {
    let mut service = CoordinatorService::new(5);
    service
        .handle_request(CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        })
        .unwrap();

    let error = service
        .handle_request(CoordinatorRequest::ReportNodeCapabilities {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            capabilities: linux_capabilities(),
            cached_environment_digests: Vec::new(),
            dependency_cache_digests: Vec::new(),
            source_snapshots: Vec::new(),
            artifact_locations: Vec::new(),
            direct_connectivity: true,
            online: true,
        })
        .unwrap_err();

    assert!(error.to_string().contains("signed_node envelope proof"));
}

#[test]
fn service_stream_accepts_multiple_requests_on_one_connection() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream_local_trusted(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::AttachNode {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            node: "node".to_owned(),
            public_key: test_node_public_key("node"),
        },
        "stream-1",
    );

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    assert!(matches!(
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
        CoordinatorResponse::NodeAttached { .. }
    ));

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::NodeHeartbeat {
            node: "node".to_owned(),
            node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")),
        },
        "stream-2",
    );

    line.clear();
    reader.read_line(&mut line).unwrap();
    assert_eq!(
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
        CoordinatorResponse::NodeHeartbeat {
            node: NodeId::from("node"),
            epoch: 3,
        }
    );

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        service
            .issue_cli_session(
                TenantId::from("tenant"),
                ProjectId::from("project"),
                UserId::from("user"),
                "strict-stream-session",
                None,
            )
            .unwrap();
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());
    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::StartProcess {
            tenant: "victim-tenant".to_owned(),
            project: "victim-project".to_owned(),
            actor_user: Some("forged-user".to_owned()),
            actor_agent: None,
            agent_public_key_fingerprint: None,
            agent_signature: None,
            process: "vp-forged".to_owned(),
            restart: false,
        },
        "strict-stream-forged",
    );

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    let CoordinatorResponse::Error { message } =
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
    else {
        panic!("expected strict body-authority denial");
    };
    assert!(message.contains("request-body identity fields are not authority"));

    write_coordinator_wire_request(
        &mut stream,
        &CoordinatorRequest::Authenticated {
            session_secret: "strict-stream-session".to_owned(),
            request: AuthenticatedCoordinatorRequest::StartProcess {
                process: "vp-authenticated".to_owned(),
                restart: false,
            },
        },
        "strict-stream-authenticated",
    );

    line.clear();
    reader.read_line(&mut line).unwrap();
    let CoordinatorResponse::ProcessStarted { process, actor, .. } =
        serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
    else {
        panic!("expected authenticated strict process start");
    };
    assert_eq!(process, ProcessId::from("vp-authenticated"));
    assert_eq!(actor.user, Some(UserId::from("user")));

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn service_stream_rejects_invalid_versioned_envelope_metadata() {
    let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
    let server = std::thread::spawn(move || {
        let mut service = CoordinatorService::new(3);
        let (stream, _) = listener.accept().unwrap();
        service.handle_stream(stream).unwrap();
    });

    let mut stream = TcpStream::connect(addr).unwrap();
    let mut reader = BufReader::new(stream.try_clone().unwrap());
    let mut wire_request = coordinator_wire_request(
        "bad-operation",
        serde_json::to_value(CoordinatorRequest::Ping).unwrap(),
    );
    wire_request["operation"] = serde_json::Value::String("attach_node".to_owned());
    serde_json::to_writer(&mut stream, &wire_request).unwrap();
    stream.write_all(b"\n").unwrap();
    stream.flush().unwrap();

    let mut line = String::new();
    reader.read_line(&mut line).unwrap();
    let response = serde_json::from_str::<CoordinatorResponse>(&line).unwrap();
    let CoordinatorResponse::Error { message } = response else {
        panic!("expected invalid wire envelope response");
    };
    assert!(message.contains("operation attach_node does not match payload operation ping"));

    stream.shutdown(std::net::Shutdown::Both).unwrap();
    server.join().unwrap();
}

#[test]
fn coordinator_protocol_rejects_client_supplied_authority_fields() {
    for payload in [
        json!({
            "type": "create_node_enrollment_grant",
            "tenant": "tenant",
            "project": "project",
            "actor_user": "user",
            "grant": "attacker-chosen",
            "ttl_seconds": 60,
        }),
        json!({
            "type": "exchange_node_enrollment_grant",
            "tenant": "tenant",
            "project": "project",
            "node": "node",
            "public_key": "key",
            "enrollment_grant": "grant",
            "now_epoch_seconds": 0,
        }),
        json!({
            "type": "schedule_task",
            "tenant": "tenant",
            "project": "project",
            "environment": null,
            "environment_digest": null,
            "required_capabilities": [],
            "dependency_cache": null,
            "source_snapshot": null,
            "required_artifacts": [],
            "quota_available": true,
            "policy_allowed": true,
            "prefer_node": null,
        }),
        json!({
            "type": "create_artifact_download_link",
            "tenant": "tenant",
            "project": "project",
            "actor_user": "user",
            "artifact": "artifact",
            "max_bytes": 1,
            "token_nonce": "attacker-chosen",
            "ttl_seconds": 60,
        }),
    ] {
        let error = serde_json::from_value::<CoordinatorRequest>(payload).unwrap_err();
        assert!(error.to_string().contains("unknown field"));
    }
}

#[test]
fn coordinator_generates_and_bounds_node_enrollment_grants() {
    let mut service = CoordinatorService::new(7);
    service.set_server_time(100);

    let first = service
        .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            ttl_seconds: u64::MAX,
        })
        .unwrap();
    let second = service
        .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant {
            tenant: "tenant".to_owned(),
            project: "project".to_owned(),
            actor_user: "user".to_owned(),
            ttl_seconds: u64::MAX,
        })
        .unwrap();

    let CoordinatorResponse::NodeEnrollmentGrantCreated {
        grant: first_grant,
        expires_at_epoch_seconds,
        ..
    } = first
    else {
        panic!("expected enrollment grant");
    };
    let CoordinatorResponse::NodeEnrollmentGrantCreated {
        grant: second_grant,
        ..
    } = second
    else {
        panic!("expected enrollment grant");
    };
    assert!(first_grant.starts_with("node_grant_"));
    assert_ne!(first_grant, second_grant);
    assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60);
}