use std::collections::{BTreeMap, BTreeSet}; use std::io::{BufRead, BufReader, Write}; use std::net::TcpStream; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_core::{ admin_request_proof, agent_ed25519_public_key_from_private_key, coordinator_wire_request, derive_ed25519_private_key_from_seed, node_ed25519_public_key_from_private_key, sign_agent_workflow_request, sign_node_request, signed_request_payload_digest, AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability, DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements, LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits, SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult, }; use serde_json::json; use crate::FallibleDurableStore; use super::keys::{process_control_key, task_control_key}; use super::*; fn test_admin_request( token: &str, operation: &str, tenant: &str, actor_user: &str, target_tenant: &str, nonce: &str, ) -> (Digest, String, u64) { let issued_at_epoch_seconds = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs()) .unwrap_or_default(); ( admin_request_proof( token, operation, tenant, actor_user, target_tenant, nonce, issued_at_epoch_seconds, ), nonce.to_owned(), issued_at_epoch_seconds, ) } fn enroll_test_node( service: &mut CoordinatorService, tenant: &str, project: &str, node: &str, public_key: &str, ) { let response = service .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { tenant: tenant.to_owned(), project: project.to_owned(), actor_user: "test-user".to_owned(), ttl_seconds: 900, }) .unwrap(); let CoordinatorResponse::NodeEnrollmentGrantCreated { grant, .. } = response else { panic!("expected node enrollment grant"); }; service .handle_request(CoordinatorRequest::ExchangeNodeEnrollmentGrant { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), public_key: public_key.to_owned(), enrollment_grant: grant, }) .unwrap(); } #[test] fn runtime_service_uses_memory_only_when_database_url_is_absent() { let service = CoordinatorService::new_with_database_url(1, None).unwrap(); assert_eq!(service.durable_store_kind(), "in_memory"); let error = CoordinatorService::new_with_database_url(1, Some("not-a-postgres-url")) .err() .expect("an invalid configured DATABASE_URL must fail closed"); assert!(error .to_string() .contains("durable coordinator state failed")); } #[test] fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() { let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else { return; }; let mut clean_store = crate::PostgresDurableStore::connect(&database_url).unwrap(); clean_store .save_state(&crate::DurableState::default()) .unwrap(); let session_secret = "postgres-runtime-service-session"; let mut first = CoordinatorService::new_with_database_url(41, Some(&database_url)).unwrap(); assert_eq!(first.durable_store_kind(), "postgres"); first .issue_cli_session( TenantId::from("tenant-pg"), ProjectId::from("project-pg"), UserId::from("user-pg"), session_secret, None, ) .unwrap(); enroll_test_node( &mut first, "tenant-pg", "project-pg", "node-pg", &test_node_public_key("node-pg"), ); let duplicate_private_key = test_node_private_key("node-pg-other-scope"); enroll_test_node( &mut first, "tenant-pg-other", "project-pg-other", "node-pg", &node_ed25519_public_key_from_private_key(&duplicate_private_key).unwrap(), ); first .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "process-ephemeral".to_owned(), restart: false, }, }) .unwrap(); drop(first); let mut restarted = CoordinatorService::new_with_database_url(42, Some(&database_url)).unwrap(); assert_eq!(restarted.durable_store_kind(), "postgres"); let auth = restarted .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap(); assert!(matches!( auth, CoordinatorResponse::AuthStatus { authenticated: true, .. } )); let heartbeat = restarted .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-pg".to_owned(), project: "project-pg".to_owned(), node: "node-pg".to_owned(), node_signature: Some(signed_node_heartbeat_in_scope( "tenant-pg", "project-pg", "node-pg", "postgres-restart-heartbeat", )), }) .unwrap(); assert!(matches!( heartbeat, CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } )); let duplicate_heartbeat = restarted .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-pg-other".to_owned(), project: "project-pg-other".to_owned(), node: "node-pg".to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( "tenant-pg-other", "project-pg-other", "node-pg", &duplicate_private_key, "postgres-restart-heartbeat", )), }) .unwrap(); assert!(matches!( duplicate_heartbeat, CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } )); let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), request: AuthenticatedCoordinatorRequest::ListProcesses, }) .unwrap() else { panic!("expected process list"); }; assert!(processes.is_empty()); restarted .handle_request(CoordinatorRequest::RevokeNodeCredential { tenant: "tenant-pg".to_owned(), project: "project-pg".to_owned(), actor_user: "user-pg".to_owned(), node: "node-pg".to_owned(), }) .unwrap(); drop(restarted); let mut after_revocation = CoordinatorService::new_with_database_url(43, Some(&database_url)).unwrap(); assert!(after_revocation .coordinator .node_identity( &TenantId::from("tenant-pg"), &ProjectId::from("project-pg"), &NodeId::from("node-pg"), ) .is_none()); assert!(after_revocation .coordinator .node_identity( &TenantId::from("tenant-pg-other"), &ProjectId::from("project-pg-other"), &NodeId::from("node-pg"), ) .is_some()); after_revocation .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-pg-other".to_owned(), project: "project-pg-other".to_owned(), node: "node-pg".to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( "tenant-pg-other", "project-pg-other", "node-pg", &duplicate_private_key, "post-revocation-restart", )), }) .expect("the duplicate node in the other scope must remain authenticated"); } fn linux_capabilities() -> NodeCapabilities { NodeCapabilities { os: Os::Linux, arch: "x86_64".to_owned(), capabilities: BTreeSet::from([ Capability::Command, Capability::Containers, Capability::RootlessPodman, Capability::VfsArtifacts, ]), environment_backends: BTreeSet::from([EnvironmentBackend::Container]), source_providers: BTreeSet::from(["filesystem".to_owned()]), } } fn windows_capabilities() -> NodeCapabilities { NodeCapabilities { os: Os::Windows, arch: "x86_64".to_owned(), capabilities: BTreeSet::from([ Capability::Command, Capability::WindowsCommandDev, Capability::VfsArtifacts, ]), environment_backends: BTreeSet::from([EnvironmentBackend::WindowsCommandDev]), source_providers: BTreeSet::from(["filesystem".to_owned()]), } } fn endpoint(name: &str) -> NodeEndpoint { NodeEndpoint { node: NodeId::from(name), advertised_addr: format!("{name}.mesh.invalid:4433"), public_key_fingerprint: Digest::sha256(format!("{name}-public-key")), } } fn data_plane_scope(project: &str) -> DataPlaneScope { DataPlaneScope { tenant: TenantId::from("tenant"), project: ProjectId::from(project), process: ProcessId::from("process"), object: DataPlaneObject::Artifact(ArtifactId::from("artifact")), authorization_subject: "node-a-to-node-b".to_owned(), } } fn assert_agent_workflow_actor(actor: &WorkflowActor, fingerprint: &Digest) { assert_eq!(actor.kind, "agent"); assert_eq!(actor.user, Some(UserId::from("user"))); assert_eq!(actor.agent, Some(AgentId::from("agent-ci"))); assert_eq!(actor.credential_kind, CredentialKind::PublicKey); assert_eq!(actor.public_key_fingerprint.as_ref(), Some(fingerprint)); assert!(actor.authenticated_without_browser); assert!(actor.scopes.iter().any(|scope| scope == "project:run")); } const TEST_AGENT_PRIVATE_KEY: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="; static TEST_NODE_REQUEST_NONCE: AtomicU64 = AtomicU64::new(1); const TEST_WASM_MODULE: &[u8] = b"clusterflux-test-wasm-module"; fn test_wasm_module_base64() -> String { BASE64_STANDARD.encode(TEST_WASM_MODULE) } fn append_test_custom_section(module: &mut Vec, name: &str, data: &[u8]) { fn leb(mut value: usize, output: &mut Vec) { loop { let mut byte = (value & 0x7f) as u8; value >>= 7; if value != 0 { byte |= 0x80; } output.push(byte); if value == 0 { break; } } } let mut payload = Vec::new(); leb(name.len(), &mut payload); payload.extend_from_slice(name.as_bytes()); payload.extend_from_slice(data); module.push(0); leb(payload.len(), module); module.extend_from_slice(&payload); } fn test_edited_task_bundle(compatibility: &Digest, edit_marker: &str) -> (String, Digest) { let descriptor = serde_json::to_vec(&serde_json::json!({ "kind": "task", "name": "compile", "function": "compile", "export": "compile_export", "stable_id": Digest::sha256("compile-stable"), "argument_schema": "u32", "result_schema": "u32", "required_capabilities": [], "restart_compatibility_hash": compatibility, "abi_version": clusterflux_core::WASM_TASK_ABI_VERSION, "probe_symbol": "clusterflux.probe.compile", })) .unwrap(); let mut module = b"\0asm\x01\0\0\0".to_vec(); append_test_custom_section(&mut module, "clusterflux.tasks", &descriptor); append_test_custom_section(&mut module, "clusterflux.edit", edit_marker.as_bytes()); let digest = Digest::sha256(&module); (BASE64_STANDARD.encode(module), digest) } fn test_task_spec( tenant: &str, project: &str, process: &str, task: &str, epoch: u64, required_capabilities: impl IntoIterator, ) -> TaskSpec { test_task_spec_instance( tenant, project, process, task, task, epoch, required_capabilities, ) } fn test_task_spec_instance( tenant: &str, project: &str, process: &str, task_definition: &str, task_instance: &str, epoch: u64, required_capabilities: impl IntoIterator, ) -> TaskSpec { TaskSpec { tenant: TenantId::from(tenant), project: ProjectId::from(project), process: ProcessId::from(process), task_definition: clusterflux_core::TaskDefinitionId::from(task_definition), task_instance: clusterflux_core::TaskInstanceId::from(task_instance), dispatch: TaskDispatch::CoordinatorNodeWasm { export: Some(task_definition.to_owned()), abi: WasmExportAbi::TaskV1, }, environment_id: None, environment: None, environment_digest: None, required_capabilities: required_capabilities.into_iter().collect(), dependency_cache: None, source_snapshot: None, required_artifacts: Vec::new(), args: Vec::new(), vfs_epoch: epoch, failure_policy: Default::default(), bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)), } } trait AuthorizedTestTaskLaunch { fn handle_authorized_test_task_launch( &mut self, request: CoordinatorRequest, ) -> Result; } impl AuthorizedTestTaskLaunch for CoordinatorService { fn handle_authorized_test_task_launch( &mut self, request: CoordinatorRequest, ) -> Result { let CoordinatorRequest::LaunchTask { task_spec, wait_for_node, artifact_path, wasm_module_base64, .. } = request else { panic!("authorized test task launch requires LaunchTask"); }; let tenant = task_spec.tenant.clone(); let project = task_spec.project.clone(); self.handle_launch_task_with_actor( tenant, project, WorkflowActor { kind: "task".to_owned(), user: None, agent: None, credential_kind: CredentialKind::TaskCredential, public_key_fingerprint: None, authenticated_without_browser: true, scopes: vec!["process:spawn-child".to_owned()], }, task_spec, wait_for_node, artifact_path, wasm_module_base64, ) } } fn register_test_task_assignment( service: &mut CoordinatorService, tenant: &str, project: &str, process: &str, node: &str, task_definition: &str, task_instance: &str, epoch: u64, ) { let task_spec = test_task_spec_instance( tenant, project, process, task_definition, task_instance, epoch, [], ); let assignment = TaskAssignment { tenant: TenantId::from(tenant), project: ProjectId::from(project), process: ProcessId::from(process), task: TaskInstanceId::from(task_instance), node: NodeId::from(node), epoch, artifact_path: format!("/vfs/artifacts/{task_instance}.bin"), task_spec, wasm_module_base64: test_wasm_module_base64(), }; service .capture_task_restart_checkpoint(&assignment) .unwrap(); service.active_tasks.insert(super::keys::task_control_key( &assignment.tenant, &assignment.project, &assignment.process, &assignment.node, &assignment.task, )); } fn service_with_completed_main_and_final_child( failure_policy: clusterflux_core::TaskFailurePolicy, ) -> CoordinatorService { let mut service = CoordinatorService::new(83); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); let node = NodeId::from("worker"); let task = TaskInstanceId::from("final-child"); service .handle_request(CoordinatorRequest::AttachNode { tenant: tenant.to_string(), project: project.to_string(), node: node.to_string(), public_key: test_node_public_key(node.as_str()), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: tenant.to_string(), project: project.to_string(), node: node.to_string(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: process.to_string(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: tenant.to_string(), project: project.to_string(), node: node.to_string(), process: process.to_string(), epoch: 83, }) .unwrap(); let mut task_spec = test_task_spec_instance( tenant.as_str(), project.as_str(), process.as_str(), "child-definition", task.as_str(), 83, [], ); task_spec.failure_policy = failure_policy; let assignment = TaskAssignment { tenant: tenant.clone(), project: project.clone(), process: process.clone(), task: task.clone(), node: node.clone(), epoch: 83, artifact_path: "/vfs/artifacts/final-child.bin".to_owned(), task_spec: task_spec.clone(), wasm_module_base64: test_wasm_module_base64(), }; service .capture_task_restart_checkpoint(&assignment) .unwrap(); service .begin_task_attempt( &task_spec, Some(node.clone()), Some(&assignment.artifact_path), false, ) .unwrap(); service .active_tasks .insert(task_control_key(&tenant, &project, &process, &node, &task)); service .task_assignments .entry((tenant.clone(), project.clone(), node.clone())) .or_default() .push_back(assignment); service .debug_epochs .insert(process_control_key(&tenant, &project, &process), 11); service.debug_breakpoints.insert( process_control_key(&tenant, &project, &process), super::debug::DebugBreakpointPlan { actor: UserId::from("user"), revision: 1, probe_symbols: BTreeSet::from(["child-probe".to_owned()]), hit_epoch: None, hit_task: None, hit_probe_symbol: None, }, ); service.debug_commands.insert( task_control_key(&tenant, &project, &process, &node, &task), super::debug::DebugPendingCommand { epoch: 11, command: "continue".to_owned(), }, ); let panel_key = super::keys::panel_stop_key(&tenant, &project, &process); service.panel_snapshots.insert( panel_key.clone(), PanelState { tenant: tenant.clone(), project: project.clone(), process: process.clone(), widgets: BTreeMap::new(), program_ui_events_enabled: false, control_plane_actions: Vec::new(), }, ); service.stopped_panels.insert(panel_key); service.record_task_completion_event(TaskCompletionEvent { tenant: tenant.clone(), project: project.clone(), process: process.clone(), node: NodeId::from("coordinator-main"), executor: TaskExecutor::CoordinatorMain, task_definition: TaskDefinitionId::from("build"), task: TaskInstanceId::from("main"), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }); service .coordinator .grant_project_debug(tenant, project, UserId::from("user")); service } fn complete_terminal_matrix_child( service: &mut CoordinatorService, terminal_state: TaskTerminalState, ) { service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "terminal-matrix".to_owned(), node: "worker".to_owned(), task: "final-child".to_owned(), terminal_state: Some(terminal_state), status_code: Some(1), stdout_bytes: 0, stderr_bytes: 4, stdout_tail: String::new(), stderr_tail: "boom".to_owned(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap(); } fn test_agent_public_key() -> String { agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap() } fn signed_agent_workflow_request( request: &CoordinatorRequest, request_kind: &str, process: &str, task: Option<&str>, nonce: &str, ) -> AgentSignedRequest { let task = task.map(TaskInstanceId::from); let payload = serde_json::to_value(request).unwrap(); let payload_digest = signed_request_payload_digest(&payload); sign_agent_workflow_request( TEST_AGENT_PRIVATE_KEY, AgentWorkflowScope { tenant: &TenantId::from("tenant"), project: &ProjectId::from("project"), agent: &AgentId::from("agent-ci"), request_kind, process: &ProcessId::from(process), task: task.as_ref(), }, &payload_digest, nonce.to_owned(), unix_timestamp_seconds_for_tests(), ) .unwrap() } fn with_signed_agent_workflow( mut request: CoordinatorRequest, request_kind: &str, process: &str, task: Option<&str>, nonce: &str, ) -> CoordinatorRequest { let signature = signed_agent_workflow_request(&request, request_kind, process, task, nonce); match &mut request { CoordinatorRequest::StartProcess { launch_attempt: None, agent_signature, .. } | CoordinatorRequest::LaunchTask { agent_signature, .. } => *agent_signature = Some(signature), _ => panic!("agent signing helper only accepts agent workflow requests"), } request } fn test_node_private_key(node: &str) -> String { derive_ed25519_private_key_from_seed(&format!("test-node-key:{node}")) } fn test_node_public_key(node: &str) -> String { node_ed25519_public_key_from_private_key(&test_node_private_key(node)).unwrap() } fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest { signed_node_heartbeat_in_scope("tenant", "project", node, nonce) } fn signed_node_heartbeat_in_scope( tenant: &str, project: &str, node: &str, nonce: &str, ) -> NodeSignedRequest { let payload = json!({ "type": "node_heartbeat", "tenant": tenant, "project": project, "node": node }); signed_node_request_with_private_key( node, &test_node_private_key(node), "node_heartbeat", &signed_request_payload_digest(&payload), nonce, ) } fn signed_node_heartbeat_with_private_key( node: &str, private_key: &str, nonce: &str, ) -> NodeSignedRequest { signed_node_heartbeat_in_scope_with_private_key("tenant", "project", node, private_key, nonce) } fn signed_node_heartbeat_in_scope_with_private_key( tenant: &str, project: &str, node: &str, private_key: &str, nonce: &str, ) -> NodeSignedRequest { let payload = json!({ "type": "node_heartbeat", "tenant": tenant, "project": project, "node": node }); signed_node_request_with_private_key( node, private_key, "node_heartbeat", &signed_request_payload_digest(&payload), nonce, ) } fn signed_node_request_with_private_key( node: &str, private_key: &str, request_kind: &str, payload_digest: &Digest, nonce: &str, ) -> NodeSignedRequest { sign_node_request( private_key, &NodeId::from(node), request_kind, payload_digest, nonce.to_owned(), unix_timestamp_seconds_for_tests(), ) .unwrap() } fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { let node = match &request { CoordinatorRequest::ReportNodeCapabilities { node, .. } | CoordinatorRequest::PollTaskAssignment { node, .. } | CoordinatorRequest::PollArtifactTransfer { node, .. } | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } | CoordinatorRequest::FailArtifactTransfer { node, .. } | CoordinatorRequest::LaunchChildTask { node, .. } | CoordinatorRequest::JoinChildTask { node, .. } | CoordinatorRequest::CompleteSourcePreparation { node, .. } | CoordinatorRequest::ReconnectNode { node, .. } | CoordinatorRequest::PollTaskControl { node, .. } | CoordinatorRequest::PollDebugCommand { node, .. } | CoordinatorRequest::ReportDebugState { node, .. } | CoordinatorRequest::ReportDebugProbeHit { node, .. } | CoordinatorRequest::ReportTaskLog { node, .. } | CoordinatorRequest::ReportTaskLogChunk { node, .. } | CoordinatorRequest::ReportVfsMetadata { node, .. } | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), _ => panic!("test helper only signs node-originated requests"), }; signed_node_request_auto_with_private_key(request, &test_node_private_key(&node)) } fn signed_node_request_auto_with_private_key( request: CoordinatorRequest, private_key: &str, ) -> CoordinatorRequest { let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap()); let (node, request_kind) = match &request { CoordinatorRequest::ReportNodeCapabilities { node, .. } => { (node.clone(), "report_node_capabilities") } CoordinatorRequest::PollTaskAssignment { node, .. } => { (node.clone(), "poll_task_assignment") } CoordinatorRequest::PollArtifactTransfer { node, .. } => { (node.clone(), "poll_artifact_transfer") } CoordinatorRequest::UploadArtifactTransferChunk { node, .. } => { (node.clone(), "upload_artifact_transfer_chunk") } CoordinatorRequest::FailArtifactTransfer { node, .. } => { (node.clone(), "fail_artifact_transfer") } CoordinatorRequest::LaunchChildTask { node, .. } => (node.clone(), "launch_child_task"), CoordinatorRequest::JoinChildTask { node, .. } => (node.clone(), "join_child_task"), CoordinatorRequest::CompleteSourcePreparation { node, .. } => { (node.clone(), "complete_source_preparation") } CoordinatorRequest::RequestRendezvous { source, .. } => { (source.node.to_string(), "request_rendezvous") } CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"), CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"), CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"), CoordinatorRequest::ReportDebugState { node, .. } => (node.clone(), "report_debug_state"), CoordinatorRequest::ReportDebugProbeHit { node, .. } => { (node.clone(), "report_debug_probe_hit") } CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"), CoordinatorRequest::ReportTaskLogChunk { node, .. } => { (node.clone(), "report_task_log_chunk") } CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"), CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"), _ => panic!("test helper only signs node-originated requests"), }; let nonce = TEST_NODE_REQUEST_NONCE .fetch_add(1, Ordering::Relaxed) .to_string(); CoordinatorRequest::SignedNode { node: node.clone(), node_signature: signed_node_request_with_private_key( &node, private_key, request_kind, &payload_digest, &format!("node-request-{nonce}"), ), request: Box::new(request), } } trait SignedNodeRequestTestExt { fn handle_signed_node_request_auto( &mut self, request: CoordinatorRequest, ) -> Result; } impl SignedNodeRequestTestExt for CoordinatorService { fn handle_signed_node_request_auto( &mut self, request: CoordinatorRequest, ) -> Result { self.handle_request(signed_node_request_auto(request)) } } fn upload_pending_artifact_transfer( service: &mut CoordinatorService, node: &str, bytes: &[u8], ) -> ArtifactTransferAssignment { let mut first = None; loop { let CoordinatorResponse::ArtifactTransferAssignment { transfer: Some(transfer), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), }) .unwrap() else { panic!("expected pending artifact reverse transfer"); }; assert_eq!(transfer.expected_digest, Digest::sha256(bytes)); assert_eq!(transfer.expected_size_bytes, bytes.len() as u64); let start = transfer.offset as usize; let end = start .saturating_add(transfer.max_chunk_bytes as usize) .min(bytes.len()); let content = &bytes[start..end]; let response = service .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), transfer_id: transfer.transfer_id.clone(), artifact: transfer.artifact.as_str().to_owned(), offset: transfer.offset, content_base64: BASE64_STANDARD.encode(content), chunk_digest: Digest::sha256(content), eof: end == bytes.len(), }) .unwrap(); let complete = matches!( response, CoordinatorResponse::ArtifactTransferChunkAccepted { complete: true, .. } ); first.get_or_insert_with(|| transfer.clone()); if complete { return first.unwrap(); } } } fn unix_timestamp_seconds_for_tests() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|duration| duration.as_secs()) .unwrap_or(0) } fn write_coordinator_wire_request( stream: &mut TcpStream, request: &CoordinatorRequest, request_id: &str, ) { let payload = serde_json::to_value(request).unwrap(); let wire_request = coordinator_wire_request(request_id, payload); serde_json::to_writer(&mut *stream, &wire_request).unwrap(); stream.write_all(b"\n").unwrap(); stream.flush().unwrap(); } #[test] fn service_creates_selects_and_lists_signed_in_user_projects() { let mut service = CoordinatorService::new(7); let CoordinatorResponse::ProjectCreated { project, actor } = service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant-a".to_owned(), actor_user: "user-a".to_owned(), project: "project-a".to_owned(), name: "Demo".to_owned(), }) .unwrap() else { panic!("expected project creation"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(project.tenant, TenantId::from("tenant-a")); assert_eq!(project.id, ProjectId::from("project-a")); assert_eq!(project.name, "Demo"); let CoordinatorResponse::ProjectSelected { project, actor } = service .handle_request(CoordinatorRequest::SelectProject { tenant: "tenant-a".to_owned(), actor_user: "user-a".to_owned(), project: "project-a".to_owned(), }) .unwrap() else { panic!("expected project selection"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(project.id, ProjectId::from("project-a")); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant-b".to_owned(), actor_user: "user-b".to_owned(), project: "project-b".to_owned(), name: "Other".to_owned(), }) .unwrap(); let CoordinatorResponse::Projects { projects, actor } = service .handle_request(CoordinatorRequest::ListProjects { tenant: "tenant-a".to_owned(), actor_user: "user-a".to_owned(), }) .unwrap() else { panic!("expected project list"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(projects.len(), 1); assert_eq!(projects[0].id, ProjectId::from("project-a")); let cross_tenant = service .handle_request(CoordinatorRequest::SelectProject { tenant: "tenant-a".to_owned(), actor_user: "user-a".to_owned(), project: "project-b".to_owned(), }) .unwrap_err(); assert!(cross_tenant.to_string().contains("tenant scope")); } #[test] fn authenticated_envelope_derives_user_scope_from_cli_session() { let mut service = CoordinatorService::new(7); service .issue_cli_session( TenantId::from("tenant-a"), ProjectId::from("project-a"), UserId::from("user-a"), "cli-session-secret", None, ) .unwrap(); let CoordinatorResponse::AuthStatus { tenant, project, actor, authenticated, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap() else { panic!("expected authenticated auth status"); }; assert_eq!(tenant, TenantId::from("tenant-a")); assert_eq!(project, ProjectId::from("project-a")); assert_eq!(actor, UserId::from("user-a")); assert!(authenticated); let CoordinatorResponse::ProjectCreated { project, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::CreateProject { project: "project-b".to_owned(), name: "From Session".to_owned(), }, }) .unwrap() else { panic!("expected authenticated project creation"); }; assert_eq!(project.tenant, TenantId::from("tenant-a")); assert_eq!(project.id, ProjectId::from("project-b")); assert_eq!(actor, UserId::from("user-a")); let CoordinatorResponse::Projects { projects, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ListProjects, }) .unwrap() else { panic!("expected authenticated project list"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(projects.len(), 2); assert!(projects .iter() .any(|project| project.id == ProjectId::from("project-a"))); assert!(projects .iter() .any(|project| project.id == ProjectId::from("project-b"))); let CoordinatorResponse::AgentPublicKey { record, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { agent: "agent-session".to_owned(), public_key: "agent-session-key-v1".to_owned(), }, }) .unwrap() else { panic!("expected authenticated agent key registration"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(record.tenant, TenantId::from("tenant-a")); assert_eq!(record.project, ProjectId::from("project-a")); assert_eq!(record.user, UserId::from("user-a")); assert_eq!(record.agent, AgentId::from("agent-session")); let CoordinatorResponse::AgentPublicKeys { records, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ListAgentPublicKeys, }) .unwrap() else { panic!("expected authenticated agent key list"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(records.len(), 1); assert_eq!(records[0].project, ProjectId::from("project-a")); let CoordinatorResponse::AgentPublicKey { record, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent: "agent-session".to_owned(), }, }) .unwrap() else { panic!("expected authenticated agent key revocation"); }; assert_eq!(actor, UserId::from("user-a")); assert!(record.revoked); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "session-node".to_owned(), public_key: test_node_public_key("session-node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "session-node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![], dependency_cache_digests: vec![], source_snapshots: vec![], artifact_locations: vec![], direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeDescriptors, }) .unwrap() else { panic!("expected authenticated node descriptor list"); }; assert_eq!(actor, UserId::from("user-a")); assert_eq!(descriptors.len(), 1); assert_eq!(descriptors[0].id, NodeId::from("session-node")); let CoordinatorResponse::TaskPlacement { placement } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ScheduleTask { environment: None, environment_digest: None, required_capabilities: vec![Capability::Command], dependency_cache: None, source_snapshot: None, required_artifacts: vec![], prefer_node: Some("session-node".to_owned()), }, }) .unwrap() else { panic!("expected authenticated task placement"); }; assert_eq!(placement.node, NodeId::from("session-node")); let CoordinatorResponse::ProcessStarted { launch_attempt: None, process, actor, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "vp-session".to_owned(), restart: false, }, }) .unwrap() else { panic!("expected authenticated process start"); }; assert_eq!(process, ProcessId::from("vp-session")); assert_eq!(actor.user, Some(UserId::from("user-a"))); let denied_external_task = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::LaunchTask { task_spec: Box::new(test_task_spec( "tenant-a", "project-a", "vp-session", "task-session", 7, [Capability::Command], )), wait_for_node: false, artifact_path: "/vfs/artifacts/session.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }, }) .unwrap_err(); assert!(denied_external_task .to_string() .contains("external callers may launch only EntrypointV1")); let CoordinatorResponse::TaskLaunched { process, task, actor, assignment, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant-a", "project-a", "vp-session", "task-session", 7, [Capability::Command], ), tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/session.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap() else { panic!("expected authenticated task launch"); }; assert_eq!(process, ProcessId::from("vp-session")); assert_eq!(task, TaskInstanceId::from("task-session")); assert_eq!(actor.kind, "task"); assert_eq!(assignment.tenant, TenantId::from("tenant-a")); assert_eq!(assignment.project, ProjectId::from("project-a")); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "session-node".to_owned(), process: "vp-session".to_owned(), epoch: 7, }) .unwrap(); let artifact_bytes = "session artifact"; service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), process: "vp-session".to_owned(), node: "session-node".to_owned(), task: "task-session".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: artifact_bytes.len() as u64, stderr_bytes: 0, stdout_tail: artifact_bytes.to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/session.txt".to_owned()), artifact_digest: Some(Digest::sha256(artifact_bytes)), artifact_size_bytes: Some(artifact_bytes.len() as u64), result: None, }) .unwrap(); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ListTaskEvents { process: Some("vp-session".to_owned()), }, }) .unwrap() else { panic!("expected authenticated task event list"); }; assert_eq!(events.len(), 1); assert_eq!(events[0].tenant, TenantId::from("tenant-a")); assert_eq!(events[0].project, ProjectId::from("project-a")); service .issue_cli_session( TenantId::from("tenant-b"), ProjectId::from("project-victim"), UserId::from("user-b"), "victim-cli-session-secret", None, ) .unwrap(); service .handle_request(CoordinatorRequest::Authenticated { session_secret: "victim-cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "vp-victim".to_owned(), restart: false, }, }) .unwrap(); let cross_tenant_events = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::ListTaskEvents { process: Some("vp-victim".to_owned()), }, }) .unwrap_err(); assert!(cross_tenant_events .to_string() .contains("outside the virtual process tenant/project scope")); let CoordinatorResponse::ArtifactDownloadLink { link } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact: "session.txt".to_owned(), max_bytes: 1024, ttl_seconds: 60, }, }) .unwrap() else { panic!("expected authenticated artifact link"); }; assert_eq!(link.tenant, TenantId::from("tenant-a")); assert_eq!(link.project, ProjectId::from("project-a")); assert_eq!(link.actor, Actor::User(UserId::from("user-a"))); let CoordinatorResponse::ProcessCancellationRequested { process, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::CancelProcess { process: "vp-session".to_owned(), }, }) .unwrap() else { panic!("expected authenticated process cancellation"); }; assert_eq!(process, ProcessId::from("vp-session")); let CoordinatorResponse::DebugAttach { actor, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::DebugAttach { process: "vp-session".to_owned(), }, }) .unwrap() else { panic!("expected authenticated debug attach"); }; assert_eq!(actor, UserId::from("user-a")); let CoordinatorResponse::NodeCredentialRevoked { node, tenant, project, actor, descriptor_removed, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::RevokeNodeCredential { node: "session-node".to_owned(), }, }) .unwrap() else { panic!("expected authenticated node credential revocation"); }; assert_eq!(node, NodeId::from("session-node")); assert_eq!(tenant, TenantId::from("tenant-a")); assert_eq!(project, ProjectId::from("project-a")); assert_eq!(actor, UserId::from("user-a")); assert!(descriptor_removed); let rejected = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "wrong-secret".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap_err(); assert!(rejected.to_string().contains("not recognized")); let expired = service .issue_cli_session( TenantId::from("tenant-a"), ProjectId::from("project-b"), UserId::from("user-a"), "expired-cli-session-secret", Some(1), ) .unwrap(); assert_eq!(expired.expires_at_epoch_seconds, Some(1)); let expired_rejected = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "expired-cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap_err(); assert!(expired_rejected.to_string().contains("expired")); let CoordinatorResponse::CliSessionRevoked { tenant, project, actor, } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::RevokeCliSession, }) .unwrap() else { panic!("expected CLI session revocation"); }; assert_eq!(tenant, TenantId::from("tenant-a")); assert_eq!(project, ProjectId::from("project-a")); assert_eq!(actor, UserId::from("user-a")); let revoked_rejected = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap_err(); assert!(revoked_rejected.to_string().contains("revoked")); } #[test] fn service_reports_and_enforces_public_admin_tenant_suspension() { let mut service = CoordinatorService::new_with_admin_token(7, "admin-token"); let CoordinatorResponse::AuthStatus { tenant, project, actor, authenticated, account_status, suspended, disabled, sanitized_reason, sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service .handle_request(CoordinatorRequest::AuthStatus { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), }) .unwrap() else { panic!("expected auth status"); }; assert_eq!(tenant, TenantId::from("tenant")); assert_eq!(project, ProjectId::from("project")); assert_eq!(actor, UserId::from("user")); assert!(authenticated); assert_eq!(account_status, "active"); assert!(!suspended); assert!(!disabled); assert!(sanitized_reason.is_none()); assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); for (tenant, policy_name, expected_status, expected_reason) in [ ( "manual-tenant", "tenant:manual_review", "manual_review", "account or tenant is pending hosted review", ), ( "disabled-tenant", "tenant:disabled", "disabled", "account or tenant is disabled by hosted policy", ), ( "deleted-tenant", "tenant:deleted", "deleted", "account or tenant is no longer active", ), ] { service.coordinator.upsert_service_policy_record( TenantId::from(tenant), policy_name, Digest::from_parts([tenant.as_bytes(), policy_name.as_bytes()]), ); let CoordinatorResponse::AuthStatus { account_status, suspended, disabled, deleted, manual_review, sanitized_reason, next_actions, sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service .handle_request(CoordinatorRequest::AuthStatus { tenant: tenant.to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), }) .unwrap() else { panic!("expected inactive auth status"); }; assert_eq!(account_status, expected_status); assert_eq!(suspended, expected_status == "suspended"); assert_eq!(disabled, expected_status == "disabled"); assert_eq!(deleted, expected_status == "deleted"); assert_eq!(manual_review, expected_status == "manual_review"); assert_eq!(sanitized_reason.as_deref(), Some(expected_reason)); assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); } let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request( "admin-token", "admin_status", "tenant", "admin", "tenant", "admin-status-1", ); let admin_status_request = CoordinatorRequest::AdminStatus { tenant: "tenant".to_owned(), actor_user: "admin".to_owned(), admin_proof, admin_nonce, issued_at_epoch_seconds, }; let CoordinatorResponse::AdminStatus { tenant, actor, suspended, safe_default, } = service .handle_request(admin_status_request.clone()) .unwrap() else { panic!("expected admin status"); }; assert_eq!(tenant, TenantId::from("tenant")); assert_eq!(actor, UserId::from("admin")); assert!(!suspended); assert_eq!(safe_default, "read_only"); let replayed_admin_status = service.handle_request(admin_status_request).unwrap_err(); assert!(replayed_admin_status .to_string() .contains("nonce was already used")); let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request( "admin-token", "suspend_tenant", "admin-tenant", "admin", "tenant", "admin-suspend-1", ); let CoordinatorResponse::TenantSuspended { tenant, actor, policy, } = service .handle_request(CoordinatorRequest::SuspendTenant { tenant: "admin-tenant".to_owned(), actor_user: "admin".to_owned(), target_tenant: "tenant".to_owned(), admin_proof, admin_nonce, issued_at_epoch_seconds, }) .unwrap() else { panic!("expected tenant suspension"); }; assert_eq!(tenant, TenantId::from("tenant")); assert_eq!(actor, UserId::from("admin")); assert_eq!(policy.name, "tenant:suspended"); let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request( "admin-token", "admin_status", "tenant", "admin", "tenant", "admin-status-2", ); let CoordinatorResponse::AdminStatus { suspended, .. } = service .handle_request(CoordinatorRequest::AdminStatus { tenant: "tenant".to_owned(), actor_user: "admin".to_owned(), admin_proof, admin_nonce, issued_at_epoch_seconds, }) .unwrap() else { panic!("expected suspended admin status"); }; assert!(suspended); let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request( "admin-token", "admin_status", "tenant", "admin", "tenant", "admin-status-unconfigured", ); let missing_admin_credential = CoordinatorService::new(7) .handle_request(CoordinatorRequest::AdminStatus { tenant: "tenant".to_owned(), actor_user: "admin".to_owned(), admin_proof, admin_nonce, issued_at_epoch_seconds, }) .unwrap_err(); assert!(missing_admin_credential .to_string() .contains("admin credential is not configured")); let (admin_proof, admin_nonce, issued_at_epoch_seconds) = test_admin_request( "wrong-token", "suspend_tenant", "admin-tenant", "admin", "other-tenant", "admin-suspend-wrong", ); let invalid_admin_credential = service .handle_request(CoordinatorRequest::SuspendTenant { tenant: "admin-tenant".to_owned(), actor_user: "admin".to_owned(), target_tenant: "other-tenant".to_owned(), admin_proof, admin_nonce, issued_at_epoch_seconds, }) .unwrap_err(); assert!(invalid_admin_credential .to_string() .contains("admin request proof is invalid")); let CoordinatorResponse::AuthStatus { account_status, suspended, disabled, sanitized_reason, next_actions, sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service .handle_request(CoordinatorRequest::AuthStatus { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), }) .unwrap() else { panic!("expected suspended auth status"); }; assert_eq!(account_status, "suspended"); assert!(suspended); assert!(!disabled); assert_eq!( sanitized_reason.as_deref(), Some("account or tenant is suspended by hosted policy") ); assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); let create = service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant".to_owned(), actor_user: "user".to_owned(), project: "project".to_owned(), name: "Demo".to_owned(), }) .unwrap_err(); assert!(create.to_string().contains("tenant is suspended")); let start = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: Some("attempt-a".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap_err(); assert!(start.to_string().contains("tenant is suspended")); } #[test] fn service_manages_project_scoped_agent_public_keys() { let mut service = CoordinatorService::new(7); let CoordinatorResponse::AgentPublicKey { record, actor } = service .handle_request(CoordinatorRequest::RegisterAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), agent: "agent-ci".to_owned(), public_key: "agent-key-v1".to_owned(), }) .unwrap() else { panic!("expected agent public key registration"); }; assert_eq!(actor, UserId::from("user")); assert_eq!(record.tenant, TenantId::from("tenant")); assert_eq!(record.project, ProjectId::from("project")); assert_eq!(record.user, UserId::from("user")); assert_eq!(record.agent, AgentId::from("agent-ci")); assert_eq!(record.version, 1); assert!(!record.revoked); assert_eq!(record.scopes, vec!["project:read", "project:run"]); assert!(!record.human_account_creation_privilege); assert!(!record.browser_interaction_required_each_run); let CoordinatorResponse::AgentPublicKey { record, .. } = service .handle_request(CoordinatorRequest::RegisterAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), agent: "agent-ci".to_owned(), public_key: "agent-key-v2".to_owned(), }) .unwrap() else { panic!("expected agent public key rotation by re-add"); }; assert_eq!(record.version, 2); assert_eq!(record.public_key, "agent-key-v2"); let CoordinatorResponse::AgentPublicKeys { records, actor } = service .handle_request(CoordinatorRequest::ListAgentPublicKeys { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), }) .unwrap() else { panic!("expected agent public key list"); }; assert_eq!(actor, UserId::from("user")); assert_eq!(records.len(), 1); assert_eq!(records[0].agent, AgentId::from("agent-ci")); let CoordinatorResponse::AgentPublicKeys { records, .. } = service .handle_request(CoordinatorRequest::ListAgentPublicKeys { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "other-user".to_owned(), }) .unwrap() else { panic!("expected foreign user agent public key list"); }; assert!(records.is_empty()); let CoordinatorResponse::AgentPublicKeys { records, .. } = service .handle_request(CoordinatorRequest::ListAgentPublicKeys { tenant: "other-tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), }) .unwrap() else { panic!("expected foreign tenant agent public key list"); }; assert!(records.is_empty()); let foreign_user_revoke = service .handle_request(CoordinatorRequest::RevokeAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "other-user".to_owned(), agent: "agent-ci".to_owned(), }) .unwrap_err(); assert!(foreign_user_revoke .to_string() .contains("signed-in user scope")); let CoordinatorResponse::AgentPublicKey { record, actor } = service .handle_request(CoordinatorRequest::RevokeAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), agent: "agent-ci".to_owned(), }) .unwrap() else { panic!("expected agent public key revocation"); }; assert_eq!(actor, UserId::from("user")); assert!(record.revoked); } #[test] fn service_runs_agent_workflows_with_scoped_key_attribution() { let mut service = CoordinatorService::new(7); let agent_public_key = test_agent_public_key(); let agent_fingerprint = Digest::sha256(&agent_public_key); service .handle_request(CoordinatorRequest::RegisterAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), agent: "agent-ci".to_owned(), public_key: agent_public_key, }) .unwrap(); let fingerprint_only = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(agent_fingerprint.clone()), agent_signature: None, process: "vp-agent-fingerprint-only".to_owned(), restart: false, }) .unwrap_err(); assert!(fingerprint_only.to_string().contains("signed request")); let wrong_fingerprint = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(Digest::sha256("other-key")), agent_signature: None, process: "vp-agent-bad-key".to_owned(), restart: false, }, "start_process", "vp-agent-bad-key", None, "bad-fingerprint-nonce", )) .unwrap_err(); assert!(wrong_fingerprint.to_string().contains("fingerprint")); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), public_key: test_node_public_key("worker-linux"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::ProcessStarted { launch_attempt: None, actor: start_actor, .. } = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(agent_fingerprint.clone()), agent_signature: None, process: "vp-agent".to_owned(), restart: false, }, "start_process", "vp-agent", None, "start-nonce", )) .unwrap() else { panic!("expected agent-authenticated process start"); }; assert_agent_workflow_actor(&start_actor, &agent_fingerprint); let replay = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(agent_fingerprint.clone()), agent_signature: None, process: "vp-agent".to_owned(), restart: true, }, "start_process", "vp-agent", None, "start-nonce", )) .unwrap_err(); assert!(replay.to_string().contains("nonce")); let denied_external_task = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "vp-agent", "compile-linux", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(agent_fingerprint.clone()), agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }, "launch_task", "vp-agent", Some("compile-linux"), "launch-nonce", )) .unwrap_err(); assert!(denied_external_task .to_string() .contains("external callers may launch only EntrypointV1")); } #[test] fn signed_node_and_agent_requests_reject_body_modification() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-bound-body".to_owned(), public_key: test_node_public_key("worker-bound-body"), }) .unwrap(); let original_node_request = CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-bound-body".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }; let original_node_digest = signed_request_payload_digest(&serde_json::to_value(&original_node_request).unwrap()); let node_signature = signed_node_request_with_private_key( "worker-bound-body", &test_node_private_key("worker-bound-body"), "report_node_capabilities", &original_node_digest, "node-body-modification", ); let mut modified_node_request = original_node_request; let CoordinatorRequest::ReportNodeCapabilities { online, .. } = &mut modified_node_request else { unreachable!(); }; *online = false; let modified_node = service .handle_request(CoordinatorRequest::SignedNode { node: "worker-bound-body".to_owned(), node_signature, request: Box::new(modified_node_request), }) .unwrap_err(); assert!(modified_node.to_string().contains("signature")); let agent_public_key = test_agent_public_key(); let agent_fingerprint = Digest::sha256(&agent_public_key); service .handle_request(CoordinatorRequest::RegisterAgentPublicKey { tenant: "tenant".to_owned(), project: "project".to_owned(), user: "user".to_owned(), agent: "agent-ci".to_owned(), public_key: agent_public_key, }) .unwrap(); let original_agent_request = CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent-ci".to_owned()), agent_public_key_fingerprint: Some(agent_fingerprint), agent_signature: None, process: "vp-agent-bound-body".to_owned(), restart: false, }; let agent_signature = signed_agent_workflow_request( &original_agent_request, "start_process", "vp-agent-bound-body", None, "agent-body-modification", ); let mut modified_agent_request = original_agent_request; let CoordinatorRequest::StartProcess { launch_attempt: None, restart, agent_signature: request_signature, .. } = &mut modified_agent_request else { unreachable!(); }; *restart = true; *request_signature = Some(agent_signature); let modified_agent = service.handle_request(modified_agent_request).unwrap_err(); assert!(modified_agent.to_string().contains("signature")); } #[test] fn service_checks_spawn_quota_before_process_or_task_work_starts() { let mut service = CoordinatorService::new(7); service.quota.set_workflow_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::Spawn, 2)]), }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), public_key: test_node_public_key("worker-linux"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::ProcessStarted { launch_attempt: None, charged_spawns, .. } = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-quota".to_owned(), restart: false, }) .unwrap() else { panic!("expected process start within spawn quota"); }; assert_eq!(charged_spawns, 1); let CoordinatorResponse::TaskLaunched { charged_spawns, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "vp-quota", "compile-linux", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap() else { panic!("expected task launch within spawn quota"); }; assert_eq!(charged_spawns, 2); let denied_task = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "vp-quota", "compile-linux-denied", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/denied.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap_err(); assert!(denied_task.to_string().contains("Spawn")); let now_epoch_seconds = service.current_epoch_seconds().unwrap(); assert_eq!( service.quota.used_workflow_spawns( &TenantId::from("tenant"), &ProjectId::from("project"), now_epoch_seconds, ), 2 ); let denied_task_key = task_control_key( &TenantId::from("tenant"), &ProjectId::from("project"), &ProcessId::from("vp-quota"), &NodeId::from("worker-linux"), &TaskInstanceId::from("compile-linux-denied"), ); assert!(!service.active_tasks.contains(&denied_task_key)); assert!(!service.task_placements.contains_key(&denied_task_key)); assert_eq!( service .task_assignments .get(&( TenantId::from("tenant"), ProjectId::from("project"), NodeId::from("worker-linux"), )) .map(VecDeque::len), Some(1) ); let other_project_process = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "other-project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-denied".to_owned(), restart: false, }) .unwrap(); assert!(matches!( other_project_process, CoordinatorResponse::ProcessStarted { launch_attempt: None, .. } )); assert_eq!( service.quota.used_workflow_spawns( &TenantId::from("tenant"), &ProjectId::from("other-project"), now_epoch_seconds, ), 1 ); assert_eq!( service.quota.used_workflow_spawns( &TenantId::from("tenant"), &ProjectId::from("project"), now_epoch_seconds, ), 2 ); } #[test] fn project_quota_resets_at_the_configured_window_boundary() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::Spawn, 1); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::Spawn, 60)]).unwrap(); let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( 7, "test-admin-token", None, quota, ) .unwrap(); service.set_server_time(59); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: Some("attempt-b".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-window-one".to_owned(), restart: false, }) .unwrap(); service .handle_request(CoordinatorRequest::AbortProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp-window-one".to_owned(), }) .unwrap(); let exhausted = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: Some("attempt-b".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-still-window-one".to_owned(), restart: false, }) .unwrap_err(); assert!(exhausted.to_string().contains("Spawn")); service.set_server_time(60); let started = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-window-two".to_owned(), restart: false, }) .unwrap(); assert!(matches!( started, CoordinatorResponse::ProcessStarted { launch_attempt: None, charged_spawns: 1, .. } )); assert_eq!(service.quota.active_meter_count(), 1); } #[test] fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::ApiCall, 1); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::ApiCall, 60)]).unwrap(); let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( 7, "test-admin-token", None, quota, ) .unwrap(); service.set_server_time(30); for (project, secret) in [("project-a", "session-a"), ("project-b", "session-b")] { service .issue_cli_session( TenantId::from("tenant"), ProjectId::from(project), UserId::from("user"), secret, None, ) .unwrap(); } service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap(); let denied = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap_err(); assert!(denied.to_string().contains("ApiCall")); service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-b".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }) .unwrap(); assert_eq!( service .quota .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-a"), 30,), 1 ); assert_eq!( service .quota .used_api_calls(&TenantId::from("tenant"), &ProjectId::from("project-b"), 30,), 1 ); } #[test] fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::LogBytes, 4); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( 7, "test-admin-token", None, quota, ) .unwrap(); service.set_server_time(30); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "task".to_owned(), stdout_bytes: 3, stderr_bytes: 1, stdout_tail: "out".to_owned(), stderr_tail: "e".to_owned(), stdout_truncated: false, stderr_truncated: false, backpressured: false, }) .unwrap(); let truncated = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "task".to_owned(), stdout_bytes: 4, stderr_bytes: 1, stdout_tail: "outx".to_owned(), stderr_tail: "e".to_owned(), stdout_truncated: false, stderr_truncated: false, backpressured: true, }) .unwrap(); let CoordinatorResponse::TaskLogRecorded { stdout_tail, stdout_bytes: 4, .. } = truncated else { panic!("expected a successful truncated task-log report"); }; assert_eq!(stdout_tail, "[log output truncated at project log quota]"); assert!(service .recent_logs .get(&(TenantId::from("tenant"), ProjectId::from("project"))) .unwrap() .iter() .any(|entry| entry.text.contains("project log quota") && entry.truncated)); assert_eq!( service .quota .used_log_bytes(&TenantId::from("tenant"), &ProjectId::from("project"), 30,), 4 ); } #[test] fn log_quota_exhaustion_cannot_strand_task_completion_or_artifact_publication() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::LogBytes, 4); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( 7, "test-admin-token", None, quota, ) .unwrap(); service.set_server_time(30); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile", "compile-one", 7, ); service.record_task_completion_event(TaskCompletionEvent { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("process"), node: NodeId::from("coordinator-main"), executor: TaskExecutor::CoordinatorMain, task_definition: TaskDefinitionId::from("build"), task: TaskInstanceId::from("main"), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-one".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 64, stderr_bytes: 0, stdout_tail: "the-real-final-tail".to_owned(), stderr_tail: String::new(), stdout_truncated: true, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/result.bin".to_owned()), artifact_digest: Some(Digest::sha256("artifact bytes")), artifact_size_bytes: Some(14), result: None, }) .unwrap(); let event = service .task_events .iter() .find(|event| event.task == TaskInstanceId::from("compile-one")) .unwrap(); assert_eq!( event.stdout_tail, "[log output truncated at project log quota]" ); assert!(event.stdout_truncated); assert!(!service .active_tasks .iter() .any(|key| key.4 == TaskInstanceId::from("compile-one"))); assert!(service .coordinator .active_process( &TenantId::from("tenant"), &ProjectId::from("project"), &ProcessId::from("process"), ) .is_none()); assert!(matches!( service .handle_request(CoordinatorRequest::GetArtifact { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "result.bin".to_owned(), }) .unwrap(), CoordinatorResponse::Artifact { .. } )); } #[test] fn service_attaches_node_starts_process_and_records_scoped_task_event() { let mut service = CoordinatorService::new(7); let attached = service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); assert!(matches!(attached, CoordinatorResponse::NodeAttached { .. })); let started = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); assert_eq!( started, CoordinatorResponse::ProcessStarted { launch_attempt: None, process: ProcessId::from("process"), epoch: 7, actor: WorkflowActor { kind: "user".to_owned(), user: Some(UserId::from("user")), agent: None, credential_kind: CredentialKind::BrowserSession, public_key_fingerprint: None, authenticated_without_browser: false, scopes: vec!["project:read".to_owned(), "project:run".to_owned()], }, charged_spawns: 1, } ); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")), }) .unwrap(); assert_eq!( heartbeat, CoordinatorResponse::NodeHeartbeat { node: NodeId::from("node"), epoch: 7, } ); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile-linux", "compile-linux", 7, ); let recorded = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 12, stderr_bytes: 0, stdout_tail: "build ok".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/app.txt".to_owned()), artifact_digest: Some(Digest::sha256("artifact")), artifact_size_bytes: Some(12), result: None, }) .unwrap(); assert_eq!( recorded, CoordinatorResponse::TaskRecorded { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), events_recorded: 1 } ); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap() else { panic!("expected task events"); }; assert_eq!(events.len(), 1); assert_eq!(events[0].node, NodeId::from("node")); assert_eq!(events[0].stdout_tail, "build ok"); assert_eq!(events[0].stderr_tail, ""); assert!(!events[0].stdout_truncated); let oversized_tail = "x".repeat(MAX_TASK_LOG_TAIL_BYTES + 1); let oversized_log = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), stdout_bytes: oversized_tail.len() as u64, stderr_bytes: 0, stdout_tail: oversized_tail.clone(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, backpressured: false, }) .unwrap_err(); assert!(oversized_log.to_string().contains("stdout_tail")); let oversized_completion = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: oversized_tail.len() as u64, stderr_bytes: 0, stdout_tail: oversized_tail, stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap_err(); assert!(oversized_completion.to_string().contains("stdout_tail")); let cross_tenant = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "other".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap_err(); assert!(cross_tenant .to_string() .contains("outside the virtual process tenant/project scope")); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("other-process".to_owned()), }) .unwrap() else { panic!("expected task events"); }; assert!(events.is_empty()); } #[test] fn service_revokes_node_credentials_and_live_descriptors() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![], dependency_cache_digests: vec![], source_snapshots: vec![], artifact_locations: vec![], direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::NodeCredentialRevoked { node, tenant, project, actor, descriptor_removed, queued_assignments_removed, } = service .handle_request(CoordinatorRequest::RevokeNodeCredential { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), node: "node".to_owned(), }) .unwrap() else { panic!("expected node credential revocation"); }; assert_eq!(node, NodeId::from("node")); assert_eq!(tenant, TenantId::from("tenant")); assert_eq!(project, ProjectId::from("project")); assert_eq!(actor, UserId::from("user")); assert!(descriptor_removed); assert_eq!(queued_assignments_removed, 0); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")), }) .unwrap_err(); assert!(heartbeat.to_string().contains("not enrolled")); let CoordinatorResponse::NodeDescriptors { descriptors, .. } = service .handle_request(CoordinatorRequest::ListNodeDescriptors { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), }) .unwrap() else { panic!("expected node descriptors"); }; assert!(descriptors.is_empty()); } #[test] fn duplicate_node_ids_are_isolated_across_identity_replay_liveness_and_revocation() { let mut service = CoordinatorService::new(19); let node = "shared-node"; let private_a = test_node_private_key("tenant-a-shared-node"); let private_b = test_node_private_key("tenant-b-shared-node"); let private_c = test_node_private_key("tenant-a-other-project-shared-node"); let public_a = node_ed25519_public_key_from_private_key(&private_a).unwrap(); let public_b = node_ed25519_public_key_from_private_key(&private_b).unwrap(); let public_c = node_ed25519_public_key_from_private_key(&private_c).unwrap(); for (tenant, project, public_key) in [ ("tenant-a", "project-a", public_a), ("tenant-b", "project-b", public_b), ("tenant-a", "project-c", public_c), ] { service .handle_request(CoordinatorRequest::AttachNode { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), public_key, }) .unwrap(); } assert_eq!( service .coordinator .node_identity_count_for_tenant(&TenantId::from("tenant-a")), 2 ); assert_eq!( service .coordinator .node_identity_count_for_tenant(&TenantId::from("tenant-b")), 1 ); for (index, (tenant, project, private_key)) in [ ("tenant-a", "project-a", private_a.as_str()), ("tenant-b", "project-b", private_b.as_str()), ("tenant-a", "project-c", private_c.as_str()), ] .into_iter() .enumerate() { service.set_server_time(100 + index as u64); service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( tenant, project, node, private_key, "same-nonce", )), }) .unwrap(); service .handle_request(signed_node_request_auto_with_private_key( CoordinatorRequest::ReportNodeCapabilities { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![Digest::sha256(format!( "cache-{tenant}-{project}" ))], dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: index != 1, online: true, }, private_key, )) .unwrap(); } let replay = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: node.to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( "tenant-a", "project-a", node, &private_a, "same-nonce", )), }) .unwrap_err(); assert!(replay.to_string().contains("nonce")); let forged = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), node: node.to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( "tenant-b", "project-b", node, &private_a, "forged-cross-scope", )), }) .unwrap_err(); assert!(forged.to_string().contains("signature")); let scope_a = crate::NodeScopeKey::new( TenantId::from("tenant-a"), ProjectId::from("project-a"), NodeId::from(node), ); let scope_b = crate::NodeScopeKey::new( TenantId::from("tenant-b"), ProjectId::from("project-b"), NodeId::from(node), ); let scope_c = crate::NodeScopeKey::new( TenantId::from("tenant-a"), ProjectId::from("project-c"), NodeId::from(node), ); assert!(service.node_descriptors.contains_key(&scope_a)); assert!(service.node_descriptors.contains_key(&scope_b)); assert!(service.node_descriptors.contains_key(&scope_c)); assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_a)); assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_b)); assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_c)); assert_eq!(service.node_last_seen_epoch_seconds[&scope_a], 100); assert_eq!(service.node_last_seen_epoch_seconds[&scope_b], 101); assert_eq!(service.node_last_seen_epoch_seconds[&scope_c], 102); assert!(service.node_descriptors[&scope_a].direct_connectivity); assert!(!service.node_descriptors[&scope_b].direct_connectivity); assert!(service.node_descriptors[&scope_c].direct_connectivity); assert!(service .node_replay_nonces .contains_key(&(scope_a.clone(), "same-nonce".to_owned()))); assert!(service .node_replay_nonces .contains_key(&(scope_b.clone(), "same-nonce".to_owned()))); assert!(service .node_replay_nonces .contains_key(&(scope_c.clone(), "same-nonce".to_owned()))); service .handle_request(CoordinatorRequest::RevokeNodeCredential { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), actor_user: "user-a".to_owned(), node: node.to_owned(), }) .unwrap(); assert!(service .coordinator .node_identity( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), &NodeId::from(node), ) .is_none()); assert!(service .coordinator .node_identity( &TenantId::from("tenant-b"), &ProjectId::from("project-b"), &NodeId::from(node), ) .is_some()); assert!(!service.node_descriptors.contains_key(&scope_a)); assert!(service.node_descriptors.contains_key(&scope_b)); assert!(service.node_descriptors.contains_key(&scope_c)); service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), node: node.to_owned(), node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( "tenant-b", "project-b", node, &private_b, "post-other-scope-revocation", )), }) .expect("revoking tenant A's duplicate node must not affect tenant B"); } #[test] fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile-linux", "compile-linux", 7, ); let cancelled = service .handle_request(CoordinatorRequest::CancelTask { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), }) .unwrap(); assert_eq!( cancelled, CoordinatorResponse::TaskCancellationRequested { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), node: NodeId::from("node"), } ); let control = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), }) .unwrap(); assert_eq!( control, CoordinatorResponse::TaskControl { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), cancel_requested: true, abort_requested: false, } ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: Some(TaskTerminalState::Cancelled), status_code: None, stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap(); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap() else { panic!("expected task events"); }; assert_eq!(events[0].terminal_state, TaskTerminalState::Cancelled); let control = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), }) .unwrap(); assert_eq!( control, CoordinatorResponse::TaskControl { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), cancel_requested: false, abort_requested: false, } ); } #[test] fn service_authorizes_debug_attach_through_public_api() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant".to_owned(), actor_user: "user".to_owned(), project: "project".to_owned(), name: "Demo".to_owned(), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); let CoordinatorResponse::DebugAttach { process, actor, authorization, audit_event, charged_debug_read_bytes, used_debug_read_bytes, } = service .handle_request(CoordinatorRequest::DebugAttach { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected debug attach authorization"); }; assert_eq!(process, ProcessId::from("process")); assert_eq!(actor, UserId::from("user")); assert!(authorization.allowed); assert_eq!(audit_event.operation, "debug_attach"); assert_eq!(audit_event.actor, UserId::from("user")); assert!(audit_event.allowed); assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES); assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES); let CoordinatorResponse::DebugAttach { authorization, audit_event, charged_debug_read_bytes, used_debug_read_bytes, .. } = service .handle_request(CoordinatorRequest::DebugAttach { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "other-user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected denied debug attach authorization"); }; assert!(!authorization.allowed); assert!(authorization.reason.contains("explicit project permission")); assert!(!audit_event.allowed); assert_eq!(audit_event.charged_debug_read_bytes, 0); assert_eq!(charged_debug_read_bytes, 0); assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES); let CoordinatorResponse::DebugAttach { authorization, audit_event, charged_debug_read_bytes, used_debug_read_bytes, .. } = service .handle_request(CoordinatorRequest::DebugAttach { tenant: "other-tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected cross-tenant debug attach denial"); }; assert!(!authorization.allowed); assert!(authorization.reason.contains("tenant or project")); assert!(!audit_event.allowed); assert_eq!(charged_debug_read_bytes, 0); assert_eq!(used_debug_read_bytes, 0); assert_eq!(service.debug_audit_events.len(), 3); } #[test] fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant".to_owned(), actor_user: "user".to_owned(), project: "project".to_owned(), name: "Demo".to_owned(), }) .unwrap(); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), public_key: test_node_public_key("worker"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![], dependency_cache_digests: vec![], source_snapshots: vec![], artifact_locations: vec![], direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "process", "compile-linux", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/out.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap(); service.main_runtime.controls.insert( process_control_key( &TenantId::from("tenant"), &ProjectId::from("project"), &ProcessId::from("process"), ), super::main_runtime::CoordinatorMainControl { task_definition: clusterflux_core::TaskDefinitionId::from("completed-main"), task_instance: TaskInstanceId::from("ti:process:main"), abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), state: "completed".to_owned(), stopped_probe_symbol: None, handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), launch_id: 1, }, ); let CoordinatorResponse::DebugBreakpoints { revision, probe_symbols, hit_epoch, .. } = service .handle_request(CoordinatorRequest::SetDebugBreakpoints { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), revision: 1, probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()], }) .unwrap() else { panic!("expected debug breakpoints response"); }; assert_eq!(revision, 1); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); assert_eq!(hit_epoch, None); let CoordinatorResponse::DebugBreakpoints { revision, probe_symbols, .. } = service .handle_request(CoordinatorRequest::SetDebugBreakpoints { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), revision: 0, probe_symbols: vec!["clusterflux.probe.stale".to_owned()], }) .unwrap() else { panic!("expected stale breakpoint response"); }; assert_eq!(revision, 1); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); let CoordinatorResponse::DebugProbeHit { breakpoint_matched, debug_epoch, probe_symbol, .. } = service .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugProbeHit { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), probe_symbol: "clusterflux.probe.compile_linux".to_owned(), }) .unwrap() else { panic!("expected executing Wasm probe hit response"); }; assert!(breakpoint_matched); assert_eq!(debug_epoch, Some(1)); assert_eq!(probe_symbol, "clusterflux.probe.compile_linux"); let CoordinatorResponse::DebugBreakpoints { hit_epoch, hit_task, hit_probe_symbol, .. } = service .handle_request(CoordinatorRequest::InspectDebugBreakpoints { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected breakpoint hit status"); }; assert_eq!(hit_epoch, Some(1)); assert_eq!(hit_task, Some(TaskInstanceId::from("compile-linux"))); assert_eq!( hit_probe_symbol.as_deref(), Some("clusterflux.probe.compile_linux") ); let CoordinatorResponse::DebugCommand { epoch: Some(1), command: Some(command), .. } = service .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), }) .unwrap() else { panic!("expected signed node poll to receive freeze command"); }; assert_eq!(command, "freeze"); let CoordinatorResponse::DebugCommand { epoch: None, command: None, .. } = service .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), }) .unwrap() else { panic!("debug commands should be consumed after poll"); }; let CoordinatorResponse::DebugEpochStatus { fully_frozen, fully_resumed, acknowledgements, .. } = service .handle_request(CoordinatorRequest::InspectDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap() else { panic!("expected pending debug epoch status"); }; assert!(!fully_frozen); assert!(!fully_resumed); assert!(acknowledgements.is_empty()); let early_resume = service .handle_request(CoordinatorRequest::ResumeDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap_err(); assert!(early_resume .to_string() .contains("no settled frozen participant set")); service .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), epoch: 1, state: DebugAcknowledgementState::Frozen, stack_frames: vec!["compile_linux::wasm".to_owned()], local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())], task_args: vec![("target".to_owned(), "linux".to_owned())], handles: vec![("artifact".to_owned(), "pending".to_owned())], command_status: Some("frozen at Wasm safepoint".to_owned()), recent_output: vec![], message: None, }) .unwrap(); let CoordinatorResponse::DebugEpochStatus { fully_frozen, fully_resumed, acknowledgements, .. } = service .handle_request(CoordinatorRequest::InspectDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap() else { panic!("expected frozen debug epoch status"); }; assert!(fully_frozen); assert!(!fully_resumed); assert_eq!(acknowledgements.len(), 1); assert_eq!(acknowledgements[0].state, DebugAcknowledgementState::Frozen); let CoordinatorResponse::DebugEpoch { epoch, command, affected_tasks, all_stop_requested, audit_event, .. } = service .handle_request(CoordinatorRequest::ResumeDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap() else { panic!("expected debug epoch resume response"); }; assert_eq!(epoch, 1); assert_eq!(command, "resume"); assert!(!all_stop_requested); assert_eq!(affected_tasks.len(), 1); assert_eq!(audit_event.operation, "resume_debug_epoch"); let CoordinatorResponse::DebugCommand { epoch: Some(1), command: Some(command), .. } = service .handle_signed_node_request_auto(CoordinatorRequest::PollDebugCommand { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), }) .unwrap() else { panic!("expected signed node poll to receive resume command"); }; assert_eq!(command, "resume"); let CoordinatorResponse::DebugEpochStatus { fully_resumed, .. } = service .handle_request(CoordinatorRequest::InspectDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap() else { panic!("expected pending resume status"); }; assert!(!fully_resumed); service .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker".to_owned(), task: "compile-linux".to_owned(), epoch: 1, state: DebugAcknowledgementState::Running, stack_frames: vec![], local_values: vec![], task_args: vec![], handles: vec![], command_status: Some("running".to_owned()), recent_output: vec![], message: None, }) .unwrap(); let CoordinatorResponse::DebugEpochStatus { fully_frozen, fully_resumed, .. } = service .handle_request(CoordinatorRequest::InspectDebugEpoch { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), epoch: 1, }) .unwrap() else { panic!("expected resumed debug epoch status"); }; assert!(!fully_frozen); assert!(fully_resumed); } #[test] fn service_reports_task_restart_boundary_through_public_api() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant".to_owned(), actor_user: "user".to_owned(), project: "project".to_owned(), name: "Demo".to_owned(), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); let denied = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "other-user".to_owned(), process: "process".to_owned(), task: "task".to_owned(), replacement_bundle: None, }) .unwrap_err(); assert!(denied.to_string().contains("task restart denied")); let CoordinatorResponse::TaskRestart { accepted, clean_boundary_available, active_task, completed_event_observed, requires_whole_process_restart, restarted_task_instance, message, audit_event, charged_debug_read_bytes, used_debug_read_bytes, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), task: "task".to_owned(), replacement_bundle: None, }) .unwrap() else { panic!("expected task restart response"); }; assert!(!accepted); assert!(!clean_boundary_available); assert!(!active_task); assert!(!completed_event_observed); assert!(restarted_task_instance.is_none()); assert!(requires_whole_process_restart); assert!(message.contains("not known")); assert_eq!(audit_event.operation, "restart_task"); assert_eq!(audit_event.task, Some(TaskInstanceId::from("task"))); assert!(audit_event.allowed); assert_eq!(charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES); assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), public_key: test_node_public_key("worker-linux"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "process", "task", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap(); let CoordinatorResponse::TaskAssignment { assignment: Some(initial_assignment), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), }) .unwrap() else { panic!("expected initial task assignment"); }; assert_eq!(initial_assignment.task, TaskInstanceId::from("task")); let CoordinatorResponse::TaskRestart { accepted, clean_boundary_available, active_task, completed_event_observed, requires_whole_process_restart, message, audit_event, used_debug_read_bytes, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), task: "task".to_owned(), replacement_bundle: None, }) .unwrap() else { panic!("expected active task restart response"); }; assert!(!accepted); assert!(!clean_boundary_available); assert!(active_task); assert!(!completed_event_observed); assert!(requires_whole_process_restart); assert!(message.contains("still active")); assert_eq!( audit_event.charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES ); assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 2); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "worker-linux".to_owned(), task: "task".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap(); let CoordinatorResponse::TaskRestart { accepted, clean_boundary_available, active_task, completed_event_observed, requires_whole_process_restart, restarted_task_instance, restarted_attempt_id, message, audit_event, used_debug_read_bytes, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), task: "task".to_owned(), replacement_bundle: None, }) .unwrap() else { panic!("expected completed task restart response"); }; assert!(accepted); assert!(clean_boundary_available); assert!(!active_task); assert!(completed_event_observed); assert!(!requires_whole_process_restart); let restarted_task_instance = restarted_task_instance.expect("restart returns logical id"); let restarted_attempt_id = restarted_attempt_id.expect("restart returns a new attempt id"); assert!(restarted_attempt_id.starts_with("ta_")); assert!(message.contains("from clean VFS entry boundary epoch 7")); let CoordinatorResponse::TaskAssignment { assignment: Some(restarted_assignment), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), }) .unwrap() else { panic!("expected restarted task assignment"); }; assert_eq!(restarted_assignment.task, restarted_task_instance); assert_eq!(restarted_assignment.task, TaskInstanceId::from("task")); let mut expected_task_spec = initial_assignment.task_spec.clone(); expected_task_spec.task_instance = restarted_task_instance; assert_eq!(restarted_assignment.task_spec, expected_task_spec); assert_eq!( restarted_assignment.wasm_module_base64, initial_assignment.wasm_module_base64 ); assert_eq!( audit_event.charged_debug_read_bytes, DEBUG_CONTROL_READ_BYTES ); assert_eq!(used_debug_read_bytes, DEBUG_CONTROL_READ_BYTES * 3); assert_eq!(service.debug_audit_events.len(), 4); } #[test] fn service_cancels_whole_process_and_blocks_new_task_launches() { let mut service = CoordinatorService::new(7); for node in ["node-a", "node-b"] { let capabilities = if node == "node-a" { linux_capabilities() } else { windows_capabilities() }; service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), public_key: test_node_public_key(node), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), capabilities, cached_environment_digests: vec![], dependency_cache_digests: vec![], source_snapshots: vec![], artifact_locations: vec![], direct_connectivity: true, online: true, }) .unwrap(); } service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); for node in ["node-a", "node-b"] { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); } for (task, required_capability, preferred_node) in [ ("compile-linux", Capability::Containers, "node-a"), ("link-windows", Capability::WindowsCommandDev, "node-b"), ] { let response = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "process", task, 7, [required_capability], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: format!("/vfs/artifacts/{task}.txt"), wasm_module_base64: test_wasm_module_base64(), }) .unwrap(); let CoordinatorResponse::TaskLaunched { assignment, .. } = response else { panic!("expected task launch"); }; assert_eq!(assignment.node, NodeId::from(preferred_node)); } let CoordinatorResponse::ProcessCancellationRequested { process, cancelled_tasks, affected_nodes, } = service .handle_request(CoordinatorRequest::CancelProcess { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected process cancellation"); }; assert_eq!(process, ProcessId::from("process")); assert_eq!(cancelled_tasks.len(), 2); assert_eq!( affected_nodes, vec![NodeId::from("node-a"), NodeId::from("node-b")] ); let control = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node-a".to_owned(), task: "compile-linux".to_owned(), }) .unwrap(); assert_eq!( control, CoordinatorResponse::TaskControl { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), cancel_requested: true, abort_requested: false, } ); let blocked = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "process", "package-linux", 7, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/package-linux.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap_err(); assert!(blocked .to_string() .contains("virtual process is cancelling")); service .handle_request(CoordinatorRequest::AbortProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap(); let abort_control = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node-a".to_owned(), task: "compile-linux".to_owned(), }) .unwrap(); assert_eq!( abort_control, CoordinatorResponse::TaskControl { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-linux"), cancel_requested: false, abort_requested: true, } ); } #[test] fn service_rejects_second_active_process_unless_restarting_same_process() { let mut service = CoordinatorService::new(7); let started = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-a".to_owned(), restart: false, }) .unwrap(); assert!(matches!( started, CoordinatorResponse::ProcessStarted { launch_attempt: None, .. } )); let same_without_restart = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-a".to_owned(), restart: false, }) .unwrap_err(); assert!(same_without_restart .to_string() .contains("already has active virtual process")); let other_process = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-b".to_owned(), restart: true, }) .unwrap_err(); assert!(other_process .to_string() .contains("already has active virtual process")); let wrong_attempt_abort = service .handle_request(CoordinatorRequest::AbortProcess { launch_attempt: Some("attempt-b".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process-a".to_owned(), }) .unwrap_err(); assert!(wrong_attempt_abort .to_string() .contains("does not own process process-a")); assert!(service .coordinator .active_process( &TenantId::from("tenant"), &ProjectId::from("project"), &ProcessId::from("process-a") ) .is_some()); let retired_main_abort = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let process_key = process_control_key( &TenantId::from("tenant"), &ProjectId::from("project"), &ProcessId::from("process-a"), ); service.main_runtime.controls.insert( process_key.clone(), super::main_runtime::CoordinatorMainControl { task_definition: clusterflux_core::TaskDefinitionId::from("build"), task_instance: TaskInstanceId::from("ti:process-a:main"), abort: std::sync::Arc::clone(&retired_main_abort), debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), state: "running".to_owned(), stopped_probe_symbol: None, handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), launch_id: 1, }, ); let restarted = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: Some("attempt-a-restart".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-a".to_owned(), restart: true, }) .unwrap(); assert_eq!( restarted, CoordinatorResponse::ProcessStarted { launch_attempt: Some("attempt-a-restart".to_owned()), process: ProcessId::from("process-a"), epoch: 7, actor: WorkflowActor { kind: "user".to_owned(), user: Some(UserId::from("user")), agent: None, credential_kind: CredentialKind::BrowserSession, public_key_fingerprint: None, authenticated_without_browser: false, scopes: vec!["project:read".to_owned(), "project:run".to_owned()], }, charged_spawns: 2, } ); assert!(retired_main_abort.load(std::sync::atomic::Ordering::Acquire)); assert!(!service.main_runtime.controls.contains_key(&process_key)); } #[test] fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() { let mut service = CoordinatorService::new(31); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("process-main-before-child"); let child = TaskInstanceId::from("child-active"); let process_key = process_control_key(&tenant, &project, &process); service .handle_request(CoordinatorRequest::AttachNode { tenant: tenant.to_string(), project: project.to_string(), node: "worker".to_owned(), public_key: test_node_public_key("worker"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: process.to_string(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: tenant.to_string(), project: project.to_string(), node: "worker".to_owned(), process: process.to_string(), epoch: 31, }) .unwrap(); register_test_task_assignment( &mut service, tenant.as_str(), project.as_str(), process.as_str(), "worker", "child-definition", child.as_str(), 31, ); let main = TaskInstanceId::from("main-instance"); service.main_runtime.controls.insert( process_key.clone(), super::main_runtime::CoordinatorMainControl { task_definition: TaskDefinitionId::from("build"), task_instance: main.clone(), abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), state: "running".to_owned(), stopped_probe_symbol: None, handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), launch_id: 1, }, ); service.debug_epochs.insert(process_key.clone(), 9); service.record_coordinator_main_completion( super::main_runtime::MainScope { tenant: tenant.clone(), project: project.clone(), process: process.clone(), task_definition: TaskDefinitionId::from("build"), task_instance: main, epoch: 31, launch_id: 1, }, Ok(WasmTaskResult::completed( TaskInstanceId::from("main-instance"), TaskBoundaryValue::SmallJson(json!("main completed")), )), ); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_some()); assert!(service.debug_epochs.contains_key(&process_key)); assert!(service .active_tasks .iter() .any(|(_, _, retained_process, _, task)| { retained_process == &process && task == &child })); let blocked_next = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-too-early".to_owned(), restart: false, }) .unwrap_err(); assert!(blocked_next .to_string() .contains("already has active virtual process")); let artifact_bytes = b"child artifact survives terminal cleanup"; service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: tenant.to_string(), project: project.to_string(), process: process.to_string(), node: "worker".to_owned(), task: child.to_string(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: artifact_bytes.len() as u64, stderr_bytes: 0, stdout_tail: "child completed".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/child-output".to_owned()), artifact_digest: Some(Digest::sha256(artifact_bytes)), artifact_size_bytes: Some(artifact_bytes.len() as u64), result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))), }) .unwrap(); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_none()); assert!(!service.debug_epochs.contains_key(&process_key)); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: tenant.to_string(), project: project.to_string(), actor_user: "user".to_owned(), process: Some(process.to_string()), }) .unwrap() else { panic!("expected retained task events"); }; assert_eq!(events.len(), 2); assert!(events.iter().any(|event| { event.executor == TaskExecutor::CoordinatorMain && event.terminal_state == TaskTerminalState::Completed })); assert!(events.iter().any(|event| { event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes)) })); let metadata = service .artifact_registry .metadata(&tenant, &project, &ArtifactId::from("child-output")) .expect("artifact metadata must survive terminal cleanup"); assert_eq!(metadata.process, process); assert_eq!(metadata.digest, Digest::sha256(artifact_bytes)); let next = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-after-cleanup".to_owned(), restart: false, }) .unwrap(); assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. })); } #[test] fn completed_main_terminal_matrix_retires_after_failed_or_cancelled_final_child() { for terminal_state in [TaskTerminalState::Failed, TaskTerminalState::Cancelled] { let mut service = service_with_completed_main_and_final_child( clusterflux_core::TaskFailurePolicy::FailFast, ); complete_terminal_matrix_child(&mut service, terminal_state.clone()); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); assert!( service .coordinator .active_process(&tenant, &project, &process) .is_none(), "{terminal_state:?} final child left the process slot active" ); assert!(!service .debug_epochs .contains_key(&process_control_key(&tenant, &project, &process))); assert!(!service .debug_breakpoints .contains_key(&process_control_key(&tenant, &project, &process))); assert!(service.debug_commands.keys().all( |(task_tenant, task_project, task_process, _, _)| { task_tenant != &tenant || task_project != &project || task_process != &process } )); assert!(!service .panel_snapshots .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); assert!(service.active_tasks.iter().all( |(task_tenant, task_project, task_process, _, _)| { task_tenant != &tenant || task_project != &project || task_process != &process } )); assert!(!service .main_runtime .controls .contains_key(&process_control_key(&tenant, &project, &process))); let join = service.task_join_result( tenant.clone(), project.clone(), process.clone(), TaskInstanceId::from("final-child"), ); assert_eq!( join.state, match terminal_state { TaskTerminalState::Failed => TaskJoinState::Failed, TaskTerminalState::Cancelled => TaskJoinState::Cancelled, TaskTerminalState::Completed => unreachable!(), } ); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "next-after-terminal".to_owned(), restart: false, }) .expect("the terminal outcome must release the one-process project slot"); } } #[test] fn completed_main_unpolled_final_assignment_completion_retires_process() { let mut service = service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); let node = NodeId::from("worker"); let assignment_key = (tenant.clone(), project.clone(), node); assert_eq!( service .task_assignments .get(&assignment_key) .map_or(0, VecDeque::len), 1 ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: tenant.to_string(), project: project.to_string(), process: process.to_string(), node: "worker".to_owned(), task: "final-child".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 2, stderr_bytes: 0, stdout_tail: "42".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(json!(42))), }) .unwrap(); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_none()); assert!(!service.task_assignments.contains_key(&assignment_key)); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "next-after-unpolled-completion".to_owned(), restart: false, }) .expect("unpolled terminal completion must release the one-process slot"); } #[test] fn completed_main_retires_from_authoritative_state_after_event_history_rotates() { let mut service = service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); for index in 0..=MAX_TASK_EVENTS_PER_PROCESS { service.record_task_completion_event(TaskCompletionEvent { tenant: tenant.clone(), project: project.clone(), process: process.clone(), node: NodeId::from("worker"), executor: TaskExecutor::Node, task_definition: TaskDefinitionId::from("historical"), task: TaskInstanceId::new(format!("historical-{index}")), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }); } assert!( service.task_events.iter().all(|event| { event.process != process || event.executor != TaskExecutor::CoordinatorMain }), "the regression requires bounded history to have rotated the main event" ); complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_none()); let summary = service .process_summaries .get(&process_control_key(&tenant, &project, &process)) .expect("the terminal process summary must remain authoritative"); assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed)); assert_eq!( summary.main_terminal_state, Some(TaskTerminalState::Completed) ); } #[test] fn completed_main_await_operator_blocks_retirement_until_each_resolution() { for resolution in [ TaskFailureResolution::AcceptFailure, TaskFailureResolution::Cancel, ] { let mut service = service_with_completed_main_and_final_child( clusterflux_core::TaskFailurePolicy::AwaitOperator, ); complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); let process_key = process_control_key(&tenant, &project, &process); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_some()); assert!(service.debug_epochs.contains_key(&process_key)); assert!(service.debug_breakpoints.contains_key(&process_key)); assert!(service .panel_snapshots .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); let attempt = service .task_attempts .get(&super::keys::task_restart_key( &tenant, &project, &process, &TaskInstanceId::from("final-child"), )) .and_then(|attempts| attempts.iter().rev().find(|attempt| attempt.current)) .unwrap(); assert_eq!(attempt.state, TaskAttemptState::FailedAwaitingAction); service .handle_request(CoordinatorRequest::ResolveTaskFailure { tenant: tenant.to_string(), project: project.to_string(), actor_user: "user".to_owned(), process: process.to_string(), task: "final-child".to_owned(), resolution, }) .unwrap(); assert!( service .coordinator .active_process(&tenant, &project, &process) .is_none(), "{resolution:?} left the process slot active" ); assert!(!service.debug_epochs.contains_key(&process_key)); assert!(!service.debug_breakpoints.contains_key(&process_key)); assert!(!service .panel_snapshots .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); assert!(service.debug_commands.keys().all( |(task_tenant, task_project, task_process, _, _)| { task_tenant != &tenant || task_project != &project || task_process != &process } )); let join = service.task_join_result( tenant.clone(), project.clone(), process.clone(), TaskInstanceId::from("final-child"), ); assert_eq!( join.state, match resolution { TaskFailureResolution::AcceptFailure => TaskJoinState::Failed, TaskFailureResolution::Cancel => TaskJoinState::Cancelled, } ); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: tenant.to_string(), project: project.to_string(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "next-after-resolution".to_owned(), restart: false, }) .expect("operator resolution must release the one-process project slot"); } } #[test] fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() { let mut service = service_with_completed_main_and_final_child( clusterflux_core::TaskFailurePolicy::AwaitOperator, ); complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); let task = TaskInstanceId::from("final-child"); let CoordinatorResponse::TaskRestart { accepted, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: tenant.to_string(), project: project.to_string(), actor_user: "user".to_owned(), process: process.to_string(), task: task.to_string(), replacement_bundle: None, }) .unwrap() else { panic!("expected task restart"); }; assert!(accepted); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: tenant.to_string(), project: project.to_string(), process: process.to_string(), node: "worker".to_owned(), task: task.to_string(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 2, stderr_bytes: 0, stdout_tail: "ok".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(json!("ok"))), }) .unwrap(); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_none()); let attempts = service .task_attempts .get(&super::keys::task_restart_key( &tenant, &project, &process, &task, )) .unwrap(); assert!( attempts .iter() .any(|attempt| !attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction) ); assert!(attempts .iter() .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed)); assert_eq!( service .process_summaries .get(&process_control_key(&tenant, &project, &process)) .and_then(|summary| summary.final_result.clone()), Some(ProcessFinalResult::Completed), "a successful current retry must override the superseded failed attempt" ); assert_eq!( service .task_join_result(tenant, project, process, task) .state, TaskJoinState::Completed ); } #[test] fn completed_main_failed_child_does_not_abort_another_active_child() { let mut service = service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); register_test_task_assignment( &mut service, "tenant", "project", "terminal-matrix", "worker", "other-child-definition", "other-child", 83, ); complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); let tenant = TenantId::from("tenant"); let project = ProjectId::from("project"); let process = ProcessId::from("terminal-matrix"); let other_key = task_control_key( &tenant, &project, &process, &NodeId::from("worker"), &TaskInstanceId::from("other-child"), ); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_some()); assert!(service.active_tasks.contains(&other_key)); assert!(!service.task_aborts.contains(&other_key)); assert_eq!( service .task_join_result( tenant.clone(), project.clone(), process.clone(), TaskInstanceId::from("final-child"), ) .state, TaskJoinState::Failed ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: tenant.to_string(), project: project.to_string(), process: process.to_string(), node: "worker".to_owned(), task: "other-child".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap(); assert!(service .coordinator .active_process(&tenant, &project, &process) .is_none()); assert!(!service.active_tasks.contains(&other_key)); } #[test] fn quiescent_cooperative_cancel_releases_slot_immediately() { let mut service = CoordinatorService::new(17); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-a".to_owned(), restart: false, }) .unwrap(); service.record_task_completion_event(TaskCompletionEvent { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("process-a"), node: NodeId::from("node"), executor: TaskExecutor::Node, task_definition: clusterflux_core::TaskDefinitionId::from("old-task"), task: TaskInstanceId::from("ti:process-a:old"), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(json!("old"))), }); service .record_debug_audit_event( TenantId::from("tenant"), ProjectId::from("project"), ProcessId::from("process-a"), None, UserId::from("user"), "old_debug_event", false, "old process incarnation", ) .unwrap(); service .handle_request(CoordinatorRequest::CancelProcess { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process-a".to_owned(), }) .unwrap(); let CoordinatorResponse::ProcessStatuses { processes, .. } = service .handle_request(CoordinatorRequest::ListProcesses { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), }) .unwrap() else { panic!("expected live process statuses"); }; assert!(processes.is_empty()); let started = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process-a".to_owned(), restart: false, }) .unwrap(); assert!(matches!( started, CoordinatorResponse::ProcessStarted { launch_attempt: None, .. } )); assert!(service.task_events.iter().all(|event| { event.tenant != TenantId::from("tenant") || event.project != ProjectId::from("project") || event.process != ProcessId::from("process-a") })); assert!(service.debug_audit_events.iter().all(|event| { event.tenant != TenantId::from("tenant") || event.project != ProjectId::from("project") || event.process != ProcessId::from("process-a") })); } #[test] fn aborted_process_accepts_signed_terminal_event_for_issued_task() { let mut service = CoordinatorService::new(17); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 17, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile", "compile-1", 17, ); let CoordinatorResponse::ProcessAborted { aborted_tasks, .. } = service .handle_request(CoordinatorRequest::AbortProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), }) .unwrap() else { panic!("expected process abort"); }; assert_eq!(aborted_tasks.len(), 1); let recorded = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-1".to_owned(), terminal_state: Some(TaskTerminalState::Cancelled), status_code: None, stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }) .unwrap(); assert_eq!( recorded, CoordinatorResponse::TaskRecorded { process: ProcessId::from("process"), task: TaskInstanceId::from("compile-1"), events_recorded: 1, } ); assert_eq!( service.task_events[0].task_definition, clusterflux_core::TaskDefinitionId::from("compile") ); assert!(service.task_restart_checkpoints.is_empty()); } #[test] fn service_download_links_are_scoped_and_streaming_is_metered() { let mut service = CoordinatorService::new(7); let artifact_bytes = b"0123456789abcdef0123456789abcdef"; service.set_server_time(10); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: false, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile-linux", "compile-linux", 7, ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 32, stderr_bytes: 0, stdout_tail: "artifact bytes".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/app.txt".to_owned()), artifact_digest: Some(Digest::sha256(artifact_bytes)), artifact_size_bytes: Some(artifact_bytes.len() as u64), result: None, }) .unwrap(); service.quota.set_download_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 31)]), }); let quota_denied_link = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 60, }) .unwrap_err(); assert!(quota_denied_link .to_string() .contains("ArtifactDownloadBytes")); service.quota.set_download_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 64)]), }); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: vec!["app.txt".to_owned()], direct_connectivity: false, online: true, }) .unwrap(); let private_node_link = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 60, }) .unwrap(); assert!(matches!( private_node_link, CoordinatorResponse::ArtifactDownloadLink { .. } )); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: vec!["app.txt".to_owned()], direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::ArtifactDownloadLink { link } = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 60, }) .unwrap() else { panic!("expected download link"); }; service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: vec!["app.txt".to_owned()], direct_connectivity: false, online: true, }) .unwrap(); let private_node_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 1, }) .unwrap(); assert!(matches!( private_node_open, CoordinatorResponse::ArtifactDownloadStream { content_bytes_available: false, .. } )); let CoordinatorResponse::ArtifactTransferAssignment { transfer: Some(pending_transfer), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollArtifactTransfer { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), }) .unwrap() else { panic!("expected reverse transfer assignment"); }; let bad_chunk = service .handle_signed_node_request_auto(CoordinatorRequest::UploadArtifactTransferChunk { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), transfer_id: pending_transfer.transfer_id, artifact: pending_transfer.artifact.as_str().to_owned(), offset: 0, content_base64: BASE64_STANDARD.encode(artifact_bytes), chunk_digest: Digest::sha256("wrong bytes"), eof: true, }) .unwrap_err(); assert!(bad_chunk.to_string().contains("chunk digest mismatch")); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: vec!["app.txt".to_owned()], direct_connectivity: true, online: true, }) .unwrap(); assert_eq!(link.tenant, TenantId::from("tenant")); assert_eq!(link.project, ProjectId::from("project")); assert_eq!(link.process, ProcessId::from("process")); assert_eq!(link.actor, Actor::User(UserId::from("user"))); assert_eq!(link.max_bytes, 64); assert!(link.policy_context_digest.is_valid_sha256()); assert_eq!(link.expires_at_epoch_seconds, 70); assert!(link.url_path.contains("/artifacts/tenant/project/process")); let cross_tenant = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "other".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 60, }) .unwrap_err(); assert!(cross_tenant.to_string().contains("does not exist")); let cross_project = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "other-project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 60, }) .unwrap_err(); assert!(cross_project.to_string().contains("does not exist")); let cross_tenant_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "other".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 1, }) .unwrap_err(); assert!(cross_tenant_open.to_string().contains("does not exist")); let cross_project_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "other-project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 1, }) .unwrap_err(); assert!(cross_project_open.to_string().contains("does not exist")); let guessed = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: Digest::sha256("guessed"), chunk_bytes: 1, }) .unwrap_err(); assert!(guessed.to_string().contains("token is invalid")); let cross_actor_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "other-user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 1, }) .unwrap_err(); assert!(cross_actor_open.to_string().contains("token is invalid")); service.set_server_time(11); upload_pending_artifact_transfer(&mut service, "node", artifact_bytes); let CoordinatorResponse::ArtifactDownloadStream { streamed_bytes, charged_download_bytes, content_base64: Some(content_base64), content_source: Some(content_source), .. } = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 16, }) .unwrap() else { panic!("expected download stream"); }; assert_eq!(streamed_bytes, 16); assert_eq!(charged_download_bytes, 16); assert_eq!( BASE64_STANDARD.decode(content_base64).unwrap(), &artifact_bytes[..16] ); assert_eq!(content_source, "retaining_node_reverse_stream"); let CoordinatorResponse::ArtifactDownloadLink { link: expiring_link, } = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, ttl_seconds: 1, }) .unwrap() else { panic!("expected expiring download link"); }; service.set_server_time(13); let expired = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: expiring_link.scoped_token_digest, chunk_bytes: 1, }) .unwrap_err(); assert!(expired.to_string().contains("expired")); service.set_server_time(11); let cross_actor_revoke = service .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "other-user".to_owned(), artifact: "app.txt".to_owned(), token_digest: link.scoped_token_digest.clone(), }) .unwrap_err(); assert!(cross_actor_revoke.to_string().contains("token is invalid")); let CoordinatorResponse::ArtifactDownloadLinkRevoked { link: revoked } = service .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), token_digest: link.scoped_token_digest.clone(), }) .unwrap() else { panic!("expected revoked download link"); }; assert_eq!(revoked.scoped_token_digest, link.scoped_token_digest); let revoked = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 64, token_digest: link.scoped_token_digest, chunk_bytes: 1, }) .unwrap_err(); assert!(revoked.to_string().contains("revoked")); } #[test] fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { let mut service = CoordinatorService::new(7); let artifact_bytes = b"0123456789abcdef"; service.quota.set_download_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 16)]), }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "compile-linux", "compile-linux", 7, ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 16, stderr_bytes: 0, stdout_tail: "artifact bytes".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/app.txt".to_owned()), artifact_digest: Some(Digest::sha256(artifact_bytes)), artifact_size_bytes: Some(artifact_bytes.len() as u64), result: None, }) .unwrap(); let CoordinatorResponse::ArtifactDownloadLink { link } = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, ttl_seconds: 60, }) .unwrap() else { panic!("expected initial download link"); }; let pending = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 16, }) .unwrap(); assert!(matches!( pending, CoordinatorResponse::ArtifactDownloadStream { content_bytes_available: false, .. } )); upload_pending_artifact_transfer(&mut service, "node", artifact_bytes); let CoordinatorResponse::ArtifactDownloadStream { charged_download_bytes, .. } = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 16, }) .unwrap() else { panic!("expected initial download stream"); }; assert_eq!(charged_download_bytes, 16); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, }) .unwrap(); let after_reconnect = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, ttl_seconds: 60, }) .unwrap_err(); assert!(after_reconnect .to_string() .contains("ArtifactDownloadBytes")); let restarted_cli_retry = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, ttl_seconds: 60, }) .unwrap_err(); assert!(restarted_cli_retry .to_string() .contains("ArtifactDownloadBytes")); let cross_tenant_retry = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "other-tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, token_digest: link.scoped_token_digest.clone(), chunk_bytes: 1, }) .unwrap_err(); assert!(cross_tenant_retry.to_string().contains("does not exist")); let cross_project_retry = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "other-project".to_owned(), actor_user: "user".to_owned(), artifact: "app.txt".to_owned(), max_bytes: 16, token_digest: link.scoped_token_digest, chunk_bytes: 1, }) .unwrap_err(); assert!(cross_project_retry.to_string().contains("does not exist")); } #[test] fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() { let mut service = CoordinatorService::new(7); let artifact_bytes = vec![0x5a; 600_000]; service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: false, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "node", "package", "package", 7, ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "package".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/large.bin".to_owned()), artifact_digest: Some(Digest::sha256(&artifact_bytes)), artifact_size_bytes: Some(artifact_bytes.len() as u64), result: Some(TaskBoundaryValue::Artifact(ArtifactHandle { id: ArtifactId::from("large.bin"), digest: Digest::sha256(&artifact_bytes), size_bytes: artifact_bytes.len() as u64, })), }) .unwrap(); let CoordinatorResponse::ArtifactDownloadLink { link } = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "large.bin".to_owned(), max_bytes: artifact_bytes.len() as u64, ttl_seconds: 60, }) .unwrap() else { panic!("expected download link"); }; let pending = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "large.bin".to_owned(), max_bytes: artifact_bytes.len() as u64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: artifact_bytes.len() as u64, }) .unwrap(); assert!(matches!( pending, CoordinatorResponse::ArtifactDownloadStream { content_bytes_available: false, .. } )); upload_pending_artifact_transfer(&mut service, "node", &artifact_bytes); let mut downloaded = Vec::new(); loop { let CoordinatorResponse::ArtifactDownloadStream { streamed_bytes, content_offset: Some(offset), content_eof, content_base64: Some(content), .. } = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), artifact: "large.bin".to_owned(), max_bytes: artifact_bytes.len() as u64, token_digest: link.scoped_token_digest.clone(), chunk_bytes: artifact_bytes.len() as u64, }) .unwrap() else { panic!("expected verified reverse-stream content chunk"); }; assert_eq!(offset, downloaded.len() as u64); assert!(streamed_bytes <= super::artifacts::MAX_ARTIFACT_REVERSE_CHUNK_BYTES); downloaded.extend(BASE64_STANDARD.decode(content).unwrap()); if content_eof { break; } } assert_eq!(downloaded, artifact_bytes); assert_eq!( service.quota.used_download_bytes( &TenantId::from("tenant"), &ProjectId::from("project"), service.current_epoch_seconds().unwrap(), ), 600_000 ); } #[test] fn windows_task_events_share_the_virtual_process_scope() { let mut service = CoordinatorService::new(7); service.record_task_completion_event(TaskCompletionEvent { tenant: TenantId::from("other-tenant"), project: ProjectId::from("other-project"), process: ProcessId::from("other-process"), node: NodeId::from("other-node"), executor: super::TaskExecutor::Node, task_definition: TaskDefinitionId::from("other-task"), task: TaskInstanceId::from("other-task"), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "windows-node".to_owned(), public_key: test_node_public_key("windows-node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "windows-node".to_owned(), capabilities: windows_capabilities(), cached_environment_digests: vec![Digest::sha256("env-windows-command-dev")], dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); register_test_task_assignment( &mut service, "tenant", "project", "process", "windows-node", "windows-command-dev", "windows-command-dev", 7, ); let recorded = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "windows-node".to_owned(), task: "windows-command-dev".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 24, stderr_bytes: 0, stdout_tail: "windows output".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/windows-output.txt".to_owned()), artifact_digest: Some(Digest::sha256("windows-artifact")), artifact_size_bytes: Some(24), result: None, }) .unwrap(); assert_eq!( recorded, CoordinatorResponse::TaskRecorded { process: ProcessId::from("process"), task: TaskInstanceId::from("windows-command-dev"), events_recorded: 1, } ); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap() else { panic!("expected task events"); }; assert_eq!(events.len(), 1); assert_eq!(events[0].process, ProcessId::from("process")); assert_eq!(events[0].node, NodeId::from("windows-node")); assert_eq!(events[0].task, TaskInstanceId::from("windows-command-dev")); assert_eq!( events[0].artifact_path, Some(VfsPath::new("/vfs/artifacts/windows-output.txt").unwrap()) ); } #[test] fn service_schedules_task_across_reported_node_descriptors() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::AttachNode { tenant: "other-tenant".to_owned(), project: "other-project".to_owned(), node: "other-node".to_owned(), public_key: test_node_public_key("other-node"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "other-tenant".to_owned(), project: "other-project".to_owned(), node: "other-node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: false, online: true, }) .unwrap(); for node in ["cold-node", "warm-node"] { service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), public_key: test_node_public_key(node), }) .unwrap(); } let recorded = service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "cold-node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: false, online: true, }) .unwrap(); assert_eq!( recorded, CoordinatorResponse::NodeCapabilitiesRecorded { node: NodeId::from("cold-node"), node_descriptors: 1, } ); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "warm-node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![Digest::sha256("env")], dependency_cache_digests: vec![Digest::sha256("deps")], source_snapshots: vec![Digest::sha256("source")], artifact_locations: vec!["build-output".to_owned()], direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::NodeDescriptors { descriptors, actor } = service .handle_request(CoordinatorRequest::ListNodeDescriptors { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "operator".to_owned(), }) .unwrap() else { panic!("expected node descriptor inspector state"); }; assert_eq!(actor, UserId::from("operator")); assert_eq!(descriptors.len(), 2); let warm = descriptors .iter() .find(|descriptor| descriptor.id == NodeId::from("warm-node")) .expect("warm node descriptor is visible to inspector state"); assert!(warm .capabilities .capabilities .contains(&Capability::Command)); assert!(warm.cached_environments.contains(&Digest::sha256("env"))); assert!(warm.dependency_caches.contains(&Digest::sha256("deps"))); assert!(warm.source_snapshots.contains(&Digest::sha256("source"))); assert!(warm .artifact_locations .contains(&ArtifactId::from("build-output"))); let CoordinatorResponse::TaskPlacement { placement } = service .handle_request(CoordinatorRequest::ScheduleTask { tenant: "tenant".to_owned(), project: "project".to_owned(), environment: Some(EnvironmentRequirements::linux_container()), environment_digest: Some(Digest::sha256("env")), required_capabilities: vec![Capability::Command], dependency_cache: Some(Digest::sha256("deps")), source_snapshot: Some(Digest::sha256("source")), required_artifacts: vec!["build-output".to_owned()], prefer_node: None, }) .unwrap() else { panic!("expected task placement"); }; assert_eq!(placement.node, NodeId::from("warm-node")); assert!(placement .reasons .iter() .any(|reason| reason.contains("environment"))); assert!(placement .reasons .iter() .any(|reason| reason.contains("source"))); assert!(placement .reasons .iter() .any(|reason| reason.contains("dependency"))); assert!(placement .reasons .iter() .any(|reason| reason.contains("artifact"))); let error = service .handle_request(CoordinatorRequest::ScheduleTask { tenant: "tenant".to_owned(), project: "project".to_owned(), environment: None, environment_digest: None, required_capabilities: vec![Capability::WindowsCommandDev], dependency_cache: None, source_snapshot: None, required_artifacts: Vec::new(), prefer_node: None, }) .unwrap_err(); assert!(error.to_string().contains("WindowsCommandDev")); service.quota.set_workflow_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::Spawn, 0)]), }); let quota_error = service .handle_request(CoordinatorRequest::ScheduleTask { tenant: "tenant".to_owned(), project: "project".to_owned(), environment: None, environment_digest: None, required_capabilities: vec![Capability::Command], dependency_cache: None, source_snapshot: None, required_artifacts: Vec::new(), prefer_node: None, }) .unwrap_err(); assert!(quota_error .to_string() .contains("quota unavailable for placement")); service.quota.set_workflow_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::Spawn, 10)]), }); service.admission.workflow_placement_allowed = false; let policy_error = service .handle_request(CoordinatorRequest::ScheduleTask { tenant: "tenant".to_owned(), project: "project".to_owned(), environment: None, environment_digest: None, required_capabilities: vec![Capability::Command], dependency_cache: None, source_snapshot: None, required_artifacts: Vec::new(), prefer_node: None, }) .unwrap_err(); assert!(policy_error.to_string().contains("policy denied placement")); } #[test] fn coordinator_side_task_launch_queues_worker_assignment() { let mut service = CoordinatorService::new(9); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), public_key: test_node_public_key("worker-linux"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: vec![Digest::sha256("env")], dependency_cache_digests: vec![Digest::sha256("deps")], source_snapshots: vec![Digest::sha256("source")], artifact_locations: vec!["bootstrap-artifact".to_owned()], direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::ProcessStarted { launch_attempt: None, epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-control".to_owned(), restart: false, }) .unwrap() else { panic!("expected coordinator-side process start"); }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), process: "vp-control".to_owned(), epoch, }) .unwrap(); service.artifact_registry.flush_metadata(ArtifactFlush { id: ArtifactId::from("bootstrap-artifact"), tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("vp-control"), producer_task: TaskInstanceId::from("bootstrap"), retaining_node: NodeId::from("worker-linux"), digest: Digest::sha256("bootstrap"), size: 9, }); let submitted_task_spec = TaskSpec { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("vp-control"), task_definition: clusterflux_core::TaskDefinitionId::from("compile-linux"), task_instance: clusterflux_core::TaskInstanceId::from("compile-linux-1"), dispatch: TaskDispatch::CoordinatorNodeWasm { export: Some("compile-linux".to_owned()), abi: WasmExportAbi::TaskV1, }, environment_id: Some("linux".to_owned()), environment: None, environment_digest: Some(Digest::sha256("env")), required_capabilities: BTreeSet::from([Capability::Command]), dependency_cache: Some(Digest::sha256("deps")), source_snapshot: Some(Digest::sha256("source")), required_artifacts: vec![ArtifactId::from("bootstrap-artifact")], args: vec![ TaskBoundaryValue::SmallJson(serde_json::json!("test")), TaskBoundaryValue::SourceSnapshot(Digest::sha256("source")), TaskBoundaryValue::Artifact(ArtifactHandle { id: ArtifactId::from("bootstrap-artifact"), digest: Digest::sha256("bootstrap"), size_bytes: 9, }), ], vfs_epoch: epoch, failure_policy: Default::default(), bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)), }; let CoordinatorResponse::TaskLaunched { process, task, placement, assignment, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: submitted_task_spec.clone(), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap() else { panic!("expected launched task"); }; assert_eq!(process, ProcessId::from("vp-control")); assert_eq!(task, TaskInstanceId::from("compile-linux-1")); assert_eq!(placement.node, NodeId::from("worker-linux")); assert!(placement .reasons .iter() .any(|reason| reason.contains("environment"))); assert!(placement .reasons .iter() .any(|reason| reason.contains("source"))); assert_eq!(assignment.node, NodeId::from("worker-linux")); assert_eq!(assignment.epoch, epoch); assert_eq!( assignment.task_spec.bundle_digest, Some(Digest::sha256(TEST_WASM_MODULE)) ); assert!(assignment.task_spec.product_mode_uses_remote_dispatch()); assert_eq!(assignment.task_spec, submitted_task_spec); assert_eq!( assignment.task_spec.environment_id.as_deref(), Some("linux") ); assert!(matches!( assignment.task_spec.dispatch, TaskDispatch::CoordinatorNodeWasm { export: Some(ref export), abi: WasmExportAbi::TaskV1, } if export == "compile-linux" )); assert_eq!(assignment.task_spec.vfs_epoch, epoch); assert_eq!(assignment.task_spec.args, submitted_task_spec.args); let CoordinatorResponse::TaskAssignment { assignment } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), }) .unwrap() else { panic!("expected worker assignment poll"); }; let assignment = assignment.expect("worker should receive queued assignment"); assert_eq!(assignment.process, ProcessId::from("vp-control")); assert_eq!(assignment.task, TaskInstanceId::from("compile-linux-1")); assert!(assignment.task_spec.product_mode_uses_remote_dispatch()); assert_eq!( assignment.task_spec.bundle_digest, Some(Digest::sha256(TEST_WASM_MODULE)) ); assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64()); let CoordinatorResponse::TaskJoined { join } = service .handle_request(CoordinatorRequest::JoinTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp-control".to_owned(), task: "compile-linux-1".to_owned(), }) .unwrap() else { panic!("expected pending join result"); }; assert_eq!(join.state, TaskJoinState::Pending); assert!(!join.remote_completion_observed); let CoordinatorResponse::TaskAssignment { assignment } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker-linux".to_owned(), }) .unwrap() else { panic!("expected empty worker assignment poll"); }; assert!(assignment.is_none()); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp-control".to_owned(), node: "worker-linux".to_owned(), task: "compile-linux-1".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 12, stderr_bytes: 0, stdout_tail: "ok".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/dap-output.txt".to_owned()), artifact_digest: Some(Digest::sha256("artifact")), artifact_size_bytes: Some(12), result: Some(TaskBoundaryValue::Artifact(ArtifactHandle { id: ArtifactId::from("dap-output.txt"), digest: Digest::sha256("artifact"), size_bytes: 12, })), }) .unwrap(); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: Some("vp-control".to_owned()), }) .unwrap() else { panic!("expected task events"); }; let event_placement = events[0] .placement .as_ref() .expect("task event should retain launch placement explanation"); assert_eq!(event_placement.node, NodeId::from("worker-linux")); assert_eq!(event_placement.reasons, placement.reasons); assert_eq!( events[0].result, Some(TaskBoundaryValue::Artifact(ArtifactHandle { id: ArtifactId::from("dap-output.txt"), digest: Digest::sha256("artifact"), size_bytes: 12, })) ); let CoordinatorResponse::TaskJoined { join } = service .handle_request(CoordinatorRequest::JoinTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp-control".to_owned(), task: "compile-linux-1".to_owned(), }) .unwrap() else { panic!("expected completed join result"); }; assert_eq!(join.state, TaskJoinState::Completed); assert!(join.remote_completion_observed); assert_eq!( join.result, Some(TaskBoundaryValue::Artifact(ArtifactHandle { id: ArtifactId::from("dap-output.txt"), digest: Digest::sha256("artifact"), size_bytes: 12, })) ); register_test_task_assignment( &mut service, "tenant", "project", "vp-control", "worker-linux", "wasm-add", "wasm-add", epoch, ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp-control".to_owned(), node: "worker-linux".to_owned(), task: "wasm-add".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 3, stderr_bytes: 0, stdout_tail: "42\n".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(42))), }) .unwrap(); let CoordinatorResponse::TaskJoined { join } = service .handle_request(CoordinatorRequest::JoinTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp-control".to_owned(), task: "wasm-add".to_owned(), }) .unwrap() else { panic!("expected serialized join result"); }; assert_eq!( join.result, Some(TaskBoundaryValue::SmallJson(serde_json::json!(42))) ); } #[test] fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order() { let mut service = CoordinatorService::new(41); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant".to_owned(), actor_user: "user".to_owned(), project: "project".to_owned(), name: "Duplicate instances".to_owned(), }) .unwrap(); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), public_key: test_node_public_key("worker"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 41, }) .unwrap(); let restart_compatibility = Digest::sha256("compile-u32-to-u32-abi-v1"); let (initial_module, initial_bundle_digest) = test_edited_task_bundle(&restart_compatibility, "initial-body"); for (instance, argument) in [("compile-1", 1), ("compile-2", 2)] { let mut spec = test_task_spec_instance("tenant", "project", "vp", "compile", instance, 41, []); spec.args = vec![TaskBoundaryValue::SmallJson(serde_json::json!(argument))]; spec.dispatch = TaskDispatch::CoordinatorNodeWasm { export: Some("compile_export".to_owned()), abi: WasmExportAbi::TaskV1, }; spec.bundle_digest = Some(initial_bundle_digest.clone()); let CoordinatorResponse::TaskLaunched { task, assignment, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, task_spec: spec, wait_for_node: false, artifact_path: format!("/vfs/artifacts/{instance}.json"), wasm_module_base64: initial_module.clone(), }) .unwrap() else { panic!("expected launched task instance"); }; assert_eq!(task, TaskInstanceId::from(instance)); assert_eq!( assignment.task_spec.task_definition, clusterflux_core::TaskDefinitionId::from("compile") ); assert_eq!( assignment.task_spec.task_instance, clusterflux_core::TaskInstanceId::from(instance) ); } for instance in ["compile-1", "compile-2"] { let CoordinatorResponse::TaskAssignment { assignment: Some(assignment), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), }) .unwrap() else { panic!("expected queued task assignment"); }; assert_eq!(assignment.task, TaskInstanceId::from(instance)); } service .handle_request(CoordinatorRequest::CancelTask { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), task: "compile-1".to_owned(), }) .unwrap(); for (instance, expected_cancelled) in [("compile-1", true), ("compile-2", false)] { let CoordinatorResponse::TaskControl { cancel_requested, abort_requested, .. } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskControl { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), task: instance.to_owned(), }) .unwrap() else { panic!("expected task control response"); }; assert_eq!(cancel_requested, expected_cancelled); assert!(!abort_requested); } for (instance, result) in [("compile-2", 22), ("compile-1", 11)] { service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), task: instance.to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(serde_json::json!(result))), }) .unwrap(); } for (instance, expected) in [("compile-1", 11), ("compile-2", 22)] { let CoordinatorResponse::TaskJoined { join } = service .handle_request(CoordinatorRequest::JoinTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp".to_owned(), task: instance.to_owned(), }) .unwrap() else { panic!("expected task join"); }; assert_eq!( join.task_instance, clusterflux_core::TaskInstanceId::from(instance) ); assert_eq!( join.result, Some(TaskBoundaryValue::SmallJson(serde_json::json!(expected))) ); } let duplicate = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, task_spec: test_task_spec_instance( "tenant", "project", "vp", "compile", "compile-1", 41, [], ), wait_for_node: false, artifact_path: "/vfs/artifacts/duplicate.json".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap_err(); assert!(duplicate.to_string().contains("fresh task-instance id")); let (replacement_module, replacement_bundle_digest) = test_edited_task_bundle(&restart_compatibility, "edited-body"); assert_ne!(replacement_bundle_digest, initial_bundle_digest); let CoordinatorResponse::TaskRestart { accepted, restarted_task_instance, restarted_attempt_id, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp".to_owned(), task: "compile-1".to_owned(), replacement_bundle: Some(TaskReplacementBundle { bundle_digest: replacement_bundle_digest.clone(), wasm_module_base64: replacement_module.clone(), source_snapshot: None, }), }) .unwrap() else { panic!("expected task restart response"); }; assert!(accepted); let restarted = restarted_task_instance.expect("restart returns the logical instance"); let restarted_attempt_id = restarted_attempt_id.expect("restart creates a new attempt"); assert!(restarted_attempt_id.starts_with("ta_")); assert_eq!(restarted, TaskInstanceId::from("compile-1")); assert_ne!(restarted, TaskInstanceId::from("compile-2")); let CoordinatorResponse::TaskAssignment { assignment: Some(restarted_assignment), } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), }) .unwrap() else { panic!("expected restarted assignment"); }; assert_eq!(restarted_assignment.task, restarted); assert_eq!( restarted_assignment.task_spec.task_definition, clusterflux_core::TaskDefinitionId::from("compile") ); assert_eq!( restarted_assignment.task_spec.args, vec![TaskBoundaryValue::SmallJson(serde_json::json!(1))] ); assert_eq!( restarted_assignment.task_spec.bundle_digest, Some(replacement_bundle_digest) ); assert_eq!( restarted_assignment.task_spec.dispatch, TaskDispatch::CoordinatorNodeWasm { export: Some("compile_export".to_owned()), abi: WasmExportAbi::TaskV1, } ); assert_eq!(restarted_assignment.wasm_module_base64, replacement_module); let incompatible_compatibility = Digest::sha256("compile-changed-signature"); let (incompatible_module, incompatible_bundle_digest) = test_edited_task_bundle(&incompatible_compatibility, "incompatible-body"); let CoordinatorResponse::TaskRestart { accepted, requires_whole_process_restart, restarted_task_instance, message, .. } = service .handle_request(CoordinatorRequest::RestartTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp".to_owned(), task: "compile-2".to_owned(), replacement_bundle: Some(TaskReplacementBundle { bundle_digest: incompatible_bundle_digest, wasm_module_base64: incompatible_module, source_snapshot: None, }), }) .unwrap() else { panic!("expected incompatible task restart response"); }; assert!(!accepted); assert!(requires_whole_process_restart); assert!(restarted_task_instance.is_none()); assert!(message.contains("task ABI changed")); let CoordinatorResponse::TaskJoined { join } = service .handle_request(CoordinatorRequest::JoinTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "vp".to_owned(), task: "compile-2".to_owned(), }) .unwrap() else { panic!("expected unaffected sibling join"); }; assert_eq!(join.state, TaskJoinState::Completed); assert_eq!( join.result, Some(TaskBoundaryValue::SmallJson(serde_json::json!(22))) ); } #[test] fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { let mut service = CoordinatorService::new(73); service .handle_request(CoordinatorRequest::CreateProject { tenant: "tenant-soak".to_owned(), actor_user: "user-soak".to_owned(), project: "project-soak".to_owned(), name: "Bounded soak".to_owned(), }) .unwrap(); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), node: "worker-soak".to_owned(), public_key: test_node_public_key("worker-soak"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), node: "worker-soak".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), actor_user: Some("user-soak".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-soak".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), node: "worker-soak".to_owned(), process: "vp-soak".to_owned(), epoch: 73, }) .unwrap(); let event_waves = 4; for index in 0..super::MAX_TASK_EVENTS_PER_PROCESS * event_waves { let task = format!("soak-task-{index}"); register_test_task_assignment( &mut service, "tenant-soak", "project-soak", "vp-soak", "worker-soak", "soak-task", &task, 73, ); service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), process: "vp-soak".to_owned(), node: "worker-soak".to_owned(), task, terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 8, stderr_bytes: 0, stdout_tail: "soak-log".to_owned(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: Some(TaskBoundaryValue::SmallJson(json!(index))), }) .unwrap(); } for index in 0..super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS * 3 { service .record_debug_audit_event( TenantId::from("tenant-soak"), ProjectId::from("project-soak"), ProcessId::from("vp-soak"), None, UserId::from("user-soak"), "soak_inspect", true, format!("bounded audit {index}"), ) .unwrap(); } for _ in 0..128 { service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), node: "worker-soak".to_owned(), }) .unwrap(); } for index in 0..2_000 { service.node_replay_nonces.insert( ( crate::NodeScopeKey::new( TenantId::from("tenant-soak"), ProjectId::from("project-soak"), NodeId::from("worker-soak"), ), format!("expired-{index}"), ), 0, ); } service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), node: "worker-soak".to_owned(), node_signature: Some(signed_node_heartbeat_in_scope( "tenant-soak", "project-soak", "worker-soak", "post-expiry-prune", )), }) .unwrap(); for index in 0..3 { service.record_task_completion_event(TaskCompletionEvent { tenant: TenantId::from("tenant-other"), project: ProjectId::from("project-other"), process: ProcessId::from("vp-other"), node: NodeId::from("worker-other"), executor: TaskExecutor::Node, task_definition: clusterflux_core::TaskDefinitionId::from("other"), task: TaskInstanceId::new(format!("other-{index}")), attempt_id: None, placement: None, terminal_state: TaskTerminalState::Completed, status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: None, artifact_digest: None, artifact_size_bytes: None, result: None, }); } let retained_soak_events = service .task_events .iter() .filter(|event| event.process == ProcessId::from("vp-soak")) .count(); let retained_other_events = service .task_events .iter() .filter(|event| event.process == ProcessId::from("vp-other")) .count(); let retained_soak_audit = service .debug_audit_events .iter() .filter(|event| event.process == ProcessId::from("vp-soak")) .count(); let retained_soak_checkpoints = service .task_restart_checkpoints .keys() .filter(|(_, _, process, _)| process == &ProcessId::from("vp-soak")) .count(); let retained_soak_nonces = service .node_replay_nonces .keys() .filter(|(scope, _)| { scope == &crate::NodeScopeKey::new( TenantId::from("tenant-soak"), ProjectId::from("project-soak"), NodeId::from("worker-soak"), ) }) .count(); assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS); assert_eq!(retained_other_events, 3); assert_eq!( retained_soak_audit, super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS ); assert_eq!( retained_soak_checkpoints, super::MAX_RESTART_CHECKPOINTS_PER_PROCESS ); assert!(retained_soak_nonces <= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY); assert!( super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY >= super::NODE_SIGNATURE_WINDOW_SECONDS as usize * 2 * 1_000 / clusterflux_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS as usize + 64, "the bounded node replay window must sustain artifact and assignment polls at the protocol minimum with control-message headroom" ); assert!(service .coordinator .active_process( &TenantId::from("tenant-soak"), &ProjectId::from("project-soak"), &ProcessId::from("vp-soak"), ) .is_some()); } #[test] fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() { let mut service = CoordinatorService::new(12); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), public_key: test_node_public_key("worker"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 12, }) .unwrap(); service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, task_spec: test_task_spec("tenant", "project", "vp", "parent", 12, []), wait_for_node: false, artifact_path: "/vfs/artifacts/parent.json".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap(); let child_request = CoordinatorRequest::LaunchChildTask { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), parent_task: "parent".to_owned(), task_spec: test_task_spec("tenant", "project", "vp", "child", 12, []), wait_for_node: false, artifact_path: "/vfs/artifacts/child.json".to_owned(), wasm_module_base64: test_wasm_module_base64(), }; let unsigned = service.handle_request(child_request.clone()).unwrap_err(); assert!(unsigned.to_string().contains("signed_node")); let CoordinatorResponse::TaskLaunched { actor, task, .. } = service .handle_signed_node_request_auto(child_request) .unwrap() else { panic!("expected signed child launch"); }; assert_eq!(task, TaskInstanceId::from("child")); assert_eq!(actor.kind, "task"); assert_eq!(actor.credential_kind, CredentialKind::TaskCredential); assert!(actor.scopes.contains(&"process:spawn-child".to_owned())); let wrong_parent = service .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), parent_task: "not-active".to_owned(), task: "child".to_owned(), }) .unwrap_err(); assert!(wrong_parent.to_string().contains("active parent")); let CoordinatorResponse::TaskJoined { join } = service .handle_signed_node_request_auto(CoordinatorRequest::JoinChildTask { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "vp".to_owned(), node: "worker".to_owned(), parent_task: "parent".to_owned(), task: "child".to_owned(), }) .unwrap() else { panic!("expected signed child join"); }; assert_eq!(join.state, TaskJoinState::Pending); } #[test] fn coordinator_rejects_named_environment_without_requirements() { let mut service = CoordinatorService::new(10); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "uncached-worker".to_owned(), public_key: test_node_public_key("uncached-worker"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "uncached-worker".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::ProcessStarted { launch_attempt: None, epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-environment".to_owned(), restart: false, }) .unwrap() else { panic!("expected process start"); }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "uncached-worker".to_owned(), process: "vp-environment".to_owned(), epoch, }) .unwrap(); let mut task_spec = test_task_spec( "tenant", "project", "vp-environment", "compile-linux", epoch, [], ); task_spec.environment_id = Some("missing-environment".to_owned()); task_spec.environment_digest = Some(Digest::sha256("missing-environment")); let error = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/environment-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap_err(); assert!(error.to_string().contains("named environment cache")); } #[test] fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { let mut service = CoordinatorService::new(10); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-control".to_owned(), restart: false, }) .unwrap(); let error = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "vp-control", "compile-linux", 10, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: false, artifact_path: "/vfs/artifacts/dap-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap_err(); assert!(error.to_string().contains("no capable node")); } #[test] fn coordinator_side_task_launch_can_wait_for_capable_worker() { let mut service = CoordinatorService::new(11); let CoordinatorResponse::ProcessStarted { epoch: other_epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "other-tenant".to_owned(), project: "other-project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "other-vp-wait".to_owned(), restart: false, }) .unwrap() else { panic!("expected unrelated process start"); }; let CoordinatorResponse::TaskQueued { queued_tasks: other_queued_tasks, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "other-tenant", "other-project", "other-vp-wait", "other-compile", other_epoch, [Capability::Command], ), tenant: "other-tenant".to_owned(), project: "other-project".to_owned(), actor_user: Some("other-user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: true, artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap() else { panic!("expected unrelated queued task launch"); }; assert_eq!(other_queued_tasks, 1); let CoordinatorResponse::ProcessStarted { launch_attempt: None, epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-wait".to_owned(), restart: false, }) .unwrap() else { panic!("expected process start"); }; let CoordinatorResponse::TaskQueued { process, task, reason, queued_tasks, charged_spawns, .. } = service .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { task_spec: test_task_spec( "tenant", "project", "vp-wait", "compile-linux", epoch, [Capability::Command], ), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, wait_for_node: true, artifact_path: "/vfs/artifacts/wait-output.txt".to_owned(), wasm_module_base64: test_wasm_module_base64(), }) .unwrap() else { panic!("expected queued task launch"); }; assert_eq!(process, ProcessId::from("vp-wait")); assert_eq!(task, TaskInstanceId::from("compile-linux")); assert!(reason.contains("waiting for any capable node")); assert_eq!(queued_tasks, 1); assert_eq!(charged_spawns, 2); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "late-worker".to_owned(), public_key: test_node_public_key("late-worker"), }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "late-worker".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); let CoordinatorResponse::TaskAssignment { assignment } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "late-worker".to_owned(), }) .unwrap() else { panic!("expected pending assignment poll"); }; let assignment = assignment.expect("late worker should receive pending assignment"); assert_eq!(assignment.process, ProcessId::from("vp-wait")); assert_eq!(assignment.task, TaskInstanceId::from("compile-linux")); assert_eq!(assignment.node, NodeId::from("late-worker")); assert_eq!(assignment.epoch, epoch); assert!(assignment.task_spec.product_mode_uses_remote_dispatch()); assert_eq!(assignment.wasm_module_base64, test_wasm_module_base64()); let CoordinatorResponse::TaskAssignment { assignment } = service .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "late-worker".to_owned(), }) .unwrap() else { panic!("expected empty assignment poll"); }; assert!(assignment.is_none()); } #[test] fn service_rejects_malformed_node_capability_report() { let mut service = CoordinatorService::new(1); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); let mut capabilities = linux_capabilities(); capabilities .source_providers .insert("../checkout".to_owned()); let error = service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities, cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap_err(); assert!(error.to_string().contains("source provider id")); assert!(service.node_descriptors.is_empty()); } #[test] fn service_meters_rendezvous_before_direct_transfer_plan() { let mut service = CoordinatorService::new(7); service.quota.set_rendezvous_limits(ResourceLimits { limits: BTreeMap::from([(LimitKind::RendezvousAttempt, 2)]), }); let CoordinatorResponse::RendezvousPlan { plan, charged_rendezvous_attempts, } = service .handle_request(CoordinatorRequest::RequestRendezvous { scope: data_plane_scope("project"), source: endpoint("node-a"), destination: endpoint("node-b"), direct_connectivity: true, failure_reason: String::new(), }) .unwrap() else { panic!("expected rendezvous plan"); }; assert_eq!(charged_rendezvous_attempts, 1); assert_eq!(plan.scope.project, ProjectId::from("project")); assert_eq!(plan.source.node, NodeId::from("node-a")); assert_eq!(plan.destination.node, NodeId::from("node-b")); assert!(plan.coordinator_assisted_rendezvous); assert!(!plan.coordinator_bulk_relay_allowed); let unavailable = service .handle_request(CoordinatorRequest::RequestRendezvous { scope: data_plane_scope("project"), source: endpoint("node-a"), destination: endpoint("node-b"), direct_connectivity: false, failure_reason: "nat traversal failed".to_owned(), }) .unwrap_err(); let unavailable = unavailable.to_string(); assert!(unavailable.contains("nat traversal failed")); assert!(unavailable.contains("coordinator bulk relay is disabled")); let quota = service .handle_request(CoordinatorRequest::RequestRendezvous { scope: data_plane_scope("project"), source: endpoint("node-a"), destination: endpoint("node-b"), direct_connectivity: true, failure_reason: String::new(), }) .unwrap_err(); assert!(quota.to_string().contains("RendezvousAttempt")); } #[test] fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() { let mut service = CoordinatorService::new(7); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node-a".to_owned(), public_key: test_node_public_key("node-a"), }) .unwrap(); let response = service .handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous { scope: data_plane_scope("project"), source: endpoint("node-a"), destination: endpoint("node-b"), direct_connectivity: true, failure_reason: String::new(), }) .unwrap(); let CoordinatorResponse::RendezvousPlan { plan, .. } = response else { panic!("expected signed rendezvous plan"); }; assert_eq!(plan.source.node, NodeId::from("node-a")); assert_eq!(plan.destination.node, NodeId::from("node-b")); } #[test] fn signed_hostile_artifact_paths_return_errors_and_the_same_service_stays_healthy() { let mut service = CoordinatorService::new(27); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "hostile-path-process".to_owned(), restart: false, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), process: "hostile-path-process".to_owned(), epoch: 27, }) .unwrap(); let invalid_metadata = service .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "hostile-path-process".to_owned(), node: "node".to_owned(), task: "metadata-task".to_owned(), artifact_path: Some("/vfs/artifacts/bad artifact!".to_owned()), artifact_digest: Some(Digest::sha256("bad")), artifact_size_bytes: Some(3), large_bytes_uploaded: false, }) .unwrap_err(); assert!( invalid_metadata .to_string() .contains("invalid VFS artifact path"), "unexpected error: {invalid_metadata}" ); let valid_metadata = service .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "hostile-path-process".to_owned(), node: "node".to_owned(), task: "metadata-task".to_owned(), artifact_path: Some("/vfs/artifacts/valid-artifact".to_owned()), artifact_digest: Some(Digest::sha256("valid")), artifact_size_bytes: Some(5), large_bytes_uploaded: false, }) .unwrap(); assert!(matches!( valid_metadata, CoordinatorResponse::VfsMetadataRecorded { .. } )); register_test_task_assignment( &mut service, "tenant", "project", "hostile-path-process", "node", "child", "child-instance", 27, ); let invalid_completion = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "hostile-path-process".to_owned(), node: "node".to_owned(), task: "child-instance".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/repeated//component".to_owned()), artifact_digest: Some(Digest::sha256("bad-completion")), artifact_size_bytes: Some(0), result: None, }) .unwrap_err(); assert!( invalid_completion .to_string() .contains("invalid VFS artifact path"), "unexpected error: {invalid_completion}" ); let valid_completion = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "hostile-path-process".to_owned(), node: "node".to_owned(), task: "child-instance".to_owned(), terminal_state: Some(TaskTerminalState::Completed), status_code: Some(0), stdout_bytes: 0, stderr_bytes: 0, stdout_tail: String::new(), stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/valid-completion".to_owned()), artifact_digest: Some(Digest::sha256("valid-completion")), artifact_size_bytes: Some(0), result: None, }) .unwrap(); assert!(matches!( valid_completion, CoordinatorResponse::TaskRecorded { .. } )); } #[test] fn service_rejects_task_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), actor_user: None, actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "process".to_owned(), restart: false, }) .unwrap(); let error = service .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "compile-linux".to_owned(), terminal_state: None, status_code: Some(0), stdout_bytes: 128, stderr_bytes: 64, stdout_tail: "foreign stdout".to_owned(), stderr_tail: "foreign stderr".to_owned(), stdout_truncated: false, stderr_truncated: false, artifact_path: Some("/vfs/artifacts/foreign.txt".to_owned()), artifact_digest: Some(Digest::sha256("foreign-artifact")), artifact_size_bytes: Some(128), result: None, }) .unwrap_err(); assert!(error.to_string().contains("not enrolled")); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap() else { panic!("expected task events"); }; assert!(events.is_empty()); } #[test] fn service_rejects_task_event_access_using_retained_process_scope() { let mut service = CoordinatorService::new(1); service.process_scope_history.push_back(( TenantId::from("tenant-a"), ProjectId::from("project-a"), ProcessId::from("process"), )); let error = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), actor_user: "user".to_owned(), process: Some("process".to_owned()), }) .unwrap_err(); assert!(error .to_string() .contains("outside the virtual process tenant/project scope")); } #[test] fn service_rejects_node_capability_report_outside_enrollment_scope() { let mut service = CoordinatorService::new(1); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); let error = service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap_err(); assert!(error.to_string().contains("not enrolled")); assert!(service.node_descriptors.is_empty()); } #[test] fn service_rejects_source_preparation_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); let error = service .handle_signed_node_request_auto(CoordinatorRequest::CompleteSourcePreparation { tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), node: "node".to_owned(), provider: SourceProviderKind::Filesystem, source_snapshot: Digest::sha256("foreign-source"), }) .unwrap_err(); assert!(error.to_string().contains("not enrolled")); } #[test] fn service_rejects_unknown_node_heartbeat() { let mut service = CoordinatorService::new(1); let error = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "missing".to_owned(), node_signature: None, }) .unwrap_err(); assert!(error.to_string().contains("not enrolled")); } #[test] fn service_requires_signed_node_heartbeat_from_enrolled_key() { let mut service = CoordinatorService::new(5); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); let unsigned = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: None, }) .unwrap_err(); assert!(unsigned.to_string().contains("signed proof")); let wrong_private_key = test_node_private_key("other-node"); let wrong_signature = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat_with_private_key( "node", &wrong_private_key, "wrong-node-key", )), }) .unwrap_err(); assert!(wrong_signature.to_string().contains("signature")); let signed = signed_node_heartbeat("node", "fresh-node-heartbeat"); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed.clone()), }) .unwrap(); assert_eq!( heartbeat, CoordinatorResponse::NodeHeartbeat { node: NodeId::from("node"), epoch: 5, } ); let replay = service .handle_request(CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed), }) .unwrap_err(); assert!(replay.to_string().contains("nonce")); } #[test] fn service_rejects_raw_node_originated_requests_without_signed_envelope() { let mut service = CoordinatorService::new(5); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }) .unwrap(); let error = service .handle_request(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap_err(); assert!(error.to_string().contains("signed_node envelope proof")); } #[test] fn service_stream_accepts_multiple_requests_on_one_connection() { let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); let server = std::thread::spawn(move || { let mut service = CoordinatorService::new(3); let (stream, _) = listener.accept().unwrap(); service.handle_stream_local_trusted(stream).unwrap(); }); let mut stream = TcpStream::connect(addr).unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); write_coordinator_wire_request( &mut stream, &CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), public_key: test_node_public_key("node"), }, "stream-1", ); let mut line = String::new(); reader.read_line(&mut line).unwrap(); assert!(matches!( serde_json::from_str::(&line).unwrap(), CoordinatorResponse::NodeAttached { .. } )); write_coordinator_wire_request( &mut stream, &CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")), }, "stream-2", ); line.clear(); reader.read_line(&mut line).unwrap(); assert_eq!( serde_json::from_str::(&line).unwrap(), CoordinatorResponse::NodeHeartbeat { node: NodeId::from("node"), epoch: 3, } ); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); } #[test] fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); let server = std::thread::spawn(move || { let mut service = CoordinatorService::new(3); service .issue_cli_session( TenantId::from("tenant"), ProjectId::from("project"), UserId::from("user"), "strict-stream-session", None, ) .unwrap(); let (stream, _) = listener.accept().unwrap(); service.handle_stream(stream).unwrap(); }); let mut stream = TcpStream::connect(addr).unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); write_coordinator_wire_request( &mut stream, &CoordinatorRequest::StartProcess { launch_attempt: None, tenant: "victim-tenant".to_owned(), project: "victim-project".to_owned(), actor_user: Some("forged-user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, process: "vp-forged".to_owned(), restart: false, }, "strict-stream-forged", ); let mut line = String::new(); reader.read_line(&mut line).unwrap(); let CoordinatorResponse::Error { error } = serde_json::from_str::(&line).unwrap() else { panic!("expected strict body-authority denial"); }; assert!(error .message .contains("request-body identity fields are not authority")); assert_eq!(error.request_id, "strict-stream-forged"); assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden); write_coordinator_wire_request( &mut stream, &CoordinatorRequest::Authenticated { session_secret: "strict-stream-session".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "vp-authenticated".to_owned(), restart: false, }, }, "strict-stream-authenticated", ); line.clear(); reader.read_line(&mut line).unwrap(); let CoordinatorResponse::ProcessStarted { launch_attempt: None, process, actor, .. } = serde_json::from_str::(&line).unwrap() else { panic!("expected authenticated strict process start"); }; assert_eq!(process, ProcessId::from("vp-authenticated")); assert_eq!(actor.user, Some(UserId::from("user"))); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); } #[test] fn service_stream_rejects_invalid_versioned_envelope_metadata() { let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); let server = std::thread::spawn(move || { let mut service = CoordinatorService::new(3); let (stream, _) = listener.accept().unwrap(); service.handle_stream(stream).unwrap(); }); let mut stream = TcpStream::connect(addr).unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); let mut wire_request = coordinator_wire_request( "bad-operation", serde_json::to_value(CoordinatorRequest::Ping).unwrap(), ); wire_request["operation"] = serde_json::Value::String("attach_node".to_owned()); serde_json::to_writer(&mut stream, &wire_request).unwrap(); stream.write_all(b"\n").unwrap(); stream.flush().unwrap(); let mut line = String::new(); reader.read_line(&mut line).unwrap(); let response = serde_json::from_str::(&line).unwrap(); let CoordinatorResponse::Error { error } = response else { panic!("expected invalid wire envelope response"); }; assert!(error .message .contains("operation attach_node does not match payload operation ping")); assert_eq!(error.request_id, "bad-operation"); assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); } #[test] fn coordinator_protocol_rejects_client_supplied_authority_fields() { for payload in [ json!({ "type": "create_node_enrollment_grant", "tenant": "tenant", "project": "project", "actor_user": "user", "grant": "attacker-chosen", "ttl_seconds": 60, }), json!({ "type": "exchange_node_enrollment_grant", "tenant": "tenant", "project": "project", "node": "node", "public_key": "key", "enrollment_grant": "grant", "now_epoch_seconds": 0, }), json!({ "type": "schedule_task", "tenant": "tenant", "project": "project", "environment": null, "environment_digest": null, "required_capabilities": [], "dependency_cache": null, "source_snapshot": null, "required_artifacts": [], "quota_available": true, "policy_allowed": true, "prefer_node": null, }), json!({ "type": "create_artifact_download_link", "tenant": "tenant", "project": "project", "actor_user": "user", "artifact": "artifact", "max_bytes": 1, "token_nonce": "attacker-chosen", "ttl_seconds": 60, }), ] { let error = serde_json::from_value::(payload).unwrap_err(); assert!(error.to_string().contains("unknown field")); } } #[test] fn coordinator_generates_and_bounds_node_enrollment_grants() { let mut service = CoordinatorService::new(7); service.set_server_time(100); let first = service .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), ttl_seconds: u64::MAX, }) .unwrap(); let second = service .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), ttl_seconds: u64::MAX, }) .unwrap(); let CoordinatorResponse::NodeEnrollmentGrantCreated { grant: first_grant, expires_at_epoch_seconds, .. } = first else { panic!("expected enrollment grant"); }; let CoordinatorResponse::NodeEnrollmentGrantCreated { grant: second_grant, .. } = second else { panic!("expected enrollment grant"); }; assert!(first_grant.starts_with("node_grant_")); assert_ne!(first_grant, second_grant); assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60); } #[test] fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() { let mut service = CoordinatorService::new(7); for (tenant, project, user, secret) in [ ("tenant-a", "project-a", "user-a", "session-a"), ("tenant-b", "project-b", "user-b", "session-b"), ] { service .issue_cli_session( TenantId::from(tenant), ProjectId::from(project), UserId::from(user), secret, None, ) .unwrap(); } service.set_server_time(100); service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "process-one".to_owned(), restart: false, }, }) .unwrap(); let CoordinatorResponse::ProcessSummaries { processes, next_cursor, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: None, limit: 1, }, }) .unwrap() else { panic!("expected process summaries"); }; assert_eq!(processes.len(), 1); assert_eq!(processes[0].process, ProcessId::from("process-one")); assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active); assert_eq!(processes[0].activity, ProcessActivityState::Running); assert_eq!(processes[0].started_at_epoch_seconds, 100); assert!(next_cursor.is_none()); let CoordinatorResponse::ProcessSummaries { processes, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-b".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: None, limit: 10, }, }) .unwrap() else { panic!("expected scoped process summaries"); }; assert!(processes.is_empty()); service.set_server_time(120); service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::AbortProcess { process: "process-one".to_owned(), launch_attempt: None, }, }) .unwrap(); let CoordinatorResponse::ProcessSummaries { processes, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: None, limit: 10, }, }) .unwrap() else { panic!("expected terminal process summary"); }; assert_eq!( processes[0].lifecycle, ProcessLifecycleState::RecentTerminal ); assert_eq!(processes[0].activity, ProcessActivityState::Cancelled); assert_eq!( processes[0].final_result, Some(ProcessFinalResult::Cancelled) ); assert_eq!(processes[0].ended_at_epoch_seconds, Some(120)); service.set_server_time(130); service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: "process-two".to_owned(), restart: false, }, }) .unwrap(); let CoordinatorResponse::ProcessSummaries { processes, next_cursor: Some(cursor), .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: None, limit: 1, }, }) .unwrap() else { panic!("expected first process summary page"); }; assert_eq!(processes[0].process, ProcessId::from("process-two")); let CoordinatorResponse::ProcessSummaries { processes, next_cursor: None, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: Some(cursor), limit: 1, }, }) .unwrap() else { panic!("expected final process summary page"); }; assert_eq!(processes[0].process, ProcessId::from("process-one")); let oversized = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor: None, limit: 101, }, }) .unwrap_err(); assert!(oversized.to_string().contains("limit")); assert!(oversized.to_string().contains("100")); } #[test] fn process_summary_eviction_releases_live_log_accounting_state() { let mut service = CoordinatorService::new(7); let tenant = TenantId::from("tenant-summary-bound"); let project = ProjectId::from("project-summary-bound"); let task = TaskInstanceId::from("task"); for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT { let process = ProcessId::new(format!("process-{index:03}")); service.record_process_started(&tenant, &project, &process, index as u64); service.record_process_terminal( &tenant, &project, &process, ProcessFinalResult::Completed, index as u64 + 1, ); let key = ( tenant.clone(), project.clone(), process, task.clone(), "stdout".to_owned(), ); service.recent_log_accounted_bytes.insert(key.clone(), 10); service.recent_log_truncated_streams.insert(key); } let evicted = ProcessId::from("process-000"); service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000); assert!(!service.process_summaries.contains_key(&( tenant.clone(), project.clone(), evicted.clone() ))); assert!(!service.recent_log_accounted_bytes.keys().any( |(entry_tenant, entry_project, process, _, _)| { entry_tenant == &tenant && entry_project == &project && process == &evicted } )); assert!(!service.recent_log_truncated_streams.iter().any( |(entry_tenant, entry_project, process, _, _)| { entry_tenant == &tenant && entry_project == &project && process == &evicted } )); } #[test] fn web_node_summaries_are_scoped_paginated_and_hard_bounded() { let mut service = CoordinatorService::new(7); service .issue_cli_session( TenantId::from("tenant"), ProjectId::from("project"), UserId::from("user"), "session", None, ) .unwrap(); for node in ["node-a", "node-b", "node-c"] { enroll_test_node( &mut service, "tenant", "project", node, &test_node_public_key(node), ); service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: "tenant".to_owned(), project: "project".to_owned(), node: node.to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: Vec::new(), direct_connectivity: true, online: true, }) .unwrap(); } let CoordinatorResponse::NodeSummaries { nodes, next_cursor: Some(cursor), .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor: None, limit: 2, }, }) .unwrap() else { panic!("expected first node-summary page"); }; assert_eq!( nodes .iter() .map(|node| node.id.as_str()) .collect::>(), ["node-a", "node-b"] ); let CoordinatorResponse::NodeSummaries { nodes, next_cursor: None, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor: Some(cursor), limit: 2, }, }) .unwrap() else { panic!("expected final node-summary page"); }; assert_eq!(nodes.len(), 1); assert_eq!(nodes[0].id, NodeId::from("node-c")); let oversized = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor: None, limit: 201, }, }) .unwrap_err(); assert!(oversized.to_string().contains("limit")); assert!(oversized.to_string().contains("200")); } #[test] fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() { let mut service = CoordinatorService::new(7); for (tenant, project, user, secret, node) in [ ("tenant-a", "project-a", "user-a", "session-a", "node-a"), ("tenant-b", "project-b", "user-b", "session-b", "node-b"), ] { service .issue_cli_session( TenantId::from(tenant), ProjectId::from(project), UserId::from(user), secret, None, ) .unwrap(); enroll_test_node( &mut service, tenant, project, node, &test_node_public_key(node), ); } service.set_server_time(100); for (tenant, project, node) in [ ("tenant-a", "project-a", "node-a"), ("tenant-b", "project-b", "node-b"), ] { service .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), capabilities: linux_capabilities(), cached_environment_digests: Vec::new(), dependency_cache_digests: Vec::new(), source_snapshots: Vec::new(), artifact_locations: vec!["shared-artifact".to_owned()], direct_connectivity: true, online: false, }) .unwrap(); } let CoordinatorResponse::NodeSummaries { nodes, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor: None, limit: 200, }, }) .unwrap() else { panic!("expected node summaries"); }; assert_eq!(nodes.len(), 1); assert_eq!(nodes[0].id, NodeId::from("node-a")); assert!(nodes[0].online); assert!(!nodes[0].stale); assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); assert_eq!(nodes[0].capabilities.os, Os::Linux); for (tenant, project, node, digest) in [ ("tenant-a", "project-a", "node-a", "tenant-a-bytes"), ("tenant-b", "project-b", "node-b", "tenant-b-bytes"), ] { service.artifact_registry.flush_metadata(ArtifactFlush { id: ArtifactId::from("shared-artifact"), tenant: TenantId::from(tenant), project: ProjectId::from(project), process: ProcessId::from("process-one"), producer_task: TaskInstanceId::from("task-one"), retaining_node: NodeId::from(node), digest: Digest::sha256(digest), size: digest.len() as u64, }); } service.artifact_registry.flush_metadata(ArtifactFlush { id: ArtifactId::from("second-artifact"), tenant: TenantId::from("tenant-a"), project: ProjectId::from("project-a"), process: ProcessId::from("process-two"), producer_task: TaskInstanceId::from("task-two"), retaining_node: NodeId::from("node-a"), digest: Digest::sha256("second"), size: 6, }); let CoordinatorResponse::Artifacts { artifacts, next_cursor: Some(cursor), } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListArtifacts { process: None, cursor: None, limit: 1, }, }) .unwrap() else { panic!("expected first artifact page"); }; assert_eq!(artifacts.len(), 1); assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact")); assert_eq!(artifacts[0].availability, ArtifactAvailability::Available); let CoordinatorResponse::Artifacts { artifacts, next_cursor: None, } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListArtifacts { process: None, cursor: Some(cursor), limit: 1, }, }) .unwrap() else { panic!("expected final artifact page"); }; assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact")); assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes")); let cross_tenant = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::GetArtifact { artifact: "tenant-b-only".to_owned(), }, }) .unwrap_err(); assert!(cross_tenant.to_string().contains("does not exist")); let CoordinatorResponse::Artifact { artifact } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-b".to_owned(), request: AuthenticatedCoordinatorRequest::GetArtifact { artifact: "shared-artifact".to_owned(), }, }) .unwrap() else { panic!("expected tenant-b artifact"); }; assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes")); service.set_server_time(131); let CoordinatorResponse::NodeSummaries { nodes, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor: None, limit: 200, }, }) .unwrap() else { panic!("expected stale node summary"); }; assert!(!nodes[0].online); assert!(nodes[0].stale); assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); let CoordinatorResponse::Artifact { artifact } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::GetArtifact { artifact: "shared-artifact".to_owned(), }, }) .unwrap() else { panic!("expected offline artifact metadata"); }; assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline); assert!(!artifact.downloadable_now); service .artifact_registry .sync_to_explicit_store( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), &ArtifactId::from("shared-artifact"), "store://tenant-a/shared-artifact", ) .unwrap(); let CoordinatorResponse::Artifact { artifact } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::GetArtifact { artifact: "shared-artifact".to_owned(), }, }) .unwrap() else { panic!("expected explicitly retained artifact metadata"); }; assert_eq!(artifact.availability, ArtifactAvailability::Available); assert_eq!( artifact.retention_state, ArtifactRetentionState::ExplicitStorage ); assert!(artifact.downloadable_now); } #[test] fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() { let mut service = CoordinatorService::new(7); for (tenant, project, user, secret, node, process) in [ ( "tenant-a", "project-a", "user-a", "session-a", "node-a", "process-shared", ), ( "tenant-b", "project-b", "user-b", "session-b", "node-b", "process-b", ), ] { service .issue_cli_session( TenantId::from(tenant), ProjectId::from(project), UserId::from(user), secret, None, ) .unwrap(); enroll_test_node( &mut service, tenant, project, node, &test_node_public_key(node), ); service .handle_request(CoordinatorRequest::Authenticated { session_secret: secret.to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { launch_attempt: None, process: process.to_owned(), restart: false, }, }) .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { tenant: tenant.to_owned(), project: project.to_owned(), node: node.to_owned(), process: process.to_owned(), epoch: 7, }) .unwrap(); } service.set_server_time(100); let first = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), process: "process-shared".to_owned(), node: "node-a".to_owned(), task: "task-one".to_owned(), stream: TaskLogStream::Stdout, offset: 0, source_bytes: 5, text: "hello".to_owned(), truncated: false, }) .unwrap(); let CoordinatorResponse::TaskLogChunkRecorded { sequence: Some(first_sequence), next_offset: 5, .. } = first else { panic!("expected first live log sequence"); }; assert_eq!( service.quota.used_log_bytes( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), 100, ), 5, "live bytes must be charged when accepted" ); let retry = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), process: "process-shared".to_owned(), node: "node-a".to_owned(), task: "task-one".to_owned(), stream: TaskLogStream::Stdout, offset: 0, source_bytes: 5, text: "hello".to_owned(), truncated: false, }) .unwrap(); assert!(matches!( retry, CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. } )); assert_eq!( service.quota.used_log_bytes( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), 100, ), 5, "a retried chunk must not be charged twice" ); service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { tenant: "tenant-a".to_owned(), project: "project-a".to_owned(), process: "process-shared".to_owned(), node: "node-a".to_owned(), task: "task-one".to_owned(), stream: TaskLogStream::Stdout, offset: 8, source_bytes: 2, text: "ok".to_owned(), truncated: false, }) .unwrap(); assert_eq!( service.quota.used_log_bytes( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), 100, ), 10, "a gap and the delivered bytes must both count toward source-byte usage" ); let CoordinatorResponse::RecentLogs { entries, next_sequence: Some(cursor), history_truncated: false, } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListRecentLogs { process: "process-shared".to_owned(), task: None, after_sequence: None, limit: 2, }, }) .unwrap() else { panic!("expected first recent-log page"); }; assert_eq!(entries.len(), 2); assert_eq!(entries[0].sequence, first_sequence); assert_eq!(entries[0].text, "hello"); assert!(entries[1].text.contains("3 bytes")); assert!(entries[1].truncated); let CoordinatorResponse::RecentLogs { entries, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListRecentLogs { process: "process-shared".to_owned(), task: None, after_sequence: Some(cursor), limit: 2, }, }) .unwrap() else { panic!("expected second recent-log page"); }; assert_eq!(entries.len(), 1); assert_eq!(entries[0].text, "ok"); service .handle_report_task_log( "tenant-a".to_owned(), "project-a".to_owned(), "process-shared".to_owned(), "node-a".to_owned(), "task-one".to_owned(), 12, 0, "hello???okZZ".to_owned(), String::new(), false, false, false, ) .unwrap(); assert_eq!( service.quota.used_log_bytes( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), 100, ), 12, "the final summary must charge only source bytes not already charged live" ); assert_eq!( service .recent_logs .get(&(TenantId::from("tenant-a"), ProjectId::from("project-a"))) .unwrap() .back() .unwrap() .text, "ZZ", "final-tail reconciliation must append only the nonduplicating suffix" ); service .handle_report_task_log( "tenant-a".to_owned(), "project-a".to_owned(), "process-shared".to_owned(), "node-a".to_owned(), "task-one".to_owned(), 12, 0, "hello???okZZ".to_owned(), String::new(), false, false, false, ) .unwrap(); assert_eq!( service.quota.used_log_bytes( &TenantId::from("tenant-a"), &ProjectId::from("project-a"), 100, ), 12, "replayed final accounting must be idempotent" ); let marker = service .handle_report_task_log_chunk( "tenant-a".to_owned(), "project-a".to_owned(), "process-shared".to_owned(), "node-a".to_owned(), "task-one".to_owned(), TaskLogStream::Stdout, 12, 0, "[log output truncated at node capture limit]".to_owned(), true, ) .unwrap(); assert!(matches!( marker, CoordinatorResponse::TaskLogChunkRecorded { sequence: Some(_), next_offset: 12, .. } )); let repeated_marker = service .handle_report_task_log_chunk( "tenant-a".to_owned(), "project-a".to_owned(), "process-shared".to_owned(), "node-a".to_owned(), "task-one".to_owned(), TaskLogStream::Stdout, 12, 0, "[log output truncated at node capture limit]".to_owned(), true, ) .unwrap(); assert!(matches!( repeated_marker, CoordinatorResponse::TaskLogChunkRecorded { sequence: None, next_offset: 12, .. } )); let cross_tenant = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-b".to_owned(), request: AuthenticatedCoordinatorRequest::ListRecentLogs { process: "process-shared".to_owned(), task: None, after_sequence: None, limit: 10, }, }) .unwrap_err(); assert!(cross_tenant.to_string().contains("outside")); let oversized = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListRecentLogs { process: "process-shared".to_owned(), task: None, after_sequence: None, limit: 201, }, }) .unwrap_err(); assert!(oversized.to_string().contains("limit")); assert!(oversized.to_string().contains("200")); service .handle_report_task_log_chunk( "tenant-b".to_owned(), "project-b".to_owned(), "process-b".to_owned(), "node-b".to_owned(), "task-b".to_owned(), TaskLogStream::Stderr, 0, 1, "b".to_owned(), false, ) .unwrap(); for offset in 10..310 { service .handle_report_task_log_chunk( "tenant-a".to_owned(), "project-a".to_owned(), "process-shared".to_owned(), "node-a".to_owned(), "task-one".to_owned(), TaskLogStream::Stdout, offset, 1, "x".to_owned(), false, ) .unwrap(); } let tenant_a_logs = &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))]; assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS); let tenant_b_logs = &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))]; assert_eq!(tenant_b_logs.len(), 1); assert_eq!(tenant_b_logs[0].text, "b"); let CoordinatorResponse::RecentLogs { entries, history_truncated, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "session-a".to_owned(), request: AuthenticatedCoordinatorRequest::ListRecentLogs { process: "process-shared".to_owned(), task: None, after_sequence: None, limit: 200, }, }) .unwrap() else { panic!("expected bounded recent-log response"); }; assert_eq!(entries.len(), 200); assert!(history_truncated); }