diff --git a/CLUSTERFLUX_PUBLIC_TREE.json b/CLUSTERFLUX_PUBLIC_TREE.json index 8bf7704..55b2238 100644 --- a/CLUSTERFLUX_PUBLIC_TREE.json +++ b/CLUSTERFLUX_PUBLIC_TREE.json @@ -1,7 +1,7 @@ { "kind": "clusterflux-filtered-public-tree", - "source_commit": "6d8c5c5b2a8b1da23eb69123e478237bd22883bd", - "release_name": "release-6d8c5c5b2a8b", + "source_commit": "e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d", + "release_name": "release-e47f9c27bbeb", "filtered_out": [ "private/**", "internal/**", diff --git a/crates/clusterflux-control/src/lib.rs b/crates/clusterflux-control/src/lib.rs index 43be04b..481e718 100644 --- a/crates/clusterflux-control/src/lib.rs +++ b/crates/clusterflux-control/src/lib.rs @@ -61,12 +61,20 @@ impl ControlSession { endpoint: &str, api_path: &str, ) -> Result { - let mut session = Self::connect(endpoint)?; + let session = Self::connect(endpoint)?; #[cfg(not(target_arch = "wasm32"))] - if let ControlTransport::Https { url, .. } = &mut session.transport { - *url = endpoint_api_url(endpoint, api_path)?; + { + let mut session = session; + if let ControlTransport::Https { url, .. } = &mut session.transport { + *url = endpoint_api_url(endpoint, api_path)?; + } + Ok(session) + } + #[cfg(target_arch = "wasm32")] + { + let _ = api_path; + Ok(session) } - Ok(session) } pub fn connect_with_timeouts( diff --git a/crates/clusterflux-coordinator/src/service/debug.rs b/crates/clusterflux-coordinator/src/service/debug.rs index efa1406..d5af181 100644 --- a/crates/clusterflux-coordinator/src/service/debug.rs +++ b/crates/clusterflux-coordinator/src/service/debug.rs @@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime { #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugBreakpointPlan { pub(super) actor: UserId, + pub(super) revision: u64, pub(super) probe_symbols: BTreeSet, pub(super) hit_epoch: Option, pub(super) hit_task: Option, @@ -85,6 +86,7 @@ impl CoordinatorService { project: String, actor_user: String, process: String, + revision: u64, probe_symbols: Vec, ) -> Result { let probe_symbols = validate_probe_symbols(probe_symbols)?; @@ -115,10 +117,28 @@ impl CoordinatorService { )) .into()); } + let key = process_control_key(&tenant, &project, &process); + if let Some(current) = self.debug_breakpoints.get(&key) { + if current.actor == actor && revision < current.revision { + return Ok(CoordinatorResponse::DebugBreakpoints { + process, + actor, + revision: current.revision, + probe_symbols: current.probe_symbols.iter().cloned().collect(), + hit_epoch: current.hit_epoch, + hit_task: current.hit_task.clone(), + hit_probe_symbol: current.hit_probe_symbol.clone(), + charged_debug_read_bytes: audit_event.charged_debug_read_bytes, + used_debug_read_bytes: audit_event.used_debug_read_bytes, + audit_event, + }); + } + } self.debug_breakpoints.insert( - process_control_key(&tenant, &project, &process), + key, DebugBreakpointPlan { actor: actor.clone(), + revision, probe_symbols: probe_symbols.iter().cloned().collect(), hit_epoch: None, hit_task: None, @@ -128,6 +148,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, + revision, probe_symbols, hit_epoch: None, hit_task: None, @@ -190,6 +211,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, + revision: plan.revision, probe_symbols: plan.probe_symbols.into_iter().collect(), hit_epoch: plan.hit_epoch, hit_task: plan.hit_task, diff --git a/crates/clusterflux-coordinator/src/service/debug_requests.rs b/crates/clusterflux-coordinator/src/service/debug_requests.rs index 4aae6c0..7447b93 100644 --- a/crates/clusterflux-coordinator/src/service/debug_requests.rs +++ b/crates/clusterflux-coordinator/src/service/debug_requests.rs @@ -33,12 +33,14 @@ impl CoordinatorService { project, actor_user, process, + revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant, project, actor_user, process, + revision, probe_symbols, ), CoordinatorRequest::InspectDebugBreakpoints { @@ -96,12 +98,14 @@ impl CoordinatorService { ), AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, + revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant.as_str().to_owned(), project.as_str().to_owned(), actor.as_str().to_owned(), process, + revision, probe_symbols, ), AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index 3818503..add2826 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -30,14 +30,14 @@ use super::{ }; #[derive(Clone)] -struct MainScope { - tenant: TenantId, - project: ProjectId, - process: ProcessId, - task_definition: TaskDefinitionId, - task_instance: TaskInstanceId, - epoch: u64, - launch_id: u64, +pub(super) struct MainScope { + pub(super) tenant: TenantId, + pub(super) project: ProjectId, + pub(super) process: ProcessId, + pub(super) task_definition: TaskDefinitionId, + pub(super) task_instance: TaskInstanceId, + pub(super) epoch: u64, + pub(super) launch_id: u64, } enum MainCommand { @@ -889,7 +889,7 @@ impl CoordinatorService { } } - fn record_coordinator_main_completion( + pub(super) fn record_coordinator_main_completion( &mut self, scope: MainScope, result: Result, @@ -1229,4 +1229,60 @@ mod tests { assert!(service.active_tasks.contains(&child_key)); assert!(!service.task_aborts.contains(&child_key)); } + + #[test] + fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() { + let mut service = CoordinatorService::new(7); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("vp-failed-main"); + let main_task = TaskInstanceId::from("ti:vp-failed-main:main"); + let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1"); + let child_node = NodeId::from("worker"); + let scope = MainScope { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + task_definition: TaskDefinitionId::from("build"), + task_instance: main_task.clone(), + epoch: 7, + launch_id: 1, + }; + service + .coordinator + .start_process(tenant.clone(), project.clone(), process.clone()); + let process_key = process_control_key(&tenant, &project, &process); + service.main_runtime.controls.insert( + process_key.clone(), + CoordinatorMainControl { + task_definition: scope.task_definition.clone(), + task_instance: main_task, + abort: Arc::new(AtomicBool::new(false)), + debug: Arc::new(WasmDebugControl::default()), + state: "running".to_owned(), + stopped_probe_symbol: None, + handles: Arc::new(Mutex::new(HashMap::new())), + launch_id: 1, + }, + ); + let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task); + service.active_tasks.insert(child_key.clone()); + service.debug_epochs.insert(process_key.clone(), 2); + + service.record_coordinator_main_completion(scope, Err("main crashed".to_owned())); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + assert!(service.active_tasks.contains(&child_key)); + assert!(service.task_aborts.contains(&child_key)); + assert!(service.process_aborts.contains(&process_key)); + assert!(!service.debug_epochs.contains_key(&process_key)); + assert!(service.task_events.iter().any(|event| { + event.process == process + && event.executor == TaskExecutor::CoordinatorMain + && event.terminal_state == TaskTerminalState::Failed + })); + } } diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index 6c989bc..3c6e516 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -2,11 +2,12 @@ use std::collections::BTreeMap; use clusterflux_core::{ AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind, - DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements, - LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, - NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, ProjectId, RequestId, - ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, - TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, UserId, VfsPath, + DataPlaneObject, DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, + EnvironmentRequirements, LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, + NodeEndpoint, NodeId, NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, + ProjectId, ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryHandle, + TaskBoundaryValue, TaskDefinitionId, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, + UserId, VfsPath, }; use serde::{Deserialize, Serialize}; @@ -338,6 +339,8 @@ pub enum CoordinatorRequest { project: String, actor_user: String, process: String, + #[serde(default)] + revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { @@ -532,10 +535,7 @@ pub enum CoordinatorRequest { impl CoordinatorRequest { pub fn validate_external_identifiers(&self) -> Result<(), String> { - let value = serde_json::to_value(self).map_err(|error| { - format!("failed to validate coordinator request identifiers: {error}") - })?; - validate_identifier_tree(&value, "request") + validate_coordinator_request(self, "request") } pub fn operation(&self) -> Result { @@ -545,104 +545,917 @@ impl CoordinatorRequest { } } -#[derive(Clone, Copy)] -enum ExternalIdentifierKind { - Agent, - Artifact, - LaunchAttempt, - Node, - Process, - Project, - Request, - TaskDefinition, - TaskInstance, - Tenant, - User, -} - -fn identifier_kind(field: &str) -> Option { - match field { - "tenant" | "target_tenant" => Some(ExternalIdentifierKind::Tenant), - "project" => Some(ExternalIdentifierKind::Project), - "actor_user" | "user" => Some(ExternalIdentifierKind::User), - "agent" | "actor_agent" => Some(ExternalIdentifierKind::Agent), - "node" | "prefer_node" | "receiver_node" => Some(ExternalIdentifierKind::Node), - "process" => Some(ExternalIdentifierKind::Process), - "task" | "parent_task" | "stopped_task" | "task_instance" => { - Some(ExternalIdentifierKind::TaskInstance) +fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Result<(), String> { + match request { + CoordinatorRequest::Ping => Ok(()), + CoordinatorRequest::Authenticated { + session_secret, + request, + } => { + validate_external_token(session_secret, &format!("{path}.session_secret"), 512)?; + validate_authenticated_request(request, &format!("{path}.request")) } - "task_definition" => Some(ExternalIdentifierKind::TaskDefinition), - "artifact" | "required_artifacts" | "artifact_locations" => { - Some(ExternalIdentifierKind::Artifact) + CoordinatorRequest::AuthStatus { + tenant, + project, + actor_user, } - "launch_attempt" => Some(ExternalIdentifierKind::LaunchAttempt), - "request_id" | "session_secret" | "transaction_id" | "polling_secret" | "transfer_id" - | "enrollment_grant" | "widget_id" | "environment_id" | "admin_nonce" => { - Some(ExternalIdentifierKind::Request) + | CoordinatorRequest::CreateProject { + tenant, + project, + actor_user, + .. + } + | CoordinatorRequest::SelectProject { + tenant, + project, + actor_user, + } => { + validate_tenant(tenant, &format!("{path}.tenant"))?; + validate_project(project, &format!("{path}.project"))?; + validate_user(actor_user, &format!("{path}.actor_user")) + } + CoordinatorRequest::ListProjects { tenant, actor_user } => { + validate_tenant(tenant, &format!("{path}.tenant"))?; + validate_user(actor_user, &format!("{path}.actor_user")) + } + CoordinatorRequest::AdminStatus { + tenant, + actor_user, + admin_nonce, + .. + } => { + validate_tenant(tenant, &format!("{path}.tenant"))?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) + } + CoordinatorRequest::SuspendTenant { + tenant, + actor_user, + target_tenant, + admin_nonce, + .. + } => { + validate_tenant(tenant, &format!("{path}.tenant"))?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_tenant(target_tenant, &format!("{path}.target_tenant"))?; + validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) + } + CoordinatorRequest::RegisterAgentPublicKey { + tenant, + project, + user, + agent, + public_key, + } + | CoordinatorRequest::RotateAgentPublicKey { + tenant, + project, + user, + agent, + public_key, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(user, &format!("{path}.user"))?; + validate_agent(agent, &format!("{path}.agent"))?; + validate_external_token(public_key, &format!("{path}.public_key"), 1024) + } + CoordinatorRequest::ListAgentPublicKeys { + tenant, + project, + user, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(user, &format!("{path}.user")) + } + CoordinatorRequest::RevokeAgentPublicKey { + tenant, + project, + user, + agent, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(user, &format!("{path}.user"))?; + validate_agent(agent, &format!("{path}.agent")) + } + CoordinatorRequest::AttachNode { + tenant, + project, + node, + public_key, + } => { + validate_tenant_project(tenant, project, path)?; + validate_node(node, &format!("{path}.node"))?; + validate_external_token(public_key, &format!("{path}.public_key"), 1024) + } + CoordinatorRequest::CreateNodeEnrollmentGrant { + tenant, + project, + actor_user, + .. + } + | CoordinatorRequest::ListNodeDescriptors { + tenant, + project, + actor_user, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user")) + } + CoordinatorRequest::ExchangeNodeEnrollmentGrant { + tenant, + project, + node, + public_key, + enrollment_grant, + } => { + validate_tenant_project(tenant, project, path)?; + validate_node(node, &format!("{path}.node"))?; + validate_external_token(public_key, &format!("{path}.public_key"), 1024)?; + validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512) + } + CoordinatorRequest::NodeHeartbeat { + node, + node_signature, + } => { + validate_node(node, &format!("{path}.node"))?; + if let Some(signature) = node_signature { + validate_node_signature(signature, &format!("{path}.node_signature"))?; + } + Ok(()) + } + CoordinatorRequest::SignedNode { + node, + node_signature, + request, + } => { + validate_node(node, &format!("{path}.node"))?; + validate_node_signature(node_signature, &format!("{path}.node_signature"))?; + validate_coordinator_request(request, &format!("{path}.request")) + } + CoordinatorRequest::ReportNodeCapabilities { + tenant, + project, + node, + artifact_locations, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_node(node, &format!("{path}.node"))?; + validate_artifact_array(artifact_locations, &format!("{path}.artifact_locations")) + } + CoordinatorRequest::RevokeNodeCredential { + tenant, + project, + actor_user, + node, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_node(node, &format!("{path}.node")) + } + CoordinatorRequest::ScheduleTask { + tenant, + project, + required_artifacts, + prefer_node, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; + validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) + } + CoordinatorRequest::LaunchTask { + tenant, + project, + actor_user, + actor_agent, + agent_signature, + task_spec, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; + validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; + if let Some(signature) = agent_signature { + validate_agent_signature(signature, &format!("{path}.agent_signature"))?; + } + validate_task_spec(task_spec, &format!("{path}.task_spec")) + } + CoordinatorRequest::LaunchChildTask { + tenant, + project, + process, + node, + parent_task, + task_spec, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_process(process, &format!("{path}.process"))?; + validate_node(node, &format!("{path}.node"))?; + validate_task_instance(parent_task, &format!("{path}.parent_task"))?; + validate_task_spec(task_spec, &format!("{path}.task_spec")) + } + CoordinatorRequest::JoinChildTask { + tenant, + project, + process, + node, + parent_task, + task, + } => { + validate_tenant_project(tenant, project, path)?; + validate_process(process, &format!("{path}.process"))?; + validate_node(node, &format!("{path}.node"))?; + validate_task_instance(parent_task, &format!("{path}.parent_task"))?; + validate_task_instance(task, &format!("{path}.task")) + } + CoordinatorRequest::PollTaskAssignment { + tenant, + project, + node, + } + | CoordinatorRequest::PollArtifactTransfer { + tenant, + project, + node, + } + | CoordinatorRequest::CompleteSourcePreparation { + tenant, + project, + node, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_node(node, &format!("{path}.node")) + } + CoordinatorRequest::UploadArtifactTransferChunk { + tenant, + project, + node, + transfer_id, + artifact, + .. + } + | CoordinatorRequest::FailArtifactTransfer { + tenant, + project, + node, + transfer_id, + artifact, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_node(node, &format!("{path}.node"))?; + validate_external_token(transfer_id, &format!("{path}.transfer_id"), 256)?; + validate_artifact(artifact, &format!("{path}.artifact")) + } + CoordinatorRequest::RequestRendezvous { + scope, + source, + destination, + .. + } => { + validate_data_plane_scope(scope, &format!("{path}.scope"))?; + validate_node_endpoint(source, &format!("{path}.source"))?; + validate_node_endpoint(destination, &format!("{path}.destination")) + } + CoordinatorRequest::RequestSourcePreparation { + tenant, project, .. + } => validate_tenant_project(tenant, project, path), + CoordinatorRequest::StartProcess { + tenant, + project, + actor_user, + actor_agent, + agent_signature, + process, + launch_attempt, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; + validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; + if let Some(signature) = agent_signature { + validate_agent_signature(signature, &format!("{path}.agent_signature"))?; + } + validate_process(process, &format!("{path}.process"))?; + validate_optional_launch_attempt( + launch_attempt.as_deref(), + &format!("{path}.launch_attempt"), + ) + } + CoordinatorRequest::ReconnectNode { node, process, .. } => { + validate_node(node, &format!("{path}.node"))?; + validate_process(process, &format!("{path}.process")) + } + CoordinatorRequest::CancelTask { + tenant, + project, + process, + node, + task, + } + | CoordinatorRequest::PollTaskControl { + tenant, + project, + process, + node, + task, + } + | CoordinatorRequest::PollDebugCommand { + tenant, + project, + process, + node, + task, + } + | CoordinatorRequest::ReportDebugProbeHit { + tenant, + project, + process, + node, + task, + .. + } + | CoordinatorRequest::ReportTaskLog { + tenant, + project, + process, + node, + task, + .. + } + | CoordinatorRequest::ReportVfsMetadata { + tenant, + project, + process, + node, + task, + .. + } + | CoordinatorRequest::TaskCompleted { + tenant, + project, + process, + node, + task, + .. + } + | CoordinatorRequest::ReportDebugState { + tenant, + project, + process, + node, + task, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_process(process, &format!("{path}.process"))?; + validate_node(node, &format!("{path}.node"))?; + validate_task_instance(task, &format!("{path}.task")) + } + CoordinatorRequest::CancelProcess { + tenant, + project, + actor_user, + process, + } + | CoordinatorRequest::DebugAttach { + tenant, + project, + actor_user, + process, + } + | CoordinatorRequest::SetDebugBreakpoints { + tenant, + project, + actor_user, + process, + .. + } + | CoordinatorRequest::InspectDebugBreakpoints { + tenant, + project, + actor_user, + process, + } + | CoordinatorRequest::ResumeDebugEpoch { + tenant, + project, + actor_user, + process, + .. + } + | CoordinatorRequest::InspectDebugEpoch { + tenant, + project, + actor_user, + process, + .. + } + | CoordinatorRequest::ListTaskSnapshots { + tenant, + project, + actor_user, + process, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process")) + } + CoordinatorRequest::AbortProcess { + tenant, + project, + actor_user, + process, + launch_attempt, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process"))?; + validate_optional_launch_attempt( + launch_attempt.as_deref(), + &format!("{path}.launch_attempt"), + ) + } + CoordinatorRequest::ListProcesses { + tenant, + project, + actor_user, + } + | CoordinatorRequest::QuotaStatus { + tenant, + project, + actor_user, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user")) + } + CoordinatorRequest::RestartTask { + tenant, + project, + actor_user, + process, + task, + .. + } + | CoordinatorRequest::ResolveTaskFailure { + tenant, + project, + actor_user, + process, + task, + .. + } + | CoordinatorRequest::JoinTask { + tenant, + project, + actor_user, + process, + task, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process"))?; + validate_task_instance(task, &format!("{path}.task")) + } + CoordinatorRequest::CreateDebugEpoch { + tenant, + project, + actor_user, + process, + stopped_task, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process"))?; + validate_task_instance(stopped_task, &format!("{path}.stopped_task")) + } + CoordinatorRequest::ListTaskEvents { + tenant, + project, + actor_user, + process, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_process(process.as_deref(), &format!("{path}.process")) + } + CoordinatorRequest::RenderOperatorPanel { + tenant, + project, + process, + actor_user, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_process(process, &format!("{path}.process"))?; + validate_user(actor_user, &format!("{path}.actor_user")) + } + CoordinatorRequest::SubmitPanelEvent { + tenant, + project, + process, + widget_id, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_process(process, &format!("{path}.process"))?; + validate_external_token(widget_id, &format!("{path}.widget_id"), 256) + } + CoordinatorRequest::CreateArtifactDownloadLink { + tenant, + project, + actor_user, + artifact, + .. + } + | CoordinatorRequest::OpenArtifactDownloadStream { + tenant, + project, + actor_user, + artifact, + .. + } + | CoordinatorRequest::RevokeArtifactDownloadLink { + tenant, + project, + actor_user, + artifact, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_artifact(artifact, &format!("{path}.artifact")) + } + CoordinatorRequest::ExportArtifactToNode { + tenant, + project, + actor_user, + artifact, + receiver_node, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_artifact(artifact, &format!("{path}.artifact"))?; + validate_node(receiver_node, &format!("{path}.receiver_node")) } - _ => None, } } -fn validate_identifier_tree(value: &serde_json::Value, path: &str) -> Result<(), String> { - match value { - serde_json::Value::Object(fields) => { - for (field, value) in fields { - let field_path = format!("{path}.{field}"); - if let Some(kind) = identifier_kind(field) { - validate_identifier_value(kind, value, &field_path)?; - } - validate_identifier_tree(value, &field_path)?; - } +fn validate_authenticated_request( + request: &AuthenticatedCoordinatorRequest, + path: &str, +) -> Result<(), String> { + match request { + AuthenticatedCoordinatorRequest::AuthStatus + | AuthenticatedCoordinatorRequest::RevokeCliSession + | AuthenticatedCoordinatorRequest::ListProjects + | AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } + | AuthenticatedCoordinatorRequest::ListNodeDescriptors + | AuthenticatedCoordinatorRequest::ListProcesses + | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), + AuthenticatedCoordinatorRequest::CreateProject { project, .. } + | AuthenticatedCoordinatorRequest::SelectProject { project } => { + validate_project(project, &format!("{path}.project")) } - serde_json::Value::Array(values) => { - for (index, value) in values.iter().enumerate() { - validate_identifier_tree(value, &format!("{path}[{index}]"))?; - } + AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { + agent, public_key, .. } - _ => {} + | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { + agent, public_key, .. + } => { + validate_agent(agent, &format!("{path}.agent"))?; + validate_external_token(public_key, &format!("{path}.public_key"), 1024) + } + AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Ok(()), + AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => { + validate_agent(agent, &format!("{path}.agent")) + } + AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => { + validate_node(node, &format!("{path}.node")) + } + AuthenticatedCoordinatorRequest::StartProcess { + process, + launch_attempt, + .. + } + | AuthenticatedCoordinatorRequest::AbortProcess { + process, + launch_attempt, + } => { + validate_process(process, &format!("{path}.process"))?; + validate_optional_launch_attempt( + launch_attempt.as_deref(), + &format!("{path}.launch_attempt"), + ) + } + AuthenticatedCoordinatorRequest::ScheduleTask { + required_artifacts, + prefer_node, + .. + } => { + validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; + validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) + } + AuthenticatedCoordinatorRequest::LaunchTask { task_spec, .. } => { + validate_task_spec(task_spec, &format!("{path}.task_spec")) + } + AuthenticatedCoordinatorRequest::CancelProcess { process } + | AuthenticatedCoordinatorRequest::DebugAttach { process } + | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, .. } + | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } + | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, .. } + | AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, .. } + | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { + validate_process(process, &format!("{path}.process")) + } + AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } + | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } + | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { + validate_process(process, &format!("{path}.process"))?; + validate_task_instance(task, &format!("{path}.task")) + } + AuthenticatedCoordinatorRequest::CreateDebugEpoch { + process, + stopped_task, + .. + } => { + validate_process(process, &format!("{path}.process"))?; + validate_task_instance(stopped_task, &format!("{path}.stopped_task")) + } + AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { + validate_optional_process(process.as_deref(), &format!("{path}.process")) + } + AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } + | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } + | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } => { + validate_artifact(artifact, &format!("{path}.artifact")) + } + AuthenticatedCoordinatorRequest::ExportArtifactToNode { + artifact, + receiver_node, + .. + } => { + validate_artifact(artifact, &format!("{path}.artifact"))?; + validate_node(receiver_node, &format!("{path}.receiver_node")) + } + } +} + +fn validate_task_spec(task_spec: &TaskSpec, path: &str) -> Result<(), String> { + validate_existing_id( + &task_spec.tenant, + &format!("{path}.tenant"), + TenantId::try_new, + )?; + validate_existing_id( + &task_spec.project, + &format!("{path}.project"), + ProjectId::try_new, + )?; + validate_existing_id( + &task_spec.process, + &format!("{path}.process"), + ProcessId::try_new, + )?; + validate_existing_id( + &task_spec.task_definition, + &format!("{path}.task_definition"), + TaskDefinitionId::try_new, + )?; + validate_existing_id( + &task_spec.task_instance, + &format!("{path}.task_instance"), + TaskInstanceId::try_new, + )?; + if let Some(environment_id) = &task_spec.environment_id { + validate_external_token(environment_id, &format!("{path}.environment_id"), 128)?; + } + for (index, artifact) in task_spec.required_artifacts.iter().enumerate() { + validate_existing_id( + artifact, + &format!("{path}.required_artifacts[{index}]"), + ArtifactId::try_new, + )?; + } + for (argument_index, argument) in task_spec.args.iter().enumerate() { + validate_task_boundary_value(argument, &format!("{path}.args[{argument_index}]"))?; } Ok(()) } -fn validate_identifier_value( - kind: ExternalIdentifierKind, - value: &serde_json::Value, - path: &str, -) -> Result<(), String> { - if value.is_null() { - return Ok(()); - } - if let Some(values) = value.as_array() { - for (index, value) in values.iter().enumerate() { - validate_identifier_value(kind, value, &format!("{path}[{index}]"))?; +fn validate_task_boundary_value(value: &TaskBoundaryValue, path: &str) -> Result<(), String> { + match value { + TaskBoundaryValue::Artifact(artifact) => validate_existing_id( + &artifact.id, + &format!("{path}.artifact.id"), + ArtifactId::try_new, + ), + TaskBoundaryValue::Structured(structured) => { + for (index, handle) in structured.handles.iter().enumerate() { + if let TaskBoundaryHandle::Artifact(artifact) = handle { + validate_existing_id( + &artifact.id, + &format!("{path}.handles[{index}].artifact.id"), + ArtifactId::try_new, + )?; + } + } + Ok(()) } - return Ok(()); + TaskBoundaryValue::SmallJson(_) + | TaskBoundaryValue::SourceSnapshot(_) + | TaskBoundaryValue::Blob(_) + | TaskBoundaryValue::VfsManifest(_) => Ok(()), } - let value = value - .as_str() - .ok_or_else(|| format!("external identifier {path} must be a string"))? - .to_owned(); - let result = match kind { - ExternalIdentifierKind::Agent => AgentId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Artifact => ArtifactId::try_new(value).map(|_| ()), - ExternalIdentifierKind::LaunchAttempt => LaunchAttemptId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Node => NodeId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Process => ProcessId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Project => ProjectId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Request => RequestId::try_new(value).map(|_| ()), - ExternalIdentifierKind::TaskDefinition => TaskDefinitionId::try_new(value).map(|_| ()), - ExternalIdentifierKind::TaskInstance => TaskInstanceId::try_new(value).map(|_| ()), - ExternalIdentifierKind::Tenant => TenantId::try_new(value).map(|_| ()), - ExternalIdentifierKind::User => UserId::try_new(value).map(|_| ()), - }; - result.map_err(|error| format!("malformed external identifier {path}: {error}")) +} + +fn validate_data_plane_scope(scope: &DataPlaneScope, path: &str) -> Result<(), String> { + validate_existing_id(&scope.tenant, &format!("{path}.tenant"), TenantId::try_new)?; + validate_existing_id( + &scope.project, + &format!("{path}.project"), + ProjectId::try_new, + )?; + validate_existing_id( + &scope.process, + &format!("{path}.process"), + ProcessId::try_new, + )?; + if let DataPlaneObject::Artifact(artifact) = &scope.object { + validate_existing_id( + artifact, + &format!("{path}.object.artifact"), + ArtifactId::try_new, + )?; + } + validate_external_token( + &scope.authorization_subject, + &format!("{path}.authorization_subject"), + 512, + ) +} + +fn validate_node_endpoint(endpoint: &NodeEndpoint, path: &str) -> Result<(), String> { + validate_existing_id(&endpoint.node, &format!("{path}.node"), NodeId::try_new)?; + validate_external_token( + &endpoint.advertised_addr, + &format!("{path}.advertised_addr"), + 512, + ) +} + +fn validate_node_signature(signature: &NodeSignedRequest, path: &str) -> Result<(), String> { + validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; + validate_external_token(&signature.signature, &format!("{path}.signature"), 512) +} + +fn validate_agent_signature(signature: &AgentSignedRequest, path: &str) -> Result<(), String> { + validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; + validate_external_token(&signature.signature, &format!("{path}.signature"), 512) +} + +fn validate_tenant_project(tenant: &str, project: &str, path: &str) -> Result<(), String> { + validate_tenant(tenant, &format!("{path}.tenant"))?; + validate_project(project, &format!("{path}.project")) +} + +fn validate_artifact_array(values: &[String], path: &str) -> Result<(), String> { + for (index, value) in values.iter().enumerate() { + validate_artifact(value, &format!("{path}[{index}]"))?; + } + Ok(()) +} + +fn validate_optional_user(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_user(value, path)) +} + +fn validate_optional_agent(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_agent(value, path)) +} + +fn validate_optional_node(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_node(value, path)) +} + +fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_process(value, path)) +} + +fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) +} + +fn validate_tenant(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, TenantId::try_new) +} + +fn validate_project(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, ProjectId::try_new) +} + +fn validate_user(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, UserId::try_new) +} + +fn validate_agent(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, AgentId::try_new) +} + +fn validate_node(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, NodeId::try_new) +} + +fn validate_process(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, ProcessId::try_new) +} + +fn validate_task_instance(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, TaskInstanceId::try_new) +} + +fn validate_artifact(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, ArtifactId::try_new) +} + +fn validate_launch_attempt(value: &str, path: &str) -> Result<(), String> { + validate_external_id(value, path, LaunchAttemptId::try_new) +} + +fn validate_external_id( + value: &str, + path: &str, + parser: impl FnOnce(String) -> Result, +) -> Result<(), String> +where + E: std::fmt::Display, +{ + parser(value.to_owned()) + .map(|_| ()) + .map_err(|error| format!("malformed external identifier {path}: {error}")) +} + +fn validate_existing_id( + value: &T, + path: &str, + parser: impl FnOnce(String) -> Result, +) -> Result<(), String> +where + T: std::fmt::Display, + E: std::fmt::Display, +{ + validate_external_id(&value.to_string(), path, parser) +} + +fn validate_external_token(value: &str, path: &str, max_bytes: usize) -> Result<(), String> { + clusterflux_core::validate_opaque_token(value, max_bytes) + .map_err(|error| format!("malformed external token {path}: {error}")) } #[cfg(test)] mod external_identifier_tests { use super::*; + fn valid_task_spec() -> TaskSpec { + TaskSpec { + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), + task_definition: TaskDefinitionId::from("compile"), + task_instance: TaskInstanceId::from("compile-1"), + dispatch: clusterflux_core::TaskDispatch::CoordinatorNodeWasm { + export: Some("compile".to_owned()), + abi: clusterflux_core::WasmExportAbi::TaskV1, + }, + environment_id: Some("linux-rootless".to_owned()), + environment: None, + environment_digest: None, + required_capabilities: Default::default(), + dependency_cache: None, + source_snapshot: None, + required_artifacts: vec![ArtifactId::from("input-artifact")], + args: Vec::new(), + vfs_epoch: 1, + failure_policy: Default::default(), + bundle_digest: None, + } + } + + fn validation_error(value: serde_json::Value) -> String { + match serde_json::from_value::(value) { + Ok(request) => request + .validate_external_identifiers() + .expect_err("request should contain one malformed external identifier"), + Err(error) => error.to_string(), + } + } + #[test] fn nested_authenticated_identifiers_are_validated() { let request = CoordinatorRequest::Authenticated { @@ -655,6 +1468,174 @@ mod external_identifier_tests { let error = request.validate_external_identifiers().unwrap_err(); assert!(error.contains("request.request.process")); } + + #[test] + fn opaque_secrets_are_bounded_as_tokens_instead_of_object_ids() { + let request = CoordinatorRequest::Authenticated { + session_secret: "opaque secret/+==".to_owned(), + request: AuthenticatedCoordinatorRequest::AuthStatus, + }; + request.validate_external_identifiers().unwrap(); + + let mut request = request; + let CoordinatorRequest::Authenticated { session_secret, .. } = &mut request else { + unreachable!() + }; + *session_secret = "bad\0secret".to_owned(); + let error = request.validate_external_identifiers().unwrap_err(); + assert!(error.contains("malformed external token request.session_secret")); + } + + #[test] + fn real_protocol_variants_reject_exactly_one_malformed_nested_identifier() { + let launch = CoordinatorRequest::LaunchTask { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: Some("user".to_owned()), + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + task_spec: valid_task_spec(), + wait_for_node: false, + artifact_path: "/vfs/artifacts/output".to_owned(), + wasm_module_base64: "AGFzbQEAAAA=".to_owned(), + }; + + let mut malformed_definition = serde_json::to_value(&launch).unwrap(); + malformed_definition["task_spec"]["task_definition"] = + serde_json::Value::String("bad task definition!".to_owned()); + let error = validation_error(malformed_definition); + assert!(error.contains("TaskDefinitionId is invalid")); + + let mut malformed_artifact = serde_json::to_value(&launch).unwrap(); + malformed_artifact["task_spec"]["required_artifacts"][0] = + serde_json::Value::String("bad artifact!".to_owned()); + let error = validation_error(malformed_artifact); + assert!(error.contains("ArtifactId is invalid")); + + let rendezvous = CoordinatorRequest::RequestRendezvous { + scope: DataPlaneScope { + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), + object: DataPlaneObject::Artifact(ArtifactId::from("artifact")), + authorization_subject: "node-a-to-node-b".to_owned(), + }, + source: NodeEndpoint { + node: NodeId::from("node-a"), + advertised_addr: "node-a.invalid:4433".to_owned(), + public_key_fingerprint: Digest::sha256("node-a"), + }, + destination: NodeEndpoint { + node: NodeId::from("node-b"), + advertised_addr: "node-b.invalid:4433".to_owned(), + public_key_fingerprint: Digest::sha256("node-b"), + }, + direct_connectivity: true, + failure_reason: String::new(), + }; + let mut malformed_endpoint = serde_json::to_value(rendezvous).unwrap(); + malformed_endpoint["destination"]["node"] = + serde_json::Value::String("bad node!".to_owned()); + let error = validation_error(malformed_endpoint); + assert!(error.contains("NodeId is invalid")); + } + + #[test] + fn signed_and_authenticated_real_variants_validate_scoped_ids_and_tokens() { + let cases = [ + CoordinatorRequest::Authenticated { + session_secret: "session-secret".to_owned(), + request: AuthenticatedCoordinatorRequest::AbortProcess { + process: "bad process!".to_owned(), + launch_attempt: Some("attempt".to_owned()), + }, + }, + CoordinatorRequest::StartProcess { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: None, + actor_agent: Some("bad agent!".to_owned()), + agent_public_key_fingerprint: Some(Digest::sha256("agent")), + agent_signature: Some(AgentSignedRequest { + nonce: "agent-nonce".to_owned(), + issued_at_epoch_seconds: 1, + signature: "ed25519:syntactically-bounded".to_owned(), + }), + process: "process".to_owned(), + launch_attempt: Some("attempt".to_owned()), + restart: false, + }, + CoordinatorRequest::NodeHeartbeat { + node: "bad node!".to_owned(), + node_signature: Some(NodeSignedRequest { + nonce: "node-nonce".to_owned(), + issued_at_epoch_seconds: 1, + signature: "ed25519:syntactically-bounded".to_owned(), + }), + }, + CoordinatorRequest::SignedNode { + node: "node".to_owned(), + node_signature: NodeSignedRequest { + nonce: "node-nonce".to_owned(), + issued_at_epoch_seconds: 1, + signature: "ed25519:syntactically-bounded".to_owned(), + }, + request: Box::new(CoordinatorRequest::PollTaskAssignment { + tenant: "tenant".to_owned(), + project: "bad project!".to_owned(), + node: "node".to_owned(), + }), + }, + CoordinatorRequest::AbortProcess { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: "user".to_owned(), + process: "process".to_owned(), + launch_attempt: Some("bad attempt!".to_owned()), + }, + ]; + + for request in cases { + let error = request.validate_external_identifiers().unwrap_err(); + assert!( + error.contains("malformed external identifier"), + "unexpected validation error for {request:?}: {error}" + ); + } + } + + #[test] + fn signed_request_nonces_are_validated_as_opaque_tokens() { + let agent_request = CoordinatorRequest::StartProcess { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: None, + actor_agent: Some("agent".to_owned()), + agent_public_key_fingerprint: Some(Digest::sha256("agent")), + agent_signature: Some(AgentSignedRequest { + nonce: String::new(), + issued_at_epoch_seconds: 1, + signature: "ed25519:syntactically-bounded".to_owned(), + }), + process: "process".to_owned(), + launch_attempt: Some("attempt".to_owned()), + restart: false, + }; + let error = agent_request.validate_external_identifiers().unwrap_err(); + assert!(error.contains("malformed external token request.agent_signature.nonce")); + + let node_request = CoordinatorRequest::NodeHeartbeat { + node: "node".to_owned(), + node_signature: Some(NodeSignedRequest { + nonce: String::new(), + issued_at_epoch_seconds: 1, + signature: "ed25519:syntactically-bounded".to_owned(), + }), + }; + let error = node_request.validate_external_identifiers().unwrap_err(); + assert!(error.contains("malformed external token request.node_signature.nonce")); + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -740,6 +1721,8 @@ pub enum AuthenticatedCoordinatorRequest { }, SetDebugBreakpoints { process: String, + #[serde(default)] + revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { diff --git a/crates/clusterflux-coordinator/src/service/protocol/responses.rs b/crates/clusterflux-coordinator/src/service/protocol/responses.rs index 54655b7..dbb3874 100644 --- a/crates/clusterflux-coordinator/src/service/protocol/responses.rs +++ b/crates/clusterflux-coordinator/src/service/protocol/responses.rs @@ -430,6 +430,7 @@ pub enum CoordinatorResponse { DebugBreakpoints { process: ProcessId, actor: UserId, + revision: u64, probe_symbols: Vec, hit_epoch: Option, hit_task: Option, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index e6e3878..10e18cd 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -19,7 +19,9 @@ impl CoordinatorService { )) })?; let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload); - let signed_node = NodeId::new(signed_node); + let signed_node = NodeId::try_new(signed_node).map_err(|error| { + CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}")) + })?; if request_node != signed_node { return Err(CoordinatorError::Unauthorized( "signed node request node does not match the wrapped request node".to_owned(), @@ -144,6 +146,19 @@ impl CoordinatorService { provider, source_snapshot, ), + CoordinatorRequest::RequestRendezvous { + scope, + source, + destination, + direct_connectivity, + failure_reason, + } => self.handle_request_rendezvous( + scope, + source, + destination, + direct_connectivity, + failure_reason, + ), CoordinatorRequest::ReconnectNode { node, process, @@ -322,6 +337,7 @@ fn signed_node_request_kind( CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"), CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"), CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"), + CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"), CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"), CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"), CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"), @@ -356,7 +372,14 @@ fn signed_node_request_node( | CoordinatorRequest::ReportDebugProbeHit { node, .. } | CoordinatorRequest::ReportTaskLog { node, .. } | CoordinatorRequest::ReportVfsMetadata { node, .. } - | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())), + | CoordinatorRequest::TaskCompleted { node, .. } => { + NodeId::try_new(node.clone()).map_err(|error| { + CoordinatorServiceError::Protocol(format!( + "invalid wrapped node identifier: {error}" + )) + }) + } + CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()), _ => Err(CoordinatorError::Unauthorized( "signed_node envelope only accepts node-originated coordinator requests".to_owned(), ) diff --git a/crates/clusterflux-coordinator/src/service/tcp.rs b/crates/clusterflux-coordinator/src/service/tcp.rs index d519f34..0b42969 100644 --- a/crates/clusterflux-coordinator/src/service/tcp.rs +++ b/crates/clusterflux-coordinator/src/service/tcp.rs @@ -189,6 +189,11 @@ fn authorize_client_request( #[cfg(test)] mod transport_boundary_tests { + use std::io::{BufRead as _, BufReader}; + + use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId}; + use serde_json::json; + use super::*; #[test] @@ -197,4 +202,96 @@ mod transport_boundary_tests { let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err(); assert!(error.to_string().contains("restricted to loopback")); } + + #[test] + fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() { + let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); + let mut coordinator = CoordinatorService::new(11); + coordinator + .issue_cli_session( + TenantId::from("tenant"), + ProjectId::from("project"), + UserId::from("user"), + "healthy-session", + None, + ) + .unwrap(); + let shared = Arc::new(Mutex::new(coordinator)); + let server_shared = Arc::clone(&shared); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap(); + }); + + let mut stream = TcpStream::connect(addr).unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + for (index, malformed_process) in [ + String::new(), + " ".to_owned(), + "bad\0process".to_owned(), + "bad process!".to_owned(), + "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1), + ] + .into_iter() + .enumerate() + { + let malformed = coordinator_wire_request( + format!("malformed-{index}"), + json!({ + "type": "authenticated", + "session_secret": "healthy-session", + "request": { + "type": "abort_process", + "process": malformed_process, + "launch_attempt": "valid-attempt" + } + }), + ); + serde_json::to_writer(&mut stream, &malformed).unwrap(); + stream.write_all(b"\n").unwrap(); + stream.flush().unwrap(); + + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let CoordinatorResponse::Error { message } = + serde_json::from_str::(&line).unwrap() + else { + panic!("malformed identifier request unexpectedly succeeded"); + }; + assert!( + message.contains("malformed external identifier") + && message.contains("request.request.process"), + "unexpected malformed identifier response: {message}" + ); + + let valid = coordinator_wire_request( + format!("healthy-{index}"), + json!({ + "type": "authenticated", + "session_secret": "healthy-session", + "request": { "type": "auth_status" } + }), + ); + serde_json::to_writer(&mut stream, &valid).unwrap(); + stream.write_all(b"\n").unwrap(); + stream.flush().unwrap(); + + line.clear(); + reader.read_line(&mut line).unwrap(); + assert!( + matches!( + serde_json::from_str::(&line).unwrap(), + CoordinatorResponse::AuthStatus { + authenticated: true, + .. + } + ), + "valid authenticated traffic failed after malformed request {index}" + ); + } + + stream.shutdown(std::net::Shutdown::Both).unwrap(); + server.join().unwrap(); + assert!(!shared.is_poisoned()); + } } diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index 43215d7..e6e788e 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -12,8 +12,8 @@ use clusterflux_core::{ AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability, DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements, LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits, - SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec, - VfsPath, WasmExportAbi, + SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, + TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult, }; use serde_json::json; @@ -521,6 +521,9 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { CoordinatorRequest::CompleteSourcePreparation { node, .. } => { (node.clone(), "complete_source_preparation") } + CoordinatorRequest::RequestRendezvous { source, .. } => { + (source.node.to_string(), "request_rendezvous") + } CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"), CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"), CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"), @@ -2752,6 +2755,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { }, ); let CoordinatorResponse::DebugBreakpoints { + revision, probe_symbols, hit_epoch, .. @@ -2761,15 +2765,37 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), + revision: 1, probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()], }) .unwrap() else { panic!("expected debug breakpoints response"); }; + assert_eq!(revision, 1); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); assert_eq!(hit_epoch, None); + let CoordinatorResponse::DebugBreakpoints { + revision, + probe_symbols, + .. + } = service + .handle_request(CoordinatorRequest::SetDebugBreakpoints { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: "user".to_owned(), + process: "process".to_owned(), + revision: 0, + probe_symbols: vec!["clusterflux.probe.stale".to_owned()], + }) + .unwrap() + else { + panic!("expected stale breakpoint response"); + }; + assert_eq!(revision, 1); + assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); + let CoordinatorResponse::DebugProbeHit { breakpoint_matched, debug_epoch, @@ -3583,6 +3609,184 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { assert!(!service.main_runtime.controls.contains_key(&process_key)); } +#[test] +fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() { + let mut service = CoordinatorService::new(31); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("process-main-before-child"); + let child = TaskInstanceId::from("child-active"); + let process_key = process_control_key(&tenant, &project, &process); + + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: tenant.to_string(), + project: project.to_string(), + node: "worker".to_owned(), + public_key: test_node_public_key("worker"), + }) + .unwrap(); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: process.to_string(), + restart: false, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + node: "worker".to_owned(), + process: process.to_string(), + epoch: 31, + }) + .unwrap(); + register_test_task_assignment( + &mut service, + tenant.as_str(), + project.as_str(), + process.as_str(), + "worker", + "child-definition", + child.as_str(), + 31, + ); + + let main = TaskInstanceId::from("main-instance"); + service.main_runtime.controls.insert( + process_key.clone(), + super::main_runtime::CoordinatorMainControl { + task_definition: TaskDefinitionId::from("build"), + task_instance: main.clone(), + abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), + state: "running".to_owned(), + stopped_probe_symbol: None, + handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + launch_id: 1, + }, + ); + service.debug_epochs.insert(process_key.clone(), 9); + + service.record_coordinator_main_completion( + super::main_runtime::MainScope { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + task_definition: TaskDefinitionId::from("build"), + task_instance: main, + epoch: 31, + launch_id: 1, + }, + Ok(WasmTaskResult::completed( + TaskInstanceId::from("main-instance"), + TaskBoundaryValue::SmallJson(json!("main completed")), + )), + ); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_some()); + assert!(service.debug_epochs.contains_key(&process_key)); + assert!(service + .active_tasks + .iter() + .any(|(_, _, retained_process, _, task)| { + retained_process == &process && task == &child + })); + let blocked_next = service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "process-too-early".to_owned(), + restart: false, + }) + .unwrap_err(); + assert!(blocked_next + .to_string() + .contains("already has active virtual process")); + + let artifact_bytes = b"child artifact survives terminal cleanup"; + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: tenant.to_string(), + project: project.to_string(), + process: process.to_string(), + node: "worker".to_owned(), + task: child.to_string(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: artifact_bytes.len() as u64, + stderr_bytes: 0, + stdout_tail: "child completed".to_owned(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: Some("/vfs/artifacts/child-output".to_owned()), + artifact_digest: Some(Digest::sha256(artifact_bytes)), + artifact_size_bytes: Some(artifact_bytes.len() as u64), + result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))), + }) + .unwrap(); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + assert!(!service.debug_epochs.contains_key(&process_key)); + let CoordinatorResponse::TaskEvents { events } = service + .handle_request(CoordinatorRequest::ListTaskEvents { + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: "user".to_owned(), + process: Some(process.to_string()), + }) + .unwrap() + else { + panic!("expected retained task events"); + }; + assert_eq!(events.len(), 2); + assert!(events.iter().any(|event| { + event.executor == TaskExecutor::CoordinatorMain + && event.terminal_state == TaskTerminalState::Completed + })); + assert!(events.iter().any(|event| { + event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes)) + })); + let metadata = service + .artifact_registry + .metadata(&ArtifactId::from("child-output")) + .expect("artifact metadata must survive terminal cleanup"); + assert_eq!(metadata.process, process); + assert_eq!(metadata.digest, Digest::sha256(artifact_bytes)); + + let next = service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "process-after-cleanup".to_owned(), + restart: false, + }) + .unwrap(); + assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. })); +} + #[test] fn quiescent_cooperative_cancel_releases_slot_immediately() { let mut service = CoordinatorService::new(17); @@ -6084,6 +6288,35 @@ fn service_meters_rendezvous_before_direct_transfer_plan() { assert!(quota.to_string().contains("RendezvousAttempt")); } +#[test] +fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() { + let mut service = CoordinatorService::new(7); + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: "node-a".to_owned(), + public_key: test_node_public_key("node-a"), + }) + .unwrap(); + + let response = service + .handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous { + scope: data_plane_scope("project"), + source: endpoint("node-a"), + destination: endpoint("node-b"), + direct_connectivity: true, + failure_reason: String::new(), + }) + .unwrap(); + + let CoordinatorResponse::RendezvousPlan { plan, .. } = response else { + panic!("expected signed rendezvous plan"); + }; + assert_eq!(plan.source.node, NodeId::from("node-a")); + assert_eq!(plan.destination.node, NodeId::from("node-b")); +} + #[test] fn service_rejects_task_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); diff --git a/crates/clusterflux-core/src/auth.rs b/crates/clusterflux-core/src/auth.rs index 61e99fb..51686a2 100644 --- a/crates/clusterflux-core/src/auth.rs +++ b/crates/clusterflux-core/src/auth.rs @@ -364,7 +364,13 @@ pub fn agent_workflow_request_scope_from_payload( .get("task_instance") .and_then(Value::as_str) .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?; - (process, Some(TaskInstanceId::from(task))) + ( + process, + Some( + TaskInstanceId::try_new(task) + .map_err(|error| format!("malformed launch_task task instance: {error}"))?, + ), + ) } _ => { return Err(format!( @@ -373,10 +379,13 @@ pub fn agent_workflow_request_scope_from_payload( } }; AgentWorkflowRequestScope::new( - TenantId::from(tenant), - ProjectId::from(project), + TenantId::try_new(tenant) + .map_err(|error| format!("malformed agent workflow tenant: {error}"))?, + ProjectId::try_new(project) + .map_err(|error| format!("malformed agent workflow project: {error}"))?, request_kind, - ProcessId::from(process), + ProcessId::try_new(process) + .map_err(|error| format!("malformed agent workflow process: {error}"))?, task, ) } diff --git a/crates/clusterflux-core/src/ids.rs b/crates/clusterflux-core/src/ids.rs index eb35777..2e2744d 100644 --- a/crates/clusterflux-core/src/ids.rs +++ b/crates/clusterflux-core/src/ids.rs @@ -1,7 +1,35 @@ +#[cfg(not(target_arch = "wasm32"))] +use serde::{de::Error as _, Deserializer}; use serde::{Deserialize, Serialize}; pub const MAX_EXTERNAL_ID_BYTES: usize = 255; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OpaqueTokenError { + reason: String, +} + +impl std::fmt::Display for OpaqueTokenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.reason) + } +} + +impl std::error::Error for OpaqueTokenError {} + +pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> { + let reason = if value.trim().is_empty() { + Some("value must not be empty or whitespace-only".to_owned()) + } else if value.len() > max_bytes { + Some(format!("value exceeds the {max_bytes}-byte limit")) + } else if value.chars().any(char::is_control) { + Some("control characters are forbidden".to_owned()) + } else { + None + }; + reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason })) +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct IdParseError { id_type: &'static str, @@ -55,7 +83,8 @@ fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> { macro_rules! id_type { ($name:ident) => { - #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] + #[cfg_attr(target_arch = "wasm32", derive(Deserialize))] + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] pub struct $name(String); impl $name { @@ -81,6 +110,17 @@ macro_rules! id_type { } } + #[cfg(not(target_arch = "wasm32"))] + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::try_new(value).map_err(D::Error::custom) + } + } + impl std::fmt::Display for $name { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.0) @@ -134,4 +174,38 @@ mod tests { assert_hostile_values(TenantId::try_new); assert_hostile_values(UserId::try_new); } + + #[test] + fn every_identifier_type_validates_during_deserialization() { + macro_rules! assert_deserialization { + ($id:ty) => { + assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok()); + let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err(); + assert!( + error.to_string().contains("is invalid"), + "{} produced an unexpected error: {error}", + stringify!($id) + ); + }; + } + + assert_deserialization!(AgentId); + assert_deserialization!(ArtifactId); + assert_deserialization!(NodeId); + assert_deserialization!(ProcessId); + assert_deserialization!(LaunchAttemptId); + assert_deserialization!(ProjectId); + assert_deserialization!(TaskDefinitionId); + assert_deserialization!(TaskInstanceId); + assert_deserialization!(TenantId); + assert_deserialization!(UserId); + } + + #[test] + fn opaque_tokens_are_bounded_without_using_identifier_syntax() { + validate_opaque_token("opaque secret/+==", 64).unwrap(); + assert!(validate_opaque_token("", 64).is_err()); + assert!(validate_opaque_token("bad\0token", 64).is_err()); + assert!(validate_opaque_token(&"x".repeat(65), 64).is_err()); + } } diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index 0b9ca2d..fe1093b 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -66,9 +66,9 @@ pub use execution::{ WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION, }; pub use ids::{ - AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId, NodeId, ProcessId, - ProjectId, RequestId, TaskDefinitionId, TaskInstanceId, TenantId, UserId, - MAX_EXTERNAL_ID_BYTES, + validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId, + NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId, + TenantId, UserId, MAX_EXTERNAL_ID_BYTES, }; pub use limits::{ LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits, diff --git a/crates/clusterflux-dap/src/adapter.rs b/crates/clusterflux-dap/src/adapter.rs index 7cff5e4..bbf9a28 100644 --- a/crates/clusterflux-dap/src/adapter.rs +++ b/crates/clusterflux-dap/src/adapter.rs @@ -64,6 +64,7 @@ struct LaunchCompletion { state: AdapterState, local_runtime_session: Option, breakpoint_revision: u64, + observation_diagnostics: Vec, } pub(crate) fn run_adapter() -> Result<()> { @@ -108,6 +109,9 @@ pub(crate) fn run_adapter() -> Result<()> { if let Some(session) = completion.local_runtime_session { _local_runtime_session = Some(session); } + for diagnostic in completion.observation_diagnostics { + writer.output("stderr", format!("{diagnostic}\n"))?; + } runtime_started = true; if state.breakpoints_installed { emit_verified_breakpoints(&mut writer, &state)?; @@ -252,7 +256,12 @@ pub(crate) fn run_adapter() -> Result<()> { revision, result, } => { - if generation == runtime_generation && revision == state.breakpoint_revision { + if breakpoint_update_is_current( + generation, + runtime_generation, + revision, + state.breakpoint_revision, + ) { match result { Ok(()) => { state.breakpoints_installed = true; @@ -478,6 +487,7 @@ pub(crate) fn run_adapter() -> Result<()> { state: completed_state, local_runtime_session: None, breakpoint_revision, + observation_diagnostics: Vec::new(), } }) .map_err(|error| format!("services runtime attach failed: {error:#}")) @@ -486,6 +496,8 @@ pub(crate) fn run_adapter() -> Result<()> { RuntimeBackend::LocalServices => { run_local_services_runtime(&completed_state) .map(|(record, session)| { + let observation_diagnostics = + runtime_observation_diagnostics(&record); completed_state.apply_runtime_record(record); let breakpoint_revision = completed_state.breakpoint_revision; @@ -493,6 +505,7 @@ pub(crate) fn run_adapter() -> Result<()> { state: completed_state, local_runtime_session: Some(session), breakpoint_revision, + observation_diagnostics, } }) .map_err(|error| { @@ -502,6 +515,8 @@ pub(crate) fn run_adapter() -> Result<()> { RuntimeBackend::LiveServices => { run_live_services_runtime(&completed_state) .map(|record| { + let observation_diagnostics = + runtime_observation_diagnostics(&record); completed_state.apply_runtime_record(record); let breakpoint_revision = completed_state.breakpoint_revision; @@ -509,6 +524,7 @@ pub(crate) fn run_adapter() -> Result<()> { state: completed_state, local_runtime_session: None, breakpoint_revision, + observation_diagnostics, } }) .map_err(|error| { @@ -1051,6 +1067,29 @@ fn spawn_runtime_observer( }); } +pub(crate) fn breakpoint_update_is_current( + update_generation: u64, + runtime_generation: u64, + update_revision: u64, + current_revision: u64, +) -> bool { + update_generation == runtime_generation && update_revision == current_revision +} + +fn runtime_observation_diagnostics( + record: &crate::virtual_model::RuntimeLaunchRecord, +) -> Vec { + record + .node_report + .get("observation_diagnostics") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> { if !state.breakpoints_installed { return Ok(()); diff --git a/crates/clusterflux-dap/src/runtime_client.rs b/crates/clusterflux-dap/src/runtime_client.rs index 3e80bcc..2cdefaf 100644 --- a/crates/clusterflux-dap/src/runtime_client.rs +++ b/crates/clusterflux-dap/src/runtime_client.rs @@ -539,7 +539,24 @@ fn new_launch_attempt_id() -> String { fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> Result<()> { static INJECTED_INSTALL_FAILURE: AtomicBool = AtomicBool::new(false); - if std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some() + if std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION") + .ok() + .and_then(|value| value.parse::().ok()) + == Some(state.breakpoint_revision) + { + let delay_ms = std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(750); + std::thread::sleep(Duration::from_millis(delay_ms)); + } + let revision_failure = + std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION") + .ok() + .and_then(|value| value.parse::().ok()) + == Some(state.breakpoint_revision); + if (revision_failure + || std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some()) && !state.breakpoints.is_empty() && !INJECTED_INSTALL_FAILURE.swap(true, Ordering::SeqCst) { @@ -557,6 +574,7 @@ fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> "project": state.project_id, "actor_user": state.actor_user, "process": state.process.as_str(), + "revision": state.breakpoint_revision, "probe_symbols": state.requested_probe_symbols(), }), ), @@ -1026,6 +1044,12 @@ pub(crate) fn observe_services_runtime( let inject_connection_loss = std::env::var("CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS").as_deref() == Ok("1"); let mut connection_loss_injected = false; + let inject_snapshot_failure = + std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE").is_some(); + let mut snapshot_failure_injected = false; + let inject_process_status_failure = + std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE").is_some(); + let mut process_status_failure_injected = false; let inject_fallback_failure = std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE").is_some(); let mut fallback_failure_stage = 0_u8; @@ -1095,11 +1119,49 @@ pub(crate) fn observe_services_runtime( continue; } if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) { - let task_snapshots = fetch_task_snapshots_in(current_session, state) - .unwrap_or_else(|_| json!({ "snapshots": [] })); - let (process_statuses, process_status) = - fetch_current_process_status_in(current_session, state) - .unwrap_or_else(|_| (json!({ "processes": [] }), None)); + let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected { + snapshot_failure_injected = true; + Err(anyhow!("injected task snapshot transport failure")) + } else { + fetch_task_snapshots_in(current_session, state) + }; + let task_snapshots = match snapshot_request { + Ok(snapshots) => snapshots, + Err(error) => { + if !emit(RuntimeContinuationOutcome::Diagnostic(format!( + "runtime terminal snapshot observation failed: {error:#}; reconnecting in {} ms", + reconnect_delay.as_millis() + ))) { + return Ok(()); + } + session = None; + std::thread::sleep(reconnect_delay); + reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); + continue; + } + }; + let process_status_request = + if inject_process_status_failure && !process_status_failure_injected { + process_status_failure_injected = true; + Err(anyhow!("injected process-status transport failure")) + } else { + fetch_current_process_status_in(current_session, state) + }; + let (process_statuses, process_status) = match process_status_request { + Ok(status) => status, + Err(error) => { + if !emit(RuntimeContinuationOutcome::Diagnostic(format!( + "runtime terminal process observation failed: {error:#}; reconnecting in {} ms", + reconnect_delay.as_millis() + ))) { + return Ok(()); + } + session = None; + std::thread::sleep(reconnect_delay); + reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); + continue; + } + }; if !has_current_runtime_task(&task_snapshots) { if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { record.node_report = json!({ @@ -1113,7 +1175,13 @@ pub(crate) fn observe_services_runtime( return Ok(()); } } - let task_snapshots = match fetch_task_snapshots_in(current_session, state) { + let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected { + snapshot_failure_injected = true; + Err(anyhow!("injected task snapshot transport failure")) + } else { + fetch_task_snapshots_in(current_session, state) + }; + let task_snapshots = match snapshot_request { Ok(snapshots) => snapshots, Err(error) => { if !emit(RuntimeContinuationOutcome::Diagnostic(format!( @@ -1128,22 +1196,28 @@ pub(crate) fn observe_services_runtime( continue; } }; - let (process_statuses, process_status) = - match fetch_current_process_status_in(current_session, state) { - Ok(status) => status, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime process observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } + let process_status_request = + if inject_process_status_failure && !process_status_failure_injected { + process_status_failure_injected = true; + Err(anyhow!("injected process-status transport failure")) + } else { + fetch_current_process_status_in(current_session, state) }; + let (process_statuses, process_status) = match process_status_request { + Ok(status) => status, + Err(error) => { + if !emit(RuntimeContinuationOutcome::Diagnostic(format!( + "runtime process observation failed: {error:#}; reconnecting in {} ms", + reconnect_delay.as_millis() + ))) { + return Ok(()); + } + session = None; + std::thread::sleep(reconnect_delay); + reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); + continue; + } + }; reconnect_delay = Duration::from_millis(100); if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) { let failed_task = failed_task.to_owned(); @@ -1277,8 +1351,21 @@ pub(crate) fn observe_services_runtime( } }; if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) { - let task_snapshots = fetch_task_snapshots_in(current_session, state) - .unwrap_or_else(|_| json!({ "snapshots": [] })); + let task_snapshots = match fetch_task_snapshots_in(current_session, state) { + Ok(snapshots) => snapshots, + Err(snapshot_error) => { + if !emit(RuntimeContinuationOutcome::Diagnostic(format!( + "runtime fallback terminal snapshot observation failed: {snapshot_error:#}; reconnecting in {} ms", + reconnect_delay.as_millis() + ))) { + return Ok(()); + } + session = None; + std::thread::sleep(reconnect_delay); + reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); + continue; + } + }; if !has_current_runtime_task(&task_snapshots) { if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { record.node_report = json!({ diff --git a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs index 38dbc2f..add7be2 100644 --- a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs +++ b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs @@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result i64 { + fn insert_runtime_thread( + &mut self, + task: TaskInstanceId, + task_definition: TaskDefinitionId, + ) -> i64 { let id = self.allocate_runtime_thread_id(); let line = self .debug_probes .iter() .find(|probe| { - probe.task.as_str() == task_definition || probe.function == task_definition + probe.task == task_definition || probe.function == task_definition.as_str() }) .map(|probe| i64::from(probe.line_start)) .unwrap_or(1); - let definition_name = task_definition.replace(['_', '-'], " "); - let name = if task == task_definition { + let definition_name = task_definition.as_str().replace(['_', '-'], " "); + let name = if task.as_str() == task_definition.as_str() { definition_name } else { format!("{definition_name} ({task})") @@ -457,9 +475,9 @@ impl AdapterState { target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: TaskInstanceId::new(task), + task, attempt_id: "unknown-attempt".to_owned(), - task_definition: TaskDefinitionId::new(task_definition), + task_definition, name, line, state: DebugRuntimeState::Running, @@ -725,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId { ProcessId::new(format!("vp-{suffix}")) } -fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread { +fn coordinator_main_thread( + entry: &str, + task: TaskInstanceId, + task_definition: TaskDefinitionId, +) -> VirtualThread { + let name = format!("{entry} coordinator main ({task})"); VirtualThread { id: MAIN_THREAD, frame_id: 1_001, @@ -737,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi target_ref: 4_501, vfs_ref: 5_001, command_ref: 5_501, - task: TaskInstanceId::from(task), + task, attempt_id: "main:unknown".to_owned(), - task_definition: TaskDefinitionId::from(task_definition), - name: format!("{entry} coordinator main ({task})"), + task_definition, + name, line: 0, state: DebugRuntimeState::Running, freeze_supported: true, @@ -776,10 +799,14 @@ fn coordinator_threads_from_events( .expect("launch thread model should include main"); if let Some(status) = process_status { if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) { - main.task = TaskInstanceId::from(task); + if let Ok(task) = TaskInstanceId::try_new(task) { + main.task = task; + } } if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) { - main.task_definition = TaskDefinitionId::from(definition); + if let Ok(definition) = TaskDefinitionId::try_new(definition) { + main.task_definition = definition; + } } main.name = format!("{entry} coordinator main ({})", main.task); main.state = if status @@ -841,6 +868,12 @@ fn coordinator_threads_from_snapshots( .get("main_task_definition") .and_then(Value::as_str) .unwrap_or(entry); + let (Ok(task), Ok(task_definition)) = ( + TaskInstanceId::try_new(task), + TaskDefinitionId::try_new(task_definition), + ) else { + return threads; + }; let mut main = coordinator_main_thread(entry, task, task_definition); main.attempt_id = format!( "main:{}", @@ -895,6 +928,12 @@ fn coordinator_threads_from_snapshots( let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else { continue; }; + let Ok(task_id) = TaskInstanceId::try_new(task) else { + continue; + }; + let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else { + continue; + }; let attempt_id = snapshot .get("attempt_id") .and_then(Value::as_str) @@ -945,9 +984,9 @@ fn coordinator_threads_from_snapshots( target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: TaskInstanceId::from(task), + task: task_id, attempt_id, - task_definition: TaskDefinitionId::from(task_definition), + task_definition: task_definition_id, name: format!("{display_name} ({task}, attempt {short_attempt})"), line: snapshot .get("source_line") @@ -994,6 +1033,8 @@ fn coordinator_threads_from_snapshots( fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option { let task = event.get("task").and_then(Value::as_str)?; let task_definition = event.get("task_definition").and_then(Value::as_str)?; + let task_id = TaskInstanceId::try_new(task).ok()?; + let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?; let definition_name = task_definition.replace(['_', '-'], " "); let name = if task == task_definition { definition_name @@ -1039,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: TaskInstanceId::from(task), + task: task_id, attempt_id: event .get("attempt_id") .and_then(Value::as_str) .unwrap_or("unknown-attempt") .to_owned(), - task_definition: TaskDefinitionId::from(task_definition), + task_definition: task_definition_id, name, line: 0, state, diff --git a/crates/clusterflux-node/src/daemon.rs b/crates/clusterflux-node/src/daemon.rs index ed2b4fb..f62c7c5 100644 --- a/crates/clusterflux-node/src/daemon.rs +++ b/crates/clusterflux-node/src/daemon.rs @@ -7,8 +7,7 @@ use std::time::{Duration, Instant}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_core::{ sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId, - ProcessId, ProjectId, RequestId, TaskInstanceId, TaskSpec, TenantId, - MIN_SIGNED_NODE_POLL_INTERVAL_MS, + ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS, }; use serde_json::{json, Value}; @@ -577,7 +576,7 @@ fn parse_args() -> Result> { ProjectId::try_new(project.clone())?; NodeId::try_new(node.clone())?; if let Some(grant) = enrollment_grant.as_ref() { - RequestId::try_new(grant.clone())?; + clusterflux_core::validate_opaque_token(grant, 512)?; } Ok(Args { coordinator: coordinator.ok_or("--coordinator is required")?, diff --git a/docs/contributing/releases.md b/docs/contributing/releases.md index fe2b7d8..6dc422e 100644 --- a/docs/contributing/releases.md +++ b/docs/contributing/releases.md @@ -6,7 +6,7 @@ path. Publication is a three-stage transaction: 1. `candidate` builds immutable archives and a manifest with paths relative to the manifest directory. 2. `live-test` downloads that exact candidate in a clean job, deploys it, and - records the full 25-check production-shaped acceptance result plus deployment, + records the full named production-shaped acceptance result plus deployment, runtime configuration, and proxy configuration identities. 3. `final` downloads the candidate and evidence in another clean job, verifies every binding, and publishes without rebuilding any binary. diff --git a/scripts/agent-signing.js b/scripts/agent-signing.js index 71c913d..bfdc461 100644 --- a/scripts/agent-signing.js +++ b/scripts/agent-signing.js @@ -47,12 +47,12 @@ function agentWorkflowSignatureMessage({ function signedAgentWorkflowProof(identity, request, options = {}) { const nonce = - options.nonce || + options.nonce ?? `${request.type}-${process.pid}-${Date.now()}-${crypto .randomBytes(8) .toString("hex")}`; const issuedAtEpochSeconds = - options.issuedAtEpochSeconds || Math.floor(Date.now() / 1000); + options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000); const processId = request.type === "launch_task" ? request.task_spec?.process : request.process; const task = diff --git a/scripts/cli-happy-path-live-smoke.js b/scripts/cli-happy-path-live-smoke.js index c73218e..041baa4 100644 --- a/scripts/cli-happy-path-live-smoke.js +++ b/scripts/cli-happy-path-live-smoke.js @@ -137,6 +137,25 @@ function strictQualityGateEvidence(manifest) { assert(Number.isFinite(gate.duration_ms) && gate.duration_ms > 0); } assert.strictEqual(evidence.public.independent_filtered_tree, true); + for (const id of [ + "formatting", + "clippy_warnings_denied", + "public_workspace_tests", + "private_hosted_policy_locked_tests", + "process_lifecycle_regressions", + "wasm_example_builds", + "filtered_public_tree_build_and_tests", + "vscode_extension_candidate", + ]) { + const check = evidence.checks?.[id]; + assert(check, `quality evidence omitted ${id}`); + assert.strictEqual(check.status, "passed"); + assert(Number.isFinite(check.duration_ms) && check.duration_ms > 0); + } + assert.strictEqual( + evidence.checks.vscode_extension_candidate.candidate_vsix.sha256, + manifest.release_candidate.extension_sha256 + ); return evidence; } @@ -225,8 +244,14 @@ function authenticatedRequest(sessionSecret, request) { } function sendHostedControl(payload) { + return sendHostedControlEnvelope( + coordinatorWireRequest(payload, "strict-live") + ); +} + +function sendHostedControlEnvelope(envelope) { const body = Buffer.from( - JSON.stringify(coordinatorWireRequest(payload, "strict-live")) + JSON.stringify(envelope) ); const url = new URL("/api/v1/control", serviceEndpoint); return new Promise((resolve, reject) => { @@ -271,7 +296,19 @@ function sendHostedControl(payload) { }); } -async function runMalformedIdentifierSuite({ sessionSecret, tenant, project, user }) { +async function runMalformedIdentifierSuite({ + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + user, + suffix, + securityNode, + bundle, + releaseArtifact, +}) { const scenarioStartedAt = Date.now(); const forms = [ { name: "empty", value: "" }, @@ -280,169 +317,716 @@ async function runMalformedIdentifierSuite({ sessionSecret, tenant, project, use { name: "invalid_format", value: "hostile id!" }, { name: "oversized", value: "x".repeat(256) }, ]; - const valid = { + const tokenForms = forms + .filter((form) => form.name !== "invalid_format") + .map((form) => + form.name === "oversized" + ? { ...form, value: "x".repeat(257) } + : form + ); + assert( + Buffer.byteLength( + tokenForms.find((form) => form.name === "oversized").value + ) > 256, + "hostile token suite must exceed the signed-nonce and hosted-login byte limit" + ); + const agent = `hostile-id-agent-${suffix}`; + const agentIdentityRecord = agentIdentity("strict-hostile-id-agent", agent); + const processId = `vp-hostile-identifiers-${suffix}`; + const launchAttempt = `hostile-launch-attempt-${suffix}`; + const mainTask = `ti:${processId}:main`; + const taskSpec = { tenant, project, - user, - agent: "strict-agent", - node: "strict-node", - process: "strict-process", - task: "strict-task", - taskDefinition: "strict-task-definition", - artifact: "strict-artifact", - launchAttempt: "strict-launch-attempt", + process: processId, + task_definition: bundle.entryStableId, + task_instance: mainTask, + dispatch: { + kind: "coordinator_node_wasm", + export: bundle.entryExport, + abi: "entrypoint_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: 1, + failure_policy: "fail_fast", + bundle_digest: bundle.digest, }; - const principals = [ - { - name: "tenant", - payload: (value) => ({ - type: "auth_status", - tenant: value, - project: valid.project, - actor_user: valid.user, - }), - }, - { - name: "project", - payload: (value) => ({ - type: "list_processes", - tenant: valid.tenant, - project: value, - actor_user: valid.user, - }), - }, - { - name: "user", - payload: (value) => ({ - type: "auth_status", - tenant: valid.tenant, - project: valid.project, - actor_user: value, - }), - }, - { - name: "agent", - payload: (value) => ({ - type: "start_process", - tenant: valid.tenant, - project: valid.project, - actor_agent: value, - process: valid.process, - restart: false, - }), - }, - { - name: "node", - payload: (value) => ({ - type: "node_heartbeat", - tenant: valid.tenant, - project: valid.project, - node: value, - }), - }, - { - name: "process", - payload: (value) => ({ - type: "list_task_events", - tenant: valid.tenant, - project: valid.project, - actor_user: valid.user, - process: value, - }), - }, - { - name: "task_instance", - payload: (value) => ({ - type: "abort_task", - tenant: valid.tenant, - project: valid.project, - actor_user: valid.user, - process: valid.process, - task: value, - }), - }, - { - name: "task_definition", - payload: (value) => ({ - type: "launch_task", - tenant: valid.tenant, - project: valid.project, - actor_user: valid.user, - process: valid.process, - task_spec: { - task_definition: value, - task_instance: valid.task, - environment: "linux", - dependencies: [], - required_artifacts: [], - arguments: [], - }, - }), - }, - { - name: "artifact", - payload: (value) => ({ - type: "fetch_artifact", - tenant: valid.tenant, - project: valid.project, - actor_user: valid.user, - artifact: value, - }), - }, - { - name: "launch_attempt", - payload: (value) => ({ - type: "launch_main_runtime", - tenant: valid.tenant, - project: valid.project, - actor_user: valid.user, - process: valid.process, - launch_attempt: value, - }), - }, - ]; const rejected = []; - for (const principal of principals) { - for (const form of forms) { - const response = await sendHostedControl( - authenticatedRequest(sessionSecret, principal.payload(form.value)) - ); - const sessionDerived = - principal.name === "tenant" || principal.name === "user"; - if (sessionDerived) { - assert.strictEqual(response.type, "auth_status", JSON.stringify(response)); - assert.strictEqual(response.tenant, valid.tenant); - assert.strictEqual(response.project, valid.project); - assert.strictEqual(response.actor, valid.user); - } else { - assertDeniedOrEmptyCollection( - response, - /identifier|invalid|empty|whitespace|control|format|255|length|unknown (?:field|variant)/i, - `${principal.name} ${form.name}` - ); + const elapsedMs = (started) => + Number(process.hrtime.bigint() - started) / 1_000_000; + const rejectMalformed = async ({ + principal, + requestVariant, + fieldPath, + form, + send, + token = false, + }) => { + const started = process.hrtime.bigint(); + const response = await send(form.value); + assert.strictEqual( + response.type, + "error", + `${principal} ${fieldPath} ${form.name} reached the real ${requestVariant} variant but was not rejected: ${JSON.stringify(response)}` + ); + assert.match( + response.message, + token + ? /malformed external token|token.*invalid|control characters|byte limit|empty|whitespace/i + : /malformed external identifier|(?:Agent|Artifact|Node|Process|Project|TaskDefinition|TaskInstance|Tenant|User|LaunchAttempt)Id is invalid/i, + `${principal} ${fieldPath} ${form.name} was not rejected for identifier validation: ${JSON.stringify(response)}` + ); + assert.doesNotMatch( + response.message, + /unknown variant|unknown field|missing field/i, + `${principal} ${fieldPath} ${form.name} only proved generic deserialization failure` + ); + const health = await sendHostedControl({ type: "ping" }); + assert.strictEqual( + health.type, + "pong", + `valid traffic failed after hostile ${principal} ${fieldPath} ${form.name}` + ); + rejected.push({ + principal, + request_variant: requestVariant, + field_path: fieldPath, + fault: form.name, + expected: token + ? "structured malformed external token error" + : "structured malformed external identifier error", + observed: response.message, + health_after: health.type, + duration_ms: elapsedMs(started), + }); + }; + + let agentRegistered = false; + let processStarted = false; + try { + const added = runJson( + clusterflux, + [ + "key", + "add", + ...scope, + "--agent", + agent, + "--public-key", + agentIdentityRecord.publicKey, + ], + { cwd: projectDir } + ); + assert.strictEqual(added.command, "key add"); + agentRegistered = true; + + const agentStartCases = [ + ["tenant", "tenant"], + ["project", "project"], + ["agent", "actor_agent"], + ["process", "process"], + ["launch_attempt", "launch_attempt"], + ]; + for (const [principal, field] of agentStartCases) { + for (const form of forms) { + await rejectMalformed({ + principal: `agent_signed_${principal}`, + requestVariant: "start_process", + fieldPath: field, + form, + send: async (value) => { + const body = { + type: "start_process", + tenant, + project, + actor_agent: agent, + process: processId, + launch_attempt: launchAttempt, + restart: false, + }; + body[field] = value; + return sendHostedControl( + signedAgentWorkflowRequest(agentIdentityRecord, body, { + nonce: `hostile-agent-${principal}-${form.name}-${Date.now()}`, + }) + ); + }, + }); } - const health = await sendHostedControl({ type: "ping" }); - assert.strictEqual( - health.type, - "pong", - `valid traffic failed after hostile ${principal.name} ${form.name}` - ); - rejected.push({ - principal: principal.name, - form: form.name, - rejected: !sessionDerived, - server_derived_identity: sessionDerived, - health_after: health.type, + } + for (const form of tokenForms) { + await rejectMalformed({ + principal: "agent_signed_nonce", + requestVariant: "start_process", + fieldPath: "agent_signature.nonce", + form, + token: true, + send: async (value) => { + const request = signedAgentWorkflowRequest( + agentIdentityRecord, + { + type: "start_process", + tenant, + project, + actor_agent: agent, + process: processId, + launch_attempt: launchAttempt, + restart: false, + }, + { nonce: value } + ); + assert.strictEqual( + request.agent_signature.nonce, + value, + "hostile Agent nonce was not preserved on the wire" + ); + return sendHostedControl(request); + }, }); } + for (const form of forms) { + await rejectMalformed({ + principal: "node_signed_heartbeat_node", + requestVariant: "node_heartbeat", + fieldPath: "node", + form, + send: async (value) => + sendHostedControl({ + type: "node_heartbeat", + node: value, + node_signature: signedNodeHeartbeat( + value, + securityNode.identity, + { + nonce: `hostile-heartbeat-node-${form.name}-${Date.now()}`, + } + ), + }), + }); + } + + const started = await sendHostedControl( + signedAgentWorkflowRequest( + agentIdentityRecord, + { + type: "start_process", + tenant, + project, + actor_agent: agent, + process: processId, + launch_attempt: launchAttempt, + restart: false, + }, + { nonce: `hostile-agent-valid-start-${suffix}` } + ) + ); + assert.strictEqual(started.type, "process_started", JSON.stringify(started)); + processStarted = true; + taskSpec.vfs_epoch = started.epoch; + + const taskSpecCases = [ + ["tenant", "task_spec.tenant"], + ["project", "task_spec.project"], + ["process", "task_spec.process"], + ["task_definition", "task_spec.task_definition"], + ["task_instance", "task_spec.task_instance"], + ]; + for (const [field, fieldPath] of taskSpecCases) { + for (const form of forms) { + await rejectMalformed({ + principal: `agent_signed_${field}`, + requestVariant: "launch_task", + fieldPath, + form, + send: async (value) => { + const malformedTaskSpec = structuredClone(taskSpec); + malformedTaskSpec[field] = value; + const body = { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: malformedTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + return sendHostedControl( + signedAgentWorkflowRequest(agentIdentityRecord, body, { + nonce: `hostile-task-spec-${field}-${form.name}-${Date.now()}`, + }) + ); + }, + }); + } + } + + for (const form of forms) { + await rejectMalformed({ + principal: "agent_signed_artifact_array", + requestVariant: "launch_task", + fieldPath: "task_spec.required_artifacts[0]", + form, + send: async (value) => { + const malformedTaskSpec = structuredClone(taskSpec); + malformedTaskSpec.required_artifacts = [releaseArtifact.artifact]; + malformedTaskSpec.args = [ + { + Artifact: { + id: releaseArtifact.artifact, + digest: releaseArtifact.digest, + size_bytes: Number(releaseArtifact.size_bytes || 1), + }, + }, + ]; + malformedTaskSpec.required_artifacts = [value]; + const body = { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: malformedTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${mainTask}-artifact-output.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + return sendHostedControl( + signedAgentWorkflowRequest(agentIdentityRecord, body, { + nonce: `hostile-artifact-array-${form.name}-${Date.now()}`, + }) + ); + }, + }); + } + + const mainLaunch = await sendHostedControl( + signedAgentWorkflowRequest( + agentIdentityRecord, + { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: taskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `hostile-agent-valid-main-${suffix}` } + ) + ); + assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch)); + + const authenticatedCases = [ + { + principal: "authenticated_cli_project", + requestVariant: "select_project", + fieldPath: "request.project", + payload: (value) => ({ type: "select_project", project: value }), + }, + { + principal: "authenticated_cli_process", + requestVariant: "list_task_events", + fieldPath: "request.process", + payload: (value) => ({ type: "list_task_events", process: value }), + }, + { + principal: "authenticated_cli_task", + requestVariant: "join_task", + fieldPath: "request.task", + payload: (value) => ({ + type: "join_task", + process: processId, + task: value, + }), + }, + { + principal: "authenticated_cli_artifact", + requestVariant: "create_artifact_download_link", + fieldPath: "request.artifact", + payload: (value) => ({ + type: "create_artifact_download_link", + artifact: value, + max_bytes: Number(releaseArtifact.size_bytes || 1), + ttl_seconds: 60, + }), + }, + { + principal: "authenticated_cli_launch_attempt", + requestVariant: "abort_process", + fieldPath: "request.launch_attempt", + payload: (value) => ({ + type: "abort_process", + process: processId, + launch_attempt: value, + }), + }, + ]; + for (const testCase of authenticatedCases) { + for (const form of forms) { + await rejectMalformed({ + ...testCase, + form, + send: async (value) => + sendHostedControl( + authenticatedRequest(sessionSecret, testCase.payload(value)) + ), + }); + } + } + + const nodeCases = [ + { + principal: "node_signed_wrapper_node", + requestVariant: "poll_task_assignment", + fieldPath: "node", + send: (value, form) => + sendHostedControl( + signedNodeRequest( + value, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant, + project, + node: securityNode.node, + }, + { nonce: `hostile-node-wrapper-${form.name}-${Date.now()}` } + ) + ), + }, + { + principal: "node_signed_poll_tenant", + requestVariant: "poll_task_assignment", + fieldPath: "request.tenant", + send: (value, form) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant: value, + project, + node: securityNode.node, + }, + { nonce: `hostile-node-tenant-${form.name}-${Date.now()}` } + ) + ), + }, + { + principal: "node_signed_poll_project", + requestVariant: "poll_task_assignment", + fieldPath: "request.project", + send: (value, form) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant, + project: value, + node: securityNode.node, + }, + { nonce: `hostile-node-project-${form.name}-${Date.now()}` } + ) + ), + }, + { + principal: "node_signed_poll_node", + requestVariant: "poll_task_assignment", + fieldPath: "request.node", + send: (value, form) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant, + project, + node: value, + }, + { nonce: `hostile-node-inner-${form.name}-${Date.now()}` } + ) + ), + }, + { + principal: "node_signed_artifact_array", + requestVariant: "report_node_capabilities", + fieldPath: "request.artifact_locations[0]", + send: (value, form) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_node_capabilities", + { + ...securityNode.capability_body, + artifact_locations: [value], + }, + { nonce: `hostile-node-artifact-${form.name}-${Date.now()}` } + ) + ), + }, + ]; + for (const testCase of nodeCases) { + for (const form of forms) { + await rejectMalformed({ + principal: testCase.principal, + requestVariant: testCase.requestVariant, + fieldPath: testCase.fieldPath, + form, + send: (value) => testCase.send(value, form), + }); + } + } + for (const form of tokenForms) { + await rejectMalformed({ + principal: "node_signed_nonce", + requestVariant: "node_heartbeat", + fieldPath: "node_signature.nonce", + form, + token: true, + send: async (value) => { + const request = { + type: "node_heartbeat", + node: securityNode.node, + node_signature: signedNodeHeartbeat( + securityNode.node, + securityNode.identity, + { nonce: value } + ), + }; + assert.strictEqual( + request.node_signature.nonce, + value, + "hostile Node nonce was not preserved on the wire" + ); + return sendHostedControl(request); + }, + }); + } + + const endpointFingerprint = sha256( + Buffer.from(securityNode.identity.publicKey) + ); + const rendezvous = { + type: "request_rendezvous", + scope: { + tenant, + project, + process: processId, + object: { Artifact: releaseArtifact.artifact }, + authorization_subject: `${securityNode.node}-self-transfer`, + }, + source: { + node: securityNode.node, + advertised_addr: "127.0.0.1:4433", + public_key_fingerprint: endpointFingerprint, + }, + destination: { + node: securityNode.node, + advertised_addr: "127.0.0.1:4433", + public_key_fingerprint: endpointFingerprint, + }, + direct_connectivity: true, + failure_reason: "", + }; + const rendezvousCases = [ + ["scope.tenant", (request, value) => (request.scope.tenant = value)], + ["scope.project", (request, value) => (request.scope.project = value)], + ["scope.process", (request, value) => (request.scope.process = value)], + [ + "scope.object.artifact", + (request, value) => (request.scope.object.Artifact = value), + ], + ["source.node", (request, value) => (request.source.node = value)], + [ + "destination.node", + (request, value) => (request.destination.node = value), + ], + ]; + for (const [fieldPath, mutate] of rendezvousCases) { + for (const form of forms) { + await rejectMalformed({ + principal: "node_signed_rendezvous", + requestVariant: "request_rendezvous", + fieldPath: `request.${fieldPath}`, + form, + send: async (value) => { + const body = structuredClone(rendezvous); + mutate(body, value); + return sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "request_rendezvous", + body, + { + nonce: `hostile-rendezvous-${fieldPath}-${form.name}-${Date.now()}`, + } + ) + ); + }, + }); + } + } + const validRendezvous = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "request_rendezvous", + rendezvous, + { nonce: `hostile-rendezvous-valid-${suffix}` } + ) + ); + assert.strictEqual( + validRendezvous.type, + "rendezvous_plan", + JSON.stringify(validRendezvous) + ); + + const login = await sendHostedLogin({ type: "begin_oidc_browser_login" }); + assert.strictEqual(login.type, "oidc_browser_login_started", JSON.stringify(login)); + const loginCases = [ + ["transaction_id", login.transaction_id, login.polling_secret], + ["polling_secret", login.transaction_id, login.polling_secret], + ]; + for (const [field, validTransaction, validSecret] of loginCases) { + for (const form of tokenForms) { + await rejectMalformed({ + principal: "private_hosted_login", + requestVariant: "poll_oidc_browser_login", + fieldPath: field, + form, + token: true, + send: async (value) => + sendHostedLogin({ + type: "poll_oidc_browser_login", + transaction_id: + field === "transaction_id" ? value : validTransaction, + polling_secret: field === "polling_secret" ? value : validSecret, + }), + }); + const validPoll = await sendHostedLogin({ + type: "poll_oidc_browser_login", + transaction_id: login.transaction_id, + polling_secret: login.polling_secret, + }); + assert.strictEqual( + validPoll.type, + "oidc_browser_login_pending", + `valid hosted login polling failed after malformed ${field}` + ); + } + } + + const operatorControlCases = [ + ["payload.tenant", "tenant"], + ["payload.project", "project"], + ["payload.process", "process"], + ]; + for (const [fieldPath, field] of operatorControlCases) { + for (const form of forms) { + await rejectMalformed({ + principal: "private_hosted_operator_control", + requestVariant: "stop_hosted_process", + fieldPath, + form, + send: async (value) => { + const payload = { + type: "stop_hosted_process", + tenant, + project, + process: processId, + }; + payload[field] = value; + return sendHostedControlEnvelope({ + type: "hosted_operator_request", + protocol_version: 1, + request_id: `hostile-operator-${field}-${form.name}-${Date.now()}`, + operation: "stop_hosted_process", + operator_proof: sha256(Buffer.from("invalid operator proof")), + operator_nonce: `hostile-operator-nonce-${field}-${form.name}-${Date.now()}`, + issued_at_epoch_seconds: Math.floor(Date.now() / 1000), + payload, + }); + }, + }); + } + } + + const authenticatedHealth = await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "auth_status" }) + ); + assert.strictEqual(authenticatedHealth.type, "auth_status"); + assert.strictEqual(authenticatedHealth.tenant, tenant); + assert.strictEqual(authenticatedHealth.project, project); + assert.strictEqual(authenticatedHealth.actor, user); + } finally { + if (processStarted) { + try { + runJson( + clusterflux, + [ + "process", + "abort", + ...scope, + "--process", + processId, + "--yes", + ], + { cwd: projectDir } + ); + } catch (_) { + // Preserve the primary validation failure while still attempting cleanup. + } + } + if (agentRegistered) { + try { + runJson( + clusterflux, + ["key", "revoke", ...scope, "--agent", agent, "--yes"], + { cwd: projectDir } + ); + } catch (_) { + // Preserve the primary validation failure while still attempting cleanup. + } + } } + + const coveredPrincipals = [...new Set(rejected.map((entry) => entry.principal))]; + const coveredVariants = [...new Set(rejected.map((entry) => entry.request_variant))]; return { - principals: principals.map((principal) => principal.name), + principals: coveredPrincipals, + request_variants: coveredVariants, forms: forms.map((form) => form.name), rejected_requests: rejected, + all_rejected_for_intended_reason: rejected.every( + (entry) => + /identifier|token|Id is invalid|control characters|byte limit|empty|whitespace/i.test( + entry.observed + ) + ), valid_after_every_rejection: rejected.every( (entry) => entry.health_after === "pong" ), + valid_authenticated_action_after_suite: true, + valid_signed_rendezvous_after_suite: true, + valid_private_login_poll_after_every_private_rejection: true, duration_ms: Date.now() - scenarioStartedAt, }; } @@ -522,6 +1106,58 @@ function sendHostedLoginStatus(extraHeaders = {}) { }); } +function sendHostedLogin(payload) { + const body = Buffer.from( + JSON.stringify( + coordinatorWireRequest( + payload, + `strict-live-login-protocol-${Date.now()}-${Math.random()}` + ) + ) + ); + const url = new URL("/api/v1/login", serviceEndpoint); + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + }, + timeout: 30_000, + }, + (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => { + const responseBody = Buffer.concat(chunks).toString("utf8"); + if (response.statusCode < 200 || response.statusCode >= 300) { + reject( + new Error( + `hosted login HTTP ${response.statusCode}: ${responseBody}` + ) + ); + return; + } + try { + resolve(JSON.parse(responseBody)); + } catch (error) { + reject( + new Error(`hosted login returned invalid JSON: ${error.message}`) + ); + } + }); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted login request timed out")) + ); + request.on("error", reject); + request.end(body); + }); +} + async function runHostedLoginIsolationBeforeRestart() { const scenarioStartedAt = Date.now(); const controlBefore = await sendHostedControl({ type: "ping" }); @@ -949,6 +1585,7 @@ async function prepareLiveNodeCredentialSecurity({ project, suffix, }) { + const startedAt = Date.now(); const node = `security-node-${suffix}`; const grant = runJson(clusterflux, ["node", "enroll", ...scope], { cwd: projectDir, @@ -1052,6 +1689,7 @@ async function prepareLiveNodeCredentialSecurity({ ); return { + duration_ms: Date.now() - startedAt, node, identity, credential_path: stored.absolute, @@ -1080,6 +1718,7 @@ async function runLiveAgentCredentialSecurity({ securityNode, workerRuntime, }) { + const scenarioStartedAt = Date.now(); const agent = `security-agent-${suffix}`; const identity = agentIdentity("strict-live-agent", agent); const added = runJson( @@ -1199,6 +1838,9 @@ async function runLiveAgentCredentialSecurity({ let liveParentTask; let liveParentAssignment; let realDebugLaunch; + let preconfiguredBreakpoint; + let attachSourcePath; + let attachBreakpointLine; const realDebugTask = `real-debug-participant-${suffix}`; let workerPaused = false; const baselineContainers = new Set( @@ -1306,6 +1948,32 @@ async function runLiveAgentCredentialSecurity({ const liveParentSpec = liveParentAssignment.task_assignment_response.task_spec; + attachSourcePath = path.join(projectDir, "src/lib.rs"); + const attachSourceLines = fs + .readFileSync(attachSourcePath, "utf8") + .split(/\r?\n/); + attachBreakpointLine = + attachSourceLines.findIndex((line) => line.includes("fn abort_probe(")) + 1; + assert( + attachBreakpointLine > 0, + "agent-security fixture omitted the abort_probe debug probe" + ); + preconfiguredBreakpoint = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "set_debug_breakpoints", + process: processId, + revision: 1, + probe_symbols: ["clusterflux.probe.abort_probe"], + }) + ); + assert.strictEqual( + preconfiguredBreakpoint.type, + "debug_breakpoints", + JSON.stringify(preconfiguredBreakpoint) + ); + assert.deepStrictEqual(preconfiguredBreakpoint.probe_symbols, [ + "clusterflux.probe.abort_probe", + ]); realDebugLaunch = await sendHostedControl( signedNodeRequest( workerRuntime.node, @@ -1392,26 +2060,7 @@ async function runLiveAgentCredentialSecurity({ "real Podman debug participant to start", 120_000 ); - const realDebugContainerDeadline = Date.now() + 120_000; let realDebugContainerIds = new Set(); - let lastRealDebugPodmanStates = []; - while (Date.now() < realDebugContainerDeadline) { - const containers = runJson("podman", ["ps", "--format", "json"]); - lastRealDebugPodmanStates = containers; - realDebugContainerIds = new Set( - containers - .map((container) => container.Id || container.ID || container.IdHex || "") - .filter((id) => id && !baselineContainers.has(id)) - ); - if (realDebugContainerIds.size > 0) break; - await delay(100); - } - assert( - realDebugContainerIds.size > 0, - `real debug participant did not start a Podman container: ${JSON.stringify( - lastRealDebugPodmanStates - )}` - ); const debugClient = new DapClient({ cwd: projectDir, @@ -1436,14 +2085,6 @@ async function runLiveAgentCredentialSecurity({ await debugClient.waitFor( (message) => message.type === "event" && message.event === "initialized" ); - const attachSourcePath = path.join(projectDir, "src/lib.rs"); - const attachSourceLines = fs.readFileSync(attachSourcePath, "utf8").split(/\r?\n/); - const attachBreakpointLine = - attachSourceLines.findIndex((line) => line.includes("fn task_trap(")) + 1; - assert( - attachBreakpointLine > 0, - "agent-security fixture omitted the task_trap debug probe" - ); const attachBreakpointRequest = debugClient.send("setBreakpoints", { source: { path: attachSourcePath }, breakpoints: [{ line: attachBreakpointLine }], @@ -1470,6 +2111,53 @@ async function runLiveAgentCredentialSecurity({ attachBreakpointInstalled.body.breakpoint.line, attachBreakpointLine ); + const attachBreakpointStop = await debugClient.waitFor( + (message) => + message.seq > attachBreakpointInstalled.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint", + 120_000 + ); + const attachStackRequest = debugClient.send("stackTrace", { + threadId: attachBreakpointStop.body.threadId, + startFrame: 0, + levels: 1, + }); + const attachStackFrames = ( + await debugClient.response(attachStackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual( + attachStackFrames[0].line, + attachBreakpointLine, + "attach stopped somewhere other than the installed configured breakpoint" + ); + const attachContinue = debugClient.send("continue", { + threadId: attachBreakpointStop.body.threadId, + }); + const attachContinueResponse = await debugClient.response( + attachContinue, + "continue" + ); + const realDebugContainerDeadline = Date.now() + 120_000; + let lastRealDebugPodmanStates = []; + while (Date.now() < realDebugContainerDeadline) { + const containers = runJson("podman", ["ps", "--format", "json"]); + lastRealDebugPodmanStates = containers; + realDebugContainerIds = new Set( + containers + .map((container) => container.Id || container.ID || container.IdHex || "") + .filter((id) => id && !baselineContainers.has(id)) + ); + if (realDebugContainerIds.size > 0) break; + await delay(100); + } + assert( + realDebugContainerIds.size > 0, + `continued real debug participant did not start a Podman container: ${JSON.stringify( + lastRealDebugPodmanStates + )}` + ); const attachDeadline = Date.now() + 120_000; let attachedThreads = []; while (Date.now() < attachDeadline) { @@ -1490,6 +2178,7 @@ async function runLiveAgentCredentialSecurity({ const dapPartialStop = await debugClient.waitFor( (message) => message.type === "event" && + message.seq > attachContinueResponse.seq && message.event === "stopped" && message.body.reason === "pause", 30_000 @@ -1620,6 +2309,7 @@ async function runLiveAgentCredentialSecurity({ ); await debugClient.close(); + const mainBeforeChildStartedAt = Date.now(); const completed = await waitForCli( "agent-authenticated coordinator main and child workflow completion", () => @@ -1642,6 +2332,152 @@ async function runLiveAgentCredentialSecurity({ ), ]; assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents)); + const mainCompletion = completedEvents.find( + (event) => + event.executor === "coordinator_main" && + event.terminal_state === "completed" + ); + assert(mainCompletion, "real coordinator main did not complete"); + assert( + !completedEvents.some( + (event) => + event.task === fakeTask && typeof event.terminal_state === "string" + ), + "the deliberately delayed final child completed before the coordinator main" + ); + const activeAfterMain = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.notStrictEqual(activeAfterMain.state, "not_active"); + assert.equal(activeAfterMain.live_process.main_state, null); + const debugStateAfterMain = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "inspect_debug_breakpoints", + process: processId, + }) + ); + assert.strictEqual( + debugStateAfterMain.type, + "debug_breakpoints", + `main completion cleared active-child debug state: ${JSON.stringify( + debugStateAfterMain + )}` + ); + const finalChildCompletion = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "task_completed", + { + type: "task_completed", + tenant, + project, + process: processId, + node: securityNode.node, + task: fakeTask, + terminal_state: "completed", + status_code: 0, + stdout_bytes: 2, + stderr_bytes: 0, + stdout_tail: "42", + stderr_tail: "", + stdout_truncated: false, + stderr_truncated: false, + artifact_path: null, + artifact_digest: null, + artifact_size_bytes: null, + result: { SmallJson: 42 }, + }, + { nonce: `strict-main-before-child-complete-${suffix}` } + ) + ); + assert.strictEqual( + finalChildCompletion.type, + "task_recorded", + JSON.stringify(finalChildCompletion) + ); + const releasedAfterFinalChild = await waitForCli( + "final delayed child to release the active process slot", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + const retainedEvents = runJson( + clusterflux, + ["task", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert( + rawTaskEvents(retainedEvents).some( + (event) => + event.task === fakeTask && event.terminal_state === "completed" + ), + "final delayed child history was not retained" + ); + const retainedArtifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + const retainedArtifact = retainedArtifacts.artifacts.find( + (artifact) => + typeof artifact.digest === "string" && + /^sha256:[0-9a-f]{64}$/.test(artifact.digest) && + artifact.artifact.startsWith("release.tar-") + ); + assert(retainedArtifact, "real main artifact metadata was not retained"); + const subsequentRun = runJson( + clusterflux, + ["run", "park-wake", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(subsequentRun.status, "main_launched"); + const subsequentAbort = runJson( + clusterflux, + ["process", "abort", ...scope, "--process", subsequentRun.process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(subsequentAbort.abort_request.accepted, true); + await waitForCli( + "subsequent run cleanup after final-child slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", subsequentRun.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + const mainBeforeChildLifecycle = { + request: { + process: processId, + main: bundle.entryStableId, + delayed_child: fakeTask, + fault_injection: + "withhold the assigned final child's signed completion until the real coordinator main completes", + }, + expected: + "process and debug state remain until the final child completes; final cleanup retains history and real artifact metadata; a subsequent run starts", + observed: { + main_terminal_state: mainCompletion.terminal_state, + active_after_main: activeAfterMain.state !== "not_active", + debug_state_after_main: debugStateAfterMain.type, + child_terminal_state: "completed", + final_process_state: releasedAfterFinalChild.state, + retained_event_count: rawTaskEvents(retainedEvents).length, + retained_artifact: retainedArtifact, + subsequent_run_status: subsequentRun.status, + }, + duration_ms: Math.max(1, Date.now() - mainBeforeChildStartedAt), + }; const revoked = runJson( clusterflux, @@ -1661,6 +2497,7 @@ async function runLiveAgentCredentialSecurity({ "revoked agent credential" ); return { + duration_ms: Date.now() - scenarioStartedAt, agent, valid_request: valid.type, replay, @@ -1681,6 +2518,26 @@ async function runLiveAgentCredentialSecurity({ flagship_completed: completedFlagship(completedEvents), concurrent_compile_tasks: concurrentCompileTasks, }, + main_before_child_lifecycle: mainBeforeChildLifecycle, + attach_with_preconfigured_breakpoint: { + request: { + process: processId, + source: attachSourcePath, + line: attachBreakpointLine, + }, + expected: + "preconfigured before attach, unverified until adapter installation, then a real breakpoint stop at the configured line", + observed: { + preconfigured_revision: preconfiguredBreakpoint.revision, + initially_verified: + attachBreakpointResponse.body.breakpoints[0].verified, + installation_event_verified: + attachBreakpointInstalled.body.breakpoint.verified, + stop_reason: attachBreakpointStop.body.reason, + stopped_line: attachStackFrames[0].line, + }, + duration_ms: Date.now() - scenarioStartedAt, + }, partial_freeze: { epoch: freeze.epoch, elapsed_ms: Date.now() - freezeStartedAt, @@ -1707,6 +2564,7 @@ async function runSecondTenantIsolation({ firstArtifact, manifest, }) { + const startedAt = Date.now(); const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE; const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE; const targetTenant = process.env.CLUSTERFLUX_ISOLATION_TARGET_TENANT; @@ -1740,6 +2598,7 @@ async function runSecondTenantIsolation({ ); return { ...isolation, + duration_ms: Date.now() - startedAt, reused_from_immutable_evidence: { report_sha256: sha256(evidenceBytes), source_commit: previous.source_commit, @@ -1859,6 +2718,7 @@ async function runSecondTenantIsolation({ }, artifact_and_download: artifact, process_control: control, + duration_ms: Date.now() - startedAt, }; } if (!file) return { executed: false, reason: "second tenant session not supplied" }; @@ -1936,6 +2796,7 @@ async function runSecondTenantIsolation({ debug, artifact_and_download: artifact, process_control: control, + duration_ms: Date.now() - startedAt, }; } @@ -2089,6 +2950,7 @@ async function revokeSecurityNode({ scope, securityNode, }) { + const startedAt = Date.now(); const revoked = runJson( clusterflux, ["node", "revoke", ...scope, "--node", securityNode.node, "--yes"], @@ -2108,10 +2970,15 @@ async function revokeSecurityNode({ /revoked|unknown node|not recognized|not enrolled/i, "revoked node credential" ); - return { node: securityNode.node, denied }; + return { + node: securityNode.node, + denied, + duration_ms: Date.now() - startedAt, + }; } async function runLiveQuotaPreallocation({ sessionSecret, suffix }) { + const startedAt = Date.now(); const readSpawnQuota = async () => { const status = await sendHostedControl( authenticatedRequest(sessionSecret, { type: "quota_status" }) @@ -2196,6 +3063,7 @@ async function runLiveQuotaPreallocation({ sessionSecret, suffix }) { `spawn quota exhaustion locked an independent read scope: ${JSON.stringify(independentScope)}` ); return { + duration_ms: Date.now() - startedAt, limit: spawnQuota.limit, initial_usage: spawnQuota.usage, window_seconds: spawnQuota.windowSeconds, @@ -2263,6 +3131,7 @@ async function runLongJoinProof({ clusterflux, projectDir, scope }) { terminal_state: completed.terminal_state, result: completed.result, duration_seconds: durationSeconds, + duration_ms: Math.max(1, Date.now() - startedAt), process_state: released.state, }; } @@ -2383,6 +3252,7 @@ async function runLaunchAttemptOwnershipGuards({ sessionSecret, activeProcess, }) { + const startedAt = Date.now(); const rejectedAttempt = `launch-guard-rejected-${Date.now()}`; const rejection = await sendHostedControl( authenticatedRequest(sessionSecret, { @@ -2414,6 +3284,7 @@ async function runLaunchAttemptOwnershipGuards({ assert.notStrictEqual(surviving.state, "not_active"); assert.strictEqual(surviving.process, activeProcess); return { + duration_ms: Date.now() - startedAt, rejected_attempt: rejectedAttempt, rejection: rejection.message, wrong_abort_attempt: wrongAttempt, @@ -2635,6 +3506,7 @@ async function runRelayEmergencyDisable({ sessionSecret, maxBytes, }) { + const startedAt = Date.now(); if (!strictVpsRestart) { throw new Error("strict relay disable proof requires VPS systemd access"); } @@ -2733,6 +3605,7 @@ async function runRelayEmergencyDisable({ assert.strictEqual(revoked.type, "artifact_download_link_revoked"); return { evidence: { + duration_ms: Date.now() - startedAt, disabled, disabled_probe_artifact: disabledArtifact.artifact, restored: restored.type, @@ -3013,6 +3886,7 @@ async function finishUserCredentialSecurity({ scope, sessionSecret, }) { + const startedAt = Date.now(); const forged = assertDenied( await sendHostedControl( authenticatedRequest( @@ -3053,7 +3927,12 @@ async function finishUserCredentialSecurity({ /revoked|not recognized/i, "revoked user session" ); - return { forged, expired, revoked }; + return { + forged, + expired, + revoked, + duration_ms: Date.now() - startedAt, + }; } async function dapVariables(client, variablesReference) { @@ -3116,6 +3995,26 @@ async function runSameDefinitionDapIdentity({ await delay(100); } assert(mainThread, "identity entry did not report its coordinator main"); + const launchWithoutBreakpointMs = Date.now() - scenarioStartedAt; + const postCommitDiagnostics = [ + "initial task observation failed after main_launched", + "initial process observation failed after main_launched", + ].map((expected) => { + const message = client.messages.find( + (candidate) => + candidate.type === "event" && + candidate.event === "output" && + String(candidate.body?.output || "").includes(expected) + ); + assert(message, `post-commit launch evidence omitted: ${expected}`); + return String(message.body.output).trim(); + }); + const prematurePostCommitEvents = client.messages.filter( + (message) => + message.type === "event" && + ["stopped", "terminated"].includes(message.event) + ); + assert.deepStrictEqual(prematurePostCommitEvents, []); const processRequest = client.send("evaluate", { expression: "virtual_process_id", context: "watch", @@ -3279,6 +4178,52 @@ async function runSameDefinitionDapIdentity({ return { duration_ms: Date.now() - scenarioStartedAt, process: processId, + launch_without_breakpoint: { + request: { + entry: "identity", + runtime_backend: "live-services", + configured_breakpoints: [], + }, + expected: "launch commits and reports a real coordinator-main thread", + observed: { + thread_id: mainThread.id, + thread_name: mainThread.name, + }, + duration_ms: launchWithoutBreakpointMs, + }, + post_main_launched_observation_recovery: { + request: { + entry: "identity", + fault_injections: [ + "initial task snapshot observation failure", + "initial process-status observation failure", + ], + }, + expected: + "main_launched is committed, no rollback or false stop occurs, and the observer reconnects until threads appear", + observed: { + diagnostics: postCommitDiagnostics, + main_thread: mainThread, + premature_stop_or_termination_events: + prematurePostCommitEvents.length, + }, + duration_ms: launchWithoutBreakpointMs, + }, + same_definition_identity: { + expected: + "two concurrent instances retain distinct thread, argument, attempt, output, freeze, and terminal identities", + observed: { + thread_evidence: threadEvidence, + completion_order: identityEvents.map((event) => ({ + task_instance: event.task, + attempt_id: event.attempt_id, + result: event.result, + })), + paused_container_count: clusterfluxPausedContainers.length, + terminal_state: released.state, + }, + duration_ms: Date.now() - scenarioStartedAt, + }, thread_evidence: threadEvidence, completion_order: identityEvents.map((event) => ({ task_instance: event.task, @@ -3325,9 +4270,13 @@ async function runLiveDapEditRestart({ env: { ...process.env, CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1", + CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE: "1", + CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE: "1", CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE: "1", CLUSTERFLUX_TEST_DAP_OBSERVER_DEBUG_EPOCH_WAIT_FAILURE: "1", - CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE: "1", + CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION: "1", + CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS: "750", + CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION: "3", }, }); let sourceRestored = false; @@ -3353,7 +4302,10 @@ async function runLiveDapEditRestart({ const configurationStartedAt = Date.now(); const configurationDone = client.send("configurationDone"); - await client.response(configurationDone, "configurationDone"); + const configurationResponse = await client.response( + configurationDone, + "configurationDone" + ); const configurationResponseMs = Date.now() - configurationStartedAt; assert( configurationResponseMs < 2_000, @@ -3362,14 +4314,67 @@ async function runLiveDapEditRestart({ const threadDeadline = Date.now() + 5 * 60 * 1000; let runningThreads = []; + let threadObservationResponse; while (Date.now() < threadDeadline) { const threadsRequest = client.send("threads"); - runningThreads = (await client.response(threadsRequest, "threads")).body - .threads; + threadObservationResponse = await client.response(threadsRequest, "threads"); + runningThreads = threadObservationResponse.body.threads; if (runningThreads.length > 0) break; await delay(100); } assert(runningThreads.length > 0, "asynchronous live DAP launch reported no threads"); + const initialObserverFaults = []; + for (const expected of [ + { + fault: "task snapshot read transport failure", + diagnostic: "runtime snapshot observation failed", + }, + { + fault: "process-status read transport failure", + diagnostic: "runtime process observation failed", + }, + { + fault: "breakpoint inspection and fallback event transport failures", + diagnostic: "runtime breakpoint and fallback event observation failed", + }, + ]) { + const message = await client.waitFor( + (candidate) => + candidate.seq > configurationResponse.seq && + candidate.type === "event" && + candidate.event === "output" && + String(candidate.body?.output || "").includes(expected.diagnostic), + 120_000 + ); + initialObserverFaults.push({ + ...expected, + expected: "reconnect and eventually report live threads without a stop", + observed: String(message.body.output).trim(), + }); + } + const recoveredThreadsRequest = client.send("threads"); + const recoveredThreadsResponse = await client.response( + recoveredThreadsRequest, + "threads" + ); + assert( + recoveredThreadsResponse.body.threads.length > 0, + "observer faults removed threads after recovery" + ); + const prematureLaunchEvents = client.messages.filter( + (message) => + message.seq > configurationResponse.seq && + message.seq < recoveredThreadsResponse.seq && + message.type === "event" && + ["stopped", "terminated"].includes(message.event) + ); + assert.deepStrictEqual( + prematureLaunchEvents, + [], + "post-launch observer recovery fabricated a stop or terminated the launch" + ); + const postLaunchObservationRecoveryMs = + Date.now() - configurationStartedAt; const pauseStartedAt = Date.now(); const pause = client.send("pause", { threadId: runningThreads[0].id }); await client.response(pause, "pause"); @@ -3397,26 +4402,81 @@ async function runLiveDapEditRestart({ assert.match(inspected.body.result, /debug epoch|frozen/i); const breakpointStartedAt = Date.now(); - const setBreakpoints = client.send("setBreakpoints", { + const staleRevisionRequest = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: mainLine }], + }); + const newestRevisionRequest = client.send("setBreakpoints", { source: { path: sourcePath }, breakpoints: [{ line: taskLine }], }); - const breakpointResponse = await client.response(setBreakpoints, "setBreakpoints"); + const staleRevisionResponse = await client.response( + staleRevisionRequest, + "setBreakpoints" + ); + const newestRevisionResponse = await client.response( + newestRevisionRequest, + "setBreakpoints" + ); const breakpointResponseMs = Date.now() - breakpointStartedAt; assert( breakpointResponseMs < 2_000, `setBreakpoints blocked for ${breakpointResponseMs} ms` ); assert.deepStrictEqual( - breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), + staleRevisionResponse.body.breakpoints.map( + (breakpoint) => breakpoint.verified + ), [false] ); assert.deepStrictEqual( - breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.message), + newestRevisionResponse.body.breakpoints.map( + (breakpoint) => breakpoint.verified + ), + [false] + ); + assert.deepStrictEqual( + newestRevisionResponse.body.breakpoints.map( + (breakpoint) => breakpoint.message + ), ["Pending coordinator breakpoint installation"] ); + const newestRevisionInstalled = await client.waitFor( + (message) => + message.seq > newestRevisionResponse.seq && + message.type === "event" && + message.event === "breakpoint" && + message.body.reason === "changed" && + message.body.breakpoint?.verified === true && + message.body.breakpoint?.line === taskLine, + 30_000 + ); + await delay(1_000); + const staleRevisionEvents = client.messages.filter( + (message) => + message.seq > newestRevisionResponse.seq && + message.type === "event" && + message.event === "breakpoint" && + message.body.reason === "changed" && + message.body.breakpoint?.line === mainLine + ); + assert.deepStrictEqual( + staleRevisionEvents, + [], + "stale breakpoint revision completion overwrote the newest set" + ); + + const rejectedRevisionRequest = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: taskLine }], + }); + const rejectedRevisionResponse = await client.response( + rejectedRevisionRequest, + "setBreakpoints" + ); const rejectedBreakpoint = await client.waitFor( (message) => + message.seq > rejectedRevisionResponse.seq && message.type === "event" && message.event === "breakpoint" && message.body.reason === "changed" && @@ -3429,7 +4489,7 @@ async function runLiveDapEditRestart({ ); assert.strictEqual( rejectedBreakpoint.body.breakpoint.id, - breakpointResponse.body.breakpoints[0].id + rejectedRevisionResponse.body.breakpoints[0].id ); const retryBreakpoints = client.send("setBreakpoints", { source: { path: sourcePath }, @@ -3441,6 +4501,7 @@ async function runLiveDapEditRestart({ ); const installedBreakpoint = await client.waitFor( (message) => + message.seq > retryBreakpointResponse.seq && message.type === "event" && message.event === "breakpoint" && message.body.reason === "changed" && @@ -3458,7 +4519,7 @@ async function runLiveDapEditRestart({ const mainContinue = client.send("continue", { threadId: mainStop.body.threadId, }); - await client.response(mainContinue, "continue"); + const mainContinueResponse = await client.response(mainContinue, "continue"); const continueResponseMs = Date.now() - continueStartedAt; assert( continueResponseMs < 2_000, @@ -3471,9 +4532,11 @@ async function runLiveDapEditRestart({ message.event === "stopped" && message.body.reason === "breakpoint" ); + const realBreakpointMs = Date.now() - breakpointStartedAt; assert.strictEqual(taskStop.body.allThreadsStopped, true); const observerReconnect = await client.waitFor( (message) => + message.seq > mainContinueResponse.seq && message.type === "event" && message.event === "output" && String(message.body?.output || "").includes( @@ -3497,6 +4560,27 @@ async function runLiveDapEditRestart({ "observer reconnection fabricated a stopped target before the real breakpoint" ); assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId); + const reconnectDiagnostics = [ + "runtime observer injected one transient connection loss", + "runtime snapshot observation failed", + "runtime process observation failed", + "runtime breakpoint and fallback event observation failed", + "runtime Debug Epoch observation failed", + ].map((diagnostic) => { + const message = client.messages.find( + (candidate) => + candidate.seq > mainContinueResponse.seq && + candidate.seq < taskStop.seq && + candidate.type === "event" && + candidate.event === "output" && + String(candidate.body?.output || "").includes(diagnostic) + ); + assert( + message, + `observer reconnection evidence omitted diagnostic: ${diagnostic}` + ); + return String(message.body.output).trim(); + }); const stackTrace = client.send("stackTrace", { threadId: taskStop.body.threadId, @@ -3674,6 +4758,72 @@ async function runLiveDapEditRestart({ original_arguments_preserved: true, clean_vfs_boundary_used: true, breakpoint_install_failure_reported: true, + real_breakpoint: { + request: { + source: sourcePath, + line: taskLine, + runtime_backend: "live-services", + }, + expected: + "coordinator installation is verified before a real all-stop breakpoint event", + observed: { + installed_line: installedBreakpoint.body.breakpoint.line, + stop_reason: taskStop.body.reason, + stopped_line: taskFrames[0].line, + all_threads_stopped: taskStop.body.allThreadsStopped, + }, + duration_ms: Math.max(1, realBreakpointMs), + }, + debug_interaction: { + request: "Pause, inspect command_status, Continue, and Disconnect", + expected: "each control remains responsive during observer recovery", + observed: { + pause_response_ms: pauseResponseMs, + inspect_response_ms: inspectionResponseMs, + continue_response_ms: continueResponseMs, + disconnect_response_ms: disconnectResponseMs, + pause_stop_reason: mainStop.body.reason, + }, + duration_ms: Math.max( + 1, + pauseResponseMs + + inspectionResponseMs + + continueResponseMs + + disconnectResponseMs + ), + }, + post_launch_observation_recovery: { + fault_injections: initialObserverFaults, + expected: + "main_launched remains committed; reconnect; threads appear; no rollback or fabricated stop", + observed: { + threads: runningThreads, + premature_stop_or_termination_events: prematureLaunchEvents.length, + terminal_state: "completed after replacement attempt", + }, + duration_ms: postLaunchObservationRecoveryMs, + }, + observer_reconnection: { + fault_injections: reconnectDiagnostics, + expected: + "all recoverable reads reconnect before the real breakpoint stop", + observed: { + real_stop_reason: taskStop.body.reason, + false_stops_before_real_stop: 0, + }, + duration_ms: Date.now() - continueStartedAt, + }, + breakpoint_revisions: { + expected: "revision 2 remains authoritative after delayed revision 1", + delayed_revision: 1, + authoritative_revision: 2, + authoritative_line: newestRevisionInstalled.body.breakpoint.line, + stale_completion_events: staleRevisionEvents.length, + failure_revision: 3, + recovery_revision: 4, + observed: "newest revision verified; stale completion ignored", + duration_ms: breakpointInstallationMs, + }, observer_idle_request_rate_bound_per_second: 0.8, observer_reconnected_without_false_stop: true, observer_fallback_reconnected: true, @@ -3803,6 +4953,8 @@ async function runHostedHelloBuild({ terminal_state: released.state, artifact: artifact.artifact, digest: artifact.digest, + downloaded_bytes: download.local_download.bytes_written, + downloaded_sha256: sha256(fs.readFileSync(outputFile)), executable_output: output, }; } @@ -4058,7 +5210,7 @@ async function main() { } const sessionSecret = loginSession.cli_session_secret || loginSession.session_secret; - const loginDurationMs = Date.now() - loginStartedAt; + const loginDurationMs = Math.max(1, Date.now() - loginStartedAt); assert(sessionSecret, "hosted browser login omitted the CLI session credential"); const tenant = loginSession.tenant; const project = loginSession.project; @@ -4151,6 +5303,7 @@ async function main() { activeProcess: processId, }); + const nodeLifecycleStartedAt = Date.now(); const grant = runJson(clusterflux, ["node", "enroll", ...scope], { cwd: projectDir, }); @@ -4297,6 +5450,7 @@ async function main() { recovered_online: nodeDescriptor(nodeStatus, workerNode).online, offline_detection_ms: Date.now() - offlineStartedAt, persisted_identity_reused: true, + duration_ms: Math.max(1, Date.now() - nodeLifecycleStartedAt), }; const processStatus = runJson( @@ -4397,6 +5551,7 @@ async function main() { worker, }); + const processCancellationStartedAt = Date.now(); const restart = runJson( clusterflux, ["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"], @@ -4421,6 +5576,20 @@ async function main() { 120000 ); assert.strictEqual(releasedDap.state, "not_active"); + const processCancellationLifecycle = { + request: { + restart_process: dapEditRestart.process, + cancel_process: dapEditRestart.process, + }, + expected: + "explicit process cancellation terminates active participants and releases the slot", + observed: { + restart_accepted: restart.restart_request.accepted, + cancel_accepted: cancel.cancel_request.accepted, + terminal_state: releasedDap.state, + }, + duration_ms: Math.max(1, Date.now() - processCancellationStartedAt), + }; const identityWorkerNode = `identity-worker-${suffix}`; const identityWorkerGrant = runJson( @@ -4623,19 +5792,9 @@ async function main() { ), }, }); + const mainBeforeChildLifecycle = + agentCredentialSecurity.main_before_child_lifecycle; - runJson( - clusterflux, - [ - "process", - "abort", - ...scope, - "--process", - `vp-agent-security-${suffix}`, - "--yes", - ], - { cwd: projectDir } - ); await stopChild(worker.child); const liveSoak = await runLiveSoak({ clusterflux, @@ -4643,6 +5802,28 @@ async function main() { scope, securityNode, }); + const malformedIdentifiers = await runMalformedIdentifierSuite({ + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + user, + suffix, + securityNode, + bundle: { + digest: runReport.bundle_digest, + moduleBase64: fs + .readFileSync( + path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) + ) + .toString("base64"), + entryExport: runReport.entry_export, + entryStableId: runReport.entry_stable_id, + }, + releaseArtifact, + }); const revokedNodeCredential = await revokeSecurityNode({ clusterflux, projectDir, @@ -4653,12 +5834,6 @@ async function main() { sessionSecret, suffix, }); - const malformedIdentifiers = await runMalformedIdentifierSuite({ - sessionSecret, - tenant, - project, - user, - }); const hostedLoginIsolation = await runHostedLoginIsolationBeforeRestart(); const serviceRestart = await restartHostedServiceAndResume({ clusterflux, @@ -4710,38 +5885,419 @@ async function main() { ), ]; assert(concurrentCompileTasks.length >= 2); - const qualityPassed = - qualityGates?.private.status === "passed" && - qualityGates?.public.status === "passed"; const extensionAsset = manifest.assets.find((asset) => asset.name.endsWith(".vsix") ); + const requirement = ({ id, passed, evidence, durationMs }) => { + const result = { + id, + passed: passed === true, + evidence, + }; + if (Number.isFinite(durationMs) && durationMs > 0) { + result.duration_ms = durationMs; + } else if (strictFullRelease) { + throw new Error( + `strict release scenario ${id} omitted a measured positive duration` + ); + } + return result; + }; + const qualityCheck = (id) => qualityGates?.checks?.[id]; + const nodeEnrollmentEvidence = { + request: "one-time enrollment followed by a grant-free worker restart", + expected: "the persisted node credential is reused unchanged", + observed: { + used_enrollment_exchange: attach.boundary.used_enrollment_exchange, + restarted_node: secondReady.node, + credential_digest: credentialDigest, + liveness: nodeLivenessTransition, + service_restart_executed: serviceRestart.executed, + }, + }; + const loginEvidence = { + request: reuseSessionFile + ? "reuse a still-valid scoped browser-login session" + : "perform browser OIDC login", + expected: "server-owned default project persists and can be selected", + observed: { + fresh_login: !reuseSessionFile, + tenant, + project, + user, + received_default_project: receivedDefaultProject, + project_select_command: projectSelect.command, + }, + }; + const credentialSecurityEvidence = { + expected: + "forged, replayed, expired, body-modified, and revoked user/Node/Agent credentials are denied", + observed: { + user: userCredentialSecurity, + node: { + initial: securityNode.evidence, + revoked: revokedNodeCredential.denied, + }, + agent: { + replay: agentCredentialSecurity.replay, + expired: agentCredentialSecurity.expired, + forged: agentCredentialSecurity.forged, + body_modified: agentCredentialSecurity.body_modified, + revoked: agentCredentialSecurity.revoked, + }, + }, + }; + const credentialSecurityDurationMs = + userCredentialSecurity.duration_ms + + securityNode.duration_ms + + revokedNodeCredential.duration_ms + + agentCredentialSecurity.duration_ms; + const relaySecurityEvidence = { + expected: + "limits are scoped independently and abandoned transfer bytes remain charged without a shared soft lockout", + observed: { + spawn_quota: quotaPreallocation, + relay_accounting: bandwidthPreallocation, + emergency_disable: relayEmergencyDisable, + }, + }; + const relaySecurityDurationMs = + quotaPreallocation.duration_ms + + bandwidthPreallocation.duration_ms + + relayEmergencyDisable.duration_ms; const strictRequirementLedger = [ - { id: "01_formatting", passed: qualityPassed }, - { id: "02_clippy_warnings_denied", passed: qualityPassed }, - { id: "03_public_workspace_tests", passed: qualityGates?.public.status === "passed" }, - { id: "04_private_hosted_policy_locked_tests", passed: qualityGates?.private.status === "passed" }, - { id: "05_wasm_example_builds", passed: qualityPassed && helloBuild.executable_output === "hello from a real Clusterflux build" && recoveryBuild.original_join_completed === true }, - { id: "06_filtered_public_tree_build_and_tests", passed: qualityGates?.public.status === "passed" && manifest.source_tree_clean === true }, - { id: "07_final_vsix_checks", passed: Boolean(extensionAsset && manifest.release_candidate.extension_sha256 && extensionAsset.sha256 === manifest.release_candidate.extension_sha256) }, - { id: "08_browser_login_and_persistent_default_project", passed: Boolean(sessionSecret && tenant && project && projectSelect.command === "project select") }, - { id: "09_one_time_node_enrollment_and_restart", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode && serviceRestart.executed === true }, - { id: "10_real_hello_build_and_artifact_download", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" && download.local_download.bytes_written > 0 }, - { id: "11_recovery_retry_and_original_join", passed: recoveryBuild.original_join_completed === true && recoveryBuild.original_attempt !== recoveryBuild.replacement_attempt && recoveryBuild.terminal_state === "not_active" }, - { id: "12_dap_launch_without_breakpoint", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.terminal_state === "not_active" }, - { id: "13_real_breakpoint_installation_and_hit", passed: dapEditRestart.breakpoint_install_failure_reported === true && dapEditRestart.clean_vfs_boundary_used === true }, - { id: "14_attach_with_preconfigured_breakpoint", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true }, - { id: "15_continue_pause_inspect_disconnect", passed: agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 && dapEditRestart.observer_reconnected_without_false_stop === true }, - { id: "16_distinct_same_definition_instances", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.completion_order.length === 2 && concurrentCompileTasks.length >= 2 }, - { id: "17_real_podman_partial_debug_epoch", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === false && agentCredentialSecurity.partial_freeze.podman_paused_container_ids.length > 0 }, - { id: "18_post_main_launched_observation_recovery", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.terminal_state === "not_active" }, - { id: "19_fallback_and_debug_epoch_observer_reconnect", passed: dapEditRestart.observer_fallback_reconnected === true && dapEditRestart.observer_debug_epoch_wait_reconnected === true && dapEditRestart.observer_reconnected_without_false_stop === true }, - { id: "20_existing_process_rejection_preserves_process", passed: launchAttemptOwnership.existing_process_survived === true }, - { id: "21_precommit_launch_failure_scoped_cleanup", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process with matching launch_attempt" && droppedConnectionRollback.terminal_state === "not_active" }, - { id: "22_malformed_identifiers_preserve_service", passed: malformedIdentifiers.valid_after_every_rejection === true && malformedIdentifiers.rejected_requests.length === malformedIdentifiers.principals.length * malformedIdentifiers.forms.length }, - { id: "23_cross_tenant_access_denied", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true }, - { id: "24_forged_replayed_expired_revoked_credentials_denied", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) && securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true && agentCredentialSecurity.revoked.denied === true }, - { id: "25_relay_limits_and_abandoned_transfer_charging", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.independent_scope.unaffected_by_spawn_quota === true && bandwidthPreallocation.bytes_served_before_denial > 0 && bandwidthPreallocation.partial_abandon.ingress_delta > 0 && bandwidthPreallocation.partial_abandon.egress_delta > 0 && bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && bandwidthPreallocation.partial_abandon.reservation_released === true && bandwidthPreallocation.restart_persistence === true && relayEmergencyDisable.disabled.denied === true && relayEmergencyDisable.runtime_drop_in_removed === true }, + requirement({ + id: "01_formatting", + passed: qualityCheck("formatting")?.status === "passed", + evidence: qualityCheck("formatting"), + durationMs: qualityCheck("formatting")?.duration_ms, + }), + requirement({ + id: "02_clippy_warnings_denied", + passed: qualityCheck("clippy_warnings_denied")?.status === "passed", + evidence: qualityCheck("clippy_warnings_denied"), + durationMs: qualityCheck("clippy_warnings_denied")?.duration_ms, + }), + requirement({ + id: "03_public_workspace_tests", + passed: qualityCheck("public_workspace_tests")?.status === "passed", + evidence: qualityCheck("public_workspace_tests"), + durationMs: qualityCheck("public_workspace_tests")?.duration_ms, + }), + requirement({ + id: "04_private_hosted_policy_locked_tests", + passed: + qualityCheck("private_hosted_policy_locked_tests")?.status === + "passed", + evidence: qualityCheck("private_hosted_policy_locked_tests"), + durationMs: qualityCheck("private_hosted_policy_locked_tests") + ?.duration_ms, + }), + requirement({ + id: "05_wasm_example_builds", + passed: qualityCheck("wasm_example_builds")?.status === "passed", + evidence: qualityCheck("wasm_example_builds"), + durationMs: qualityCheck("wasm_example_builds")?.duration_ms, + }), + requirement({ + id: "06_filtered_public_tree_build_and_tests", + passed: + qualityCheck("filtered_public_tree_build_and_tests")?.status === + "passed", + evidence: qualityCheck("filtered_public_tree_build_and_tests"), + durationMs: qualityCheck("filtered_public_tree_build_and_tests") + ?.duration_ms, + }), + requirement({ + id: "07_final_vsix_checks", + passed: Boolean( + extensionAsset && + qualityCheck("vscode_extension_candidate")?.status === "passed" && + extensionAsset.sha256 === + manifest.release_candidate.extension_sha256 && + qualityCheck("vscode_extension_candidate")?.candidate_vsix + ?.sha256 === extensionAsset.sha256 + ), + evidence: qualityCheck("vscode_extension_candidate"), + durationMs: qualityCheck("vscode_extension_candidate")?.duration_ms, + }), + requirement({ + id: "08_browser_login_and_persistent_default_project", + passed: Boolean( + sessionSecret && + tenant && + project && + receivedDefaultProject && + projectSelect.command === "project select" + ), + evidence: loginEvidence, + durationMs: loginDurationMs, + }), + requirement({ + id: "09_one_time_node_enrollment_and_restart", + passed: + attach.boundary.used_enrollment_exchange === true && + secondReady.node === workerNode && + nodeLivenessTransition.persisted_identity_reused === true && + serviceRestart.executed === true, + evidence: nodeEnrollmentEvidence, + durationMs: nodeLivenessTransition.duration_ms, + }), + requirement({ + id: "10_real_hello_build_and_artifact_download", + passed: + helloBuild.executable_output === + "hello from a real Clusterflux build" && + helloBuild.downloaded_bytes > 0 && + helloBuild.downloaded_sha256 === helloBuild.digest, + evidence: helloBuild, + durationMs: helloBuild.duration_ms, + }), + requirement({ + id: "11_recovery_retry_and_original_join", + passed: + recoveryBuild.original_join_completed === true && + recoveryBuild.original_attempt !== + recoveryBuild.replacement_attempt && + recoveryBuild.terminal_state === "not_active", + evidence: recoveryBuild, + durationMs: recoveryBuild.duration_ms, + }), + requirement({ + id: "12_long_lived_coordinator_main", + passed: + longJoin.duration_seconds > 120 && + longJoin.terminal_state === "completed" && + longJoin.process_state === "not_active", + evidence: longJoin, + durationMs: longJoin.duration_ms, + }), + requirement({ + id: "13_main_completes_before_child_lifecycle", + passed: + mainBeforeChildLifecycle.observed.active_after_main === true && + mainBeforeChildLifecycle.observed.debug_state_after_main === + "debug_breakpoints" && + mainBeforeChildLifecycle.observed.child_terminal_state === + "completed" && + mainBeforeChildLifecycle.observed.final_process_state === + "not_active" && + mainBeforeChildLifecycle.observed.subsequent_run_status === + "main_launched" && + qualityCheck("process_lifecycle_regressions")?.status === "passed", + evidence: { + live: mainBeforeChildLifecycle, + compiled_failure_and_cancellation: + qualityCheck("process_lifecycle_regressions"), + }, + durationMs: + mainBeforeChildLifecycle.duration_ms + + (qualityCheck("process_lifecycle_regressions")?.duration_ms || 0), + }), + requirement({ + id: "14_dap_launch_without_breakpoint", + passed: + sameDefinitionIdentity.launch_without_breakpoint.observed.thread_id > + 0 && + sameDefinitionIdentity.terminal_state === "not_active", + evidence: sameDefinitionIdentity.launch_without_breakpoint, + durationMs: + sameDefinitionIdentity.launch_without_breakpoint.duration_ms, + }), + requirement({ + id: "15_real_breakpoint_installation_and_hit", + passed: + dapEditRestart.real_breakpoint.observed.stop_reason === + "breakpoint" && + dapEditRestart.real_breakpoint.observed.stopped_line === + dapEditRestart.real_breakpoint.request.line && + dapEditRestart.real_breakpoint.observed.all_threads_stopped === true, + evidence: dapEditRestart.real_breakpoint, + durationMs: dapEditRestart.real_breakpoint.duration_ms, + }), + requirement({ + id: "16_attach_with_preconfigured_breakpoint", + passed: + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .observed.initially_verified === false && + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .observed.installation_event_verified === true && + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .observed.stop_reason === "breakpoint" && + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .observed.stopped_line === + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .request.line, + evidence: + agentCredentialSecurity.attach_with_preconfigured_breakpoint, + durationMs: + agentCredentialSecurity.attach_with_preconfigured_breakpoint + .duration_ms, + }), + requirement({ + id: "17_breakpoint_revision_ordering", + passed: + dapEditRestart.breakpoint_revisions.authoritative_revision > + dapEditRestart.breakpoint_revisions.delayed_revision && + dapEditRestart.breakpoint_revisions.stale_completion_events === 0, + evidence: dapEditRestart.breakpoint_revisions, + durationMs: dapEditRestart.breakpoint_revisions.duration_ms, + }), + requirement({ + id: "18_continue_pause_inspect_disconnect", + passed: + dapEditRestart.debug_interaction.observed.pause_response_ms < + 2_000 && + dapEditRestart.debug_interaction.observed.inspect_response_ms < + 2_000 && + dapEditRestart.debug_interaction.observed.continue_response_ms < + 2_000 && + dapEditRestart.debug_interaction.observed.disconnect_response_ms < + 2_000, + evidence: dapEditRestart.debug_interaction, + durationMs: dapEditRestart.debug_interaction.duration_ms, + }), + requirement({ + id: "19_distinct_same_definition_instances", + passed: + sameDefinitionIdentity.thread_evidence.length === 2 && + sameDefinitionIdentity.completion_order.length === 2 && + sameDefinitionIdentity.thread_evidence[0].task_instance !== + sameDefinitionIdentity.thread_evidence[1].task_instance && + sameDefinitionIdentity.thread_evidence[0].attempt_id !== + sameDefinitionIdentity.thread_evidence[1].attempt_id && + concurrentCompileTasks.length >= 2, + evidence: sameDefinitionIdentity.same_definition_identity, + durationMs: + sameDefinitionIdentity.same_definition_identity.duration_ms, + }), + requirement({ + id: "20_real_podman_partial_debug_epoch", + passed: + agentCredentialSecurity.partial_freeze.partially_frozen === true && + agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === + false && + agentCredentialSecurity.partial_freeze.podman_paused_container_ids + .length > 0 && + agentCredentialSecurity.partial_freeze.resumed_participants.length > + 0, + evidence: agentCredentialSecurity.partial_freeze, + durationMs: agentCredentialSecurity.partial_freeze.elapsed_ms, + }), + requirement({ + id: "21_post_main_launched_observation_recovery", + passed: + sameDefinitionIdentity.post_main_launched_observation_recovery + .observed.diagnostics.length === 2 && + sameDefinitionIdentity.post_main_launched_observation_recovery + .observed + .premature_stop_or_termination_events === 0 && + sameDefinitionIdentity.post_main_launched_observation_recovery + .observed.main_thread.id > 0, + evidence: + sameDefinitionIdentity.post_main_launched_observation_recovery, + durationMs: + sameDefinitionIdentity.post_main_launched_observation_recovery + .duration_ms, + }), + requirement({ + id: "22_observer_reconnection", + passed: + dapEditRestart.observer_reconnection.fault_injections.length === 5 && + dapEditRestart.observer_reconnection.observed + .false_stops_before_real_stop === 0 && + dapEditRestart.observer_reconnection.observed.real_stop_reason === + "breakpoint", + evidence: dapEditRestart.observer_reconnection, + durationMs: dapEditRestart.observer_reconnection.duration_ms, + }), + requirement({ + id: "23_malformed_identifiers_preserve_service", + passed: + malformedIdentifiers.valid_after_every_rejection === true && + malformedIdentifiers.all_rejected_for_intended_reason === true && + malformedIdentifiers.valid_authenticated_action_after_suite === + true && + malformedIdentifiers.valid_signed_rendezvous_after_suite === true && + malformedIdentifiers.rejected_requests.every( + (entry) => + Number.isFinite(entry.duration_ms) && entry.duration_ms > 0 + ), + evidence: malformedIdentifiers, + durationMs: malformedIdentifiers.duration_ms, + }), + requirement({ + id: "24_cross_tenant_access_denied", + passed: + tenantIsolation.executed === true && + tenantIsolation.process_hidden === true && + tenantIsolation.node_hidden === true && + tenantIsolation.tasks_and_logs.denied === true && + tenantIsolation.debug.denied === true && + tenantIsolation.artifact_and_download.denied === true, + evidence: tenantIsolation, + durationMs: tenantIsolation.duration_ms, + }), + requirement({ + id: "25_forged_replayed_expired_revoked_credentials_denied", + passed: + Boolean( + userCredentialSecurity.expired && + userCredentialSecurity.forged && + userCredentialSecurity.revoked + ) && + securityNode.evidence.replay.denied === true && + securityNode.evidence.expired.denied === true && + securityNode.evidence.forged.denied === true && + revokedNodeCredential.denied.denied === true && + agentCredentialSecurity.revoked.denied === true, + evidence: credentialSecurityEvidence, + durationMs: credentialSecurityDurationMs, + }), + requirement({ + id: "26_relay_limits_and_abandoned_transfer_charging", + passed: + quotaPreallocation.denied_process_allocated === false && + quotaPreallocation.independent_scope + .unaffected_by_spawn_quota === true && + bandwidthPreallocation.bytes_served_before_denial > 0 && + bandwidthPreallocation.partial_abandon.ingress_delta > 0 && + bandwidthPreallocation.partial_abandon.egress_delta > 0 && + bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && + bandwidthPreallocation.partial_abandon.reservation_released === + true && + bandwidthPreallocation.restart_persistence === true && + relayEmergencyDisable.disabled.denied === true && + relayEmergencyDisable.runtime_drop_in_removed === true, + evidence: relaySecurityEvidence, + durationMs: relaySecurityDurationMs, + }), + requirement({ + id: "27_existing_process_rejection_preserves_process", + passed: launchAttemptOwnership.existing_process_survived === true, + evidence: launchAttemptOwnership, + durationMs: launchAttemptOwnership.duration_ms, + }), + requirement({ + id: "28_precommit_launch_failure_scoped_cleanup", + passed: + droppedConnectionRollback.cli_exit_status !== 0 && + droppedConnectionRollback.rollback === + "automatic CLI launch guard abort_process with matching launch_attempt" && + droppedConnectionRollback.terminal_state === "not_active", + evidence: droppedConnectionRollback, + durationMs: droppedConnectionRollback.duration_ms, + }), + requirement({ + id: "29_explicit_process_cancellation_cleanup", + passed: + processCancellationLifecycle.observed.restart_accepted === true && + processCancellationLifecycle.observed.cancel_accepted === true && + processCancellationLifecycle.observed.terminal_state === + "not_active", + evidence: processCancellationLifecycle, + durationMs: processCancellationLifecycle.duration_ms, + }), ]; const fullReleasePassed = strictFullRelease && @@ -4753,7 +6309,9 @@ async function main() { .replace(/^\d+_/, "") .replaceAll("_", " "), status: requirement.passed ? "passed" : "failed", - duration_ms: 0, + ...(requirement.duration_ms + ? { duration_ms: requirement.duration_ms } + : {}), })); const report = { @@ -4806,10 +6364,12 @@ async function main() { flagship_same_definition_task_instances: concurrentCompileTasks, repeated_park_wake: repeatedParkWake, long_join: longJoin, + main_before_child_lifecycle: mainBeforeChildLifecycle, live_dap_edit_restart: dapEditRestart, process_restart_requested: true, process_cancel_requested: true, dap_process_aborted: true, + process_cancellation_lifecycle: processCancellationLifecycle, tenant_isolation: tenantIsolation, credential_security: { user: userCredentialSecurity, diff --git a/scripts/dap-smoke.js b/scripts/dap-smoke.js index da74986..3ab3318 100644 --- a/scripts/dap-smoke.js +++ b/scripts/dap-smoke.js @@ -14,6 +14,55 @@ const { DapClient } = require("./dap-client"); sourceLines.findIndex((line) => line.includes("pub async fn build_main()")) + 1; assert(buildMainLine > 0, "flagship source must contain build_main"); + const hostileDapClient = new DapClient(); + try { + const initialize = hostileDapClient.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await hostileDapClient.response(initialize, "initialize"); + for (const processId of [ + "", + " ", + "bad\u0000process", + "bad process!", + "x".repeat(256), + ]) { + const malformedLaunch = hostileDapClient.send("launch", { + entry: "build", + project, + runtimeBackend: "local-services", + processId, + }); + const rejection = await hostileDapClient.failure( + malformedLaunch, + "launch" + ); + assert.match( + rejection.message, + /invalid DAP processId: ProcessId is invalid/, + `unexpected malformed DAP identifier response: ${JSON.stringify(rejection)}` + ); + } + const validLaunch = hostileDapClient.send("launch", { + entry: "build", + project, + runtimeBackend: "local-services", + processId: "vp-valid-after-malformed", + }); + await hostileDapClient.response(validLaunch, "launch"); + await hostileDapClient.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + await hostileDapClient.close(); + } catch (error) { + if (hostileDapClient.child.exitCode === null) { + hostileDapClient.child.kill("SIGKILL"); + } + throw error; + } + const asynchronousClient = new DapClient(); try { const initialize = asynchronousClient.send("initialize", { diff --git a/scripts/hostile-input-contract-smoke.js b/scripts/hostile-input-contract-smoke.js index f7402df..e2c8b25 100644 --- a/scripts/hostile-input-contract-smoke.js +++ b/scripts/hostile-input-contract-smoke.js @@ -3,9 +3,53 @@ const assert = require("assert"); const fs = require("fs"); const path = require("path"); +const { + agentIdentity, + signedAgentWorkflowRequest, +} = require("./agent-signing"); +const { + nodeIdentity, + signedNodeHeartbeat, +} = require("./node-signing"); const repo = path.resolve(__dirname, ".."); +const signingInstrumentAgent = agentIdentity( + "hostile-input-contract-agent", + "agent-hostile-input-contract" +); +const explicitlyEmptyAgentNonce = signedAgentWorkflowRequest( + signingInstrumentAgent, + { + type: "start_process", + tenant: "tenant", + project: "project", + actor_agent: "agent-hostile-input-contract", + process: "process", + launch_attempt: "attempt", + restart: false, + }, + { nonce: "" } +); +assert.strictEqual( + explicitlyEmptyAgentNonce.agent_signature.nonce, + "", + "Agent signing instrument replaced an explicitly empty hostile nonce" +); +const signingInstrumentNode = nodeIdentity( + "hostile-input-contract-node", + "node-hostile-input-contract" +); +assert.strictEqual( + signedNodeHeartbeat( + "node-hostile-input-contract", + signingInstrumentNode, + { nonce: "" } + ).nonce, + "", + "Node signing instrument replaced an explicitly empty hostile nonce" +); + function read(relativePath) { return fs.readFileSync(path.join(repo, relativePath), "utf8"); } diff --git a/scripts/node-signing.js b/scripts/node-signing.js index e50a27a..7a6a4e9 100644 --- a/scripts/node-signing.js +++ b/scripts/node-signing.js @@ -127,7 +127,7 @@ function nodeSignatureMessage( function signedNodeProof(node, identity, requestKind, request, options = {}) { const nonce = - options.nonce || + options.nonce ?? `${requestKind}-${process.pid}-${Date.now()}-${crypto .randomBytes(8) .toString("hex")}`; diff --git a/scripts/prepare-manual-github-release.js b/scripts/prepare-manual-github-release.js index 2712e79..6c91f8d 100755 --- a/scripts/prepare-manual-github-release.js +++ b/scripts/prepare-manual-github-release.js @@ -51,7 +51,7 @@ function copyVerifiedAsset(asset, destinationRoot) { return destination; } -function extractBinaries(archive, destinationRoot) { +function extractBinaries(archive, destinationRoot, expectedDigests) { const staging = path.join(destinationRoot, ".binary-extract"); fs.mkdirSync(staging, { recursive: true }); const extracted = spawnSync("tar", ["-xzf", archive, "-C", staging], { @@ -69,6 +69,11 @@ function extractBinaries(archive, destinationRoot) { const destination = path.join(destinationRoot, "binaries", name); fs.copyFileSync(source, destination); fs.chmodSync(destination, 0o755); + assert.strictEqual( + `sha256:${sha256(destination)}`, + expectedDigests[name], + `tested binary digest changed: ${name}` + ); copied.push(destination); } fs.rmSync(staging, { recursive: true, force: true }); @@ -87,11 +92,21 @@ function main() { assert.strictEqual(result.source_tree_digest, manifest.source_tree_digest); assert.deepStrictEqual(result.binary_digests, manifest.binary_digests); assert.strictEqual(result.scenario_skips.length, 0); - assert.strictEqual(result.strict_requirement_ledger.length, 25); + assert(result.strict_requirement_ledger.length >= 29); assert( - result.strict_requirement_ledger.every((requirement) => requirement.passed === true), + result.strict_requirement_ledger.every( + (requirement) => + requirement.passed === true && + Number.isFinite(requirement.duration_ms) && + requirement.duration_ms > 0 && + requirement.evidence + ), "final validation contains a failed launch requirement" ); + assert.strictEqual( + result.named_scenarios.length, + result.strict_requirement_ledger.length + ); const destinationRoot = path.resolve( process.env.CLUSTERFLUX_GITHUB_RELEASE_DIR || @@ -104,11 +119,26 @@ function main() { const copiedAssets = manifest.assets.map((asset) => copyVerifiedAsset(asset, destinationRoot) ); + const publicationGuidance = [ + JSON.stringify(manifest.notes || []), + ...copiedAssets + .filter((file) => file.endsWith(".md")) + .map((file) => fs.readFileSync(file, "utf8")), + ].join("\n"); + assert.doesNotMatch( + publicationGuidance, + /(?:download|upload|release downloads?)[^\n]*Forgejo/i, + "manual GitHub package contains Forgejo release-publication instructions" + ); const binaryArchive = copiedAssets.find((file) => path.basename(file).startsWith("clusterflux-public-binaries-") ); assert(binaryArchive, "final manifest omitted the public binary archive"); - const binaries = extractBinaries(binaryArchive, destinationRoot); + const binaries = extractBinaries( + binaryArchive, + destinationRoot, + manifest.binary_digests + ); const vsix = copiedAssets.find((file) => file.endsWith(".vsix")); assert(vsix, "final manifest omitted the tested VSIX"); assert.strictEqual( @@ -124,7 +154,6 @@ function main() { ); write(path.join(destinationRoot, "VSIX_SHA256"), `${sha256(vsix)} ${path.basename(vsix)}\n`); fs.copyFileSync(manifestPath, path.join(destinationRoot, "public-release-manifest.json")); - fs.copyFileSync(resultPath, path.join(destinationRoot, "final-result.json")); fs.copyFileSync(transcriptPath, path.join(destinationRoot, "VALIDATION_TRANSCRIPT.log")); write( path.join(destinationRoot, "RELEASE_NOTES.md"), @@ -138,6 +167,31 @@ function main() { "- Full Windows sandbox validation is deferred.\n" + "- Providers, billing, teams, managed compute, and operator-panel UI are not part of this release.\n" ); + const testedUploadAssets = [ + ...copiedAssets, + ...binaries, + ] + .sort() + .map((file) => ({ + file: path.relative(destinationRoot, file), + sha256: `sha256:${sha256(file)}`, + })); + write( + path.join(destinationRoot, "final-result.json"), + `${JSON.stringify( + { + ...result, + manual_release: { + source_revision: manifest.source_commit, + source_tree_digest: manifest.source_tree_digest, + tested_upload_assets: testedUploadAssets, + publication: "manual GitHub upload from this directory", + }, + }, + null, + 2 + )}\n` + ); const checksummed = [ ...copiedAssets, diff --git a/scripts/prepare-public-release.js b/scripts/prepare-public-release.js index 2568097..27fa754 100755 --- a/scripts/prepare-public-release.js +++ b/scripts/prepare-public-release.js @@ -632,10 +632,22 @@ function stageEvidenceAsset( } if ( !Array.isArray(liveResult.named_scenarios) || - liveResult.named_scenarios.length !== 25 || - liveResult.named_scenarios.some((scenario) => scenario.status !== "passed") + !Array.isArray(liveResult.strict_requirement_ledger) || + liveResult.named_scenarios.length !== + liveResult.strict_requirement_ledger.length || + liveResult.named_scenarios.length < 29 || + liveResult.named_scenarios.some( + (scenario) => + scenario.status !== "passed" || + !Number.isFinite(scenario.duration_ms) || + scenario.duration_ms <= 0 + ) || + new Set(liveResult.named_scenarios.map((scenario) => scenario.id)) + .size !== liveResult.named_scenarios.length ) { - throw new Error("all twenty-five named final live checks must pass"); + throw new Error( + "every named final live check must be unique, measured, and passed" + ); } if ( !liveResult.release_binding?.deployment || @@ -649,7 +661,17 @@ function stageEvidenceAsset( if ( !Array.isArray(liveResult.strict_requirement_ledger) || !liveResult.strict_requirement_ledger.length || - liveResult.strict_requirement_ledger.some((requirement) => requirement.passed !== true) + liveResult.strict_requirement_ledger.some( + (requirement) => + requirement.passed !== true || + !Number.isFinite(requirement.duration_ms) || + requirement.duration_ms <= 0 || + !requirement.evidence + ) || + liveResult.strict_requirement_ledger.some( + (requirement, index) => + requirement.id !== liveResult.named_scenarios[index].id + ) ) { throw new Error("the strict final requirement ledger must be complete and passed"); } @@ -685,8 +707,7 @@ function stageEvidenceAsset( evidence_packaged_at: packagedAt, }, release_commands: [ - "nix develop -c ./scripts/acceptance-private.sh", - "nix develop -c ./scripts/acceptance-public.sh", + "nix develop -c node scripts/release-quality-gates.js", "node scripts/cli-happy-path-live-smoke.js (strict production-shaped environment)", ], }; @@ -827,7 +848,7 @@ ${resolution} ## Install -1. Download the binary archive for your platform from the Forgejo release. +1. Download the binary archive for your platform from the manually published GitHub release. 2. Check the archive against \`SHA256SUMS\`. 3. Extract it and put \`bin/\` on your \`PATH\`. 4. Install \`clusterflux-vscode-*.vsix\` when you want the VS Code debugger: @@ -880,8 +901,8 @@ function writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution) { process.env.CLUSTERFLUX_PUBLIC_REPO_URL || "https://git.michelpaulissen.com/michel/clusterflux-public"; const releaseUrl = - process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || - "the Forgejo release attached to the public repository"; + process.env.CLUSTERFLUX_GITHUB_RELEASE_URL || + "the manually published GitHub release"; fs.writeFileSync( file, `# Clusterflux Release Notes @@ -1200,7 +1221,8 @@ function main() { public_repo_remote: process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || candidate?.public_repo_remote || null, public_tree_publish: publicTreePublish, - forgejo_release_url: process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || null, + github_release_url: process.env.CLUSTERFLUX_GITHUB_RELEASE_URL || null, + forgejo_release_url: null, default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, dns_publication_state: process.env.CLUSTERFLUX_DNS_PUBLICATION_STATE || "published", @@ -1239,10 +1261,18 @@ function main() { name: path.basename(asset), sha256: sha256File(asset), })), - notes: [ - "Upload the assets to the Forgejo Release for the filtered public repository.", - "The real service deployment and full e2e release remain separate acceptance evidence.", - ], + notes: + releaseStage === "final" + ? [ + "Publish manually to GitHub from the prepared manual release directory.", + "The manifest embeds the exact production-shaped validation evidence and candidate binding.", + "Forgejo release publication is not launch authority.", + ] + : [ + "This is a validation candidate, not a published release.", + "GitHub publication remains manual after final production-shaped validation.", + "Forgejo release publication is not launch authority.", + ], }; const manifestPath = path.join(outputRoot, "public-release-manifest.json"); diff --git a/scripts/public-release-preflight.js b/scripts/public-release-preflight.js index 8176d0a..53749b9 100755 --- a/scripts/public-release-preflight.js +++ b/scripts/public-release-preflight.js @@ -316,14 +316,27 @@ if (manifest.candidate_proxy_configuration_identity) { } assert( Array.isArray(finalResult.named_scenarios) && - finalResult.named_scenarios.length === 25 && - finalResult.named_scenarios.every((scenario) => scenario.status === "passed"), - "all twenty-five named final checks must be present and passed" + finalResult.named_scenarios.length >= 29 && + finalResult.named_scenarios.length === + finalResult.strict_requirement_ledger.length && + finalResult.named_scenarios.every( + (scenario) => + scenario.status === "passed" && + Number.isFinite(scenario.duration_ms) && + scenario.duration_ms > 0 + ), + "all named final checks must be present, measured, and passed" ); assert( Array.isArray(finalResult.strict_requirement_ledger) && finalResult.strict_requirement_ledger.length > 0 && - finalResult.strict_requirement_ledger.every((requirement) => requirement.passed === true), + finalResult.strict_requirement_ledger.every( + (requirement) => + requirement.passed === true && + Number.isFinite(requirement.duration_ms) && + requirement.duration_ms > 0 && + requirement.evidence + ), "the strict final requirement ledger must be present and passed" ); const finalTranscript = readArchiveMember( diff --git a/scripts/release-quality-gates.js b/scripts/release-quality-gates.js new file mode 100755 index 0000000..5a350da --- /dev/null +++ b/scripts/release-quality-gates.js @@ -0,0 +1,322 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const manifestPath = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || + path.join( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/release-candidate"), + "public-release-manifest.json" + ) +); +const evidencePath = path.resolve( + process.env.CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH || + path.join(repo, "target/acceptance/quality-gates.json") +); +const transcriptPath = path.resolve( + process.env.CLUSTERFLUX_QUALITY_GATE_TRANSCRIPT_PATH || + path.join(repo, "target/acceptance/quality-gates-transcript.txt") +); + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function sha256(file) { + return crypto + .createHash("sha256") + .update(fs.readFileSync(file)) + .digest("hex"); +} + +function commandText(command, args) { + return [command, ...args] + .map((value) => + /^[A-Za-z0-9_./:=+-]+$/.test(value) + ? value + : JSON.stringify(value) + ) + .join(" "); +} + +function main() { + assert(fs.existsSync(manifestPath), `missing release manifest ${manifestPath}`); + const manifest = readJson(manifestPath); + fs.mkdirSync(path.dirname(evidencePath), { recursive: true }); + fs.mkdirSync(path.dirname(transcriptPath), { recursive: true }); + const transcript = fs.openSync(transcriptPath, "w"); + const evidence = { + kind: "clusterflux-release-quality-gates", + source_commit: manifest.source_commit, + source_tree_digest: manifest.source_tree_digest, + public_tree_identity: manifest.public_tree_identity, + checks: {}, + }; + + const runGroup = (id, commands, extra = {}) => { + const startedAt = Date.now(); + const commandEvidence = []; + let status = "passed"; + for (const command of commands) { + const args = command.args || []; + const cwd = command.cwd || repo; + const text = commandText(command.command, args); + fs.writeSync(transcript, `\n[${id}] ${text}\n`); + const commandStartedAt = Date.now(); + const result = cp.spawnSync(command.command, args, { + cwd, + env: { ...process.env, ...(command.env || {}) }, + stdio: ["ignore", transcript, transcript], + }); + const durationMs = Math.max(1, Date.now() - commandStartedAt); + commandEvidence.push({ + command: text, + cwd, + exit_status: result.status, + duration_ms: durationMs, + }); + if (result.error || result.status !== 0) { + status = "failed"; + evidence.checks[id] = { + status, + duration_ms: Math.max(1, Date.now() - startedAt), + commands: commandEvidence, + error: result.error?.message || `command exited ${result.status}`, + ...extra, + }; + fs.writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + throw new Error(`${id} failed: ${text}`); + } + } + evidence.checks[id] = { + status, + duration_ms: Math.max(1, Date.now() - startedAt), + commands: commandEvidence, + ...extra, + }; + }; + + try { + runGroup("repository_structure", [ + { command: path.join(repo, "scripts/check-old-name.sh") }, + { command: "node", args: [path.join(repo, "scripts/check-docs.js")] }, + { command: path.join(repo, "scripts/check-code-size.sh") }, + ]); + runGroup("formatting", [ + { command: "cargo", args: ["fmt", "--all", "--check"] }, + { + command: "cargo", + args: [ + "fmt", + "--all", + "--manifest-path", + "private/hosted-policy/Cargo.toml", + "--check", + ], + }, + ]); + runGroup("clippy_warnings_denied", [ + { + command: "cargo", + args: ["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"], + }, + { + command: "cargo", + args: [ + "clippy", + "--manifest-path", + "private/hosted-policy/Cargo.toml", + "--all-targets", + "--", + "-D", + "warnings", + ], + }, + ]); + runGroup("public_workspace_tests", [ + { + command: "cargo", + args: ["test", "--workspace", "--all-targets"], + }, + { + command: "cargo", + args: ["build", "--workspace", "--all-targets"], + }, + ]); + runGroup("private_hosted_policy_locked_tests", [ + { + command: "cargo", + args: [ + "test", + "--locked", + "--manifest-path", + "private/hosted-policy/Cargo.toml", + "--all-targets", + ], + }, + ]); + runGroup("process_lifecycle_regressions", [ + { + command: "cargo", + args: [ + "test", + "-p", + "clusterflux-coordinator", + "completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot", + ], + }, + { + command: "cargo", + args: [ + "test", + "-p", + "clusterflux-coordinator", + "failed_main_aborts_unfinished_children_and_clears_process_debug_state", + ], + }, + { + command: "cargo", + args: [ + "test", + "-p", + "clusterflux-coordinator", + "service_cancels_whole_process_and_blocks_new_task_launches", + ], + }, + ]); + runGroup("wasm_example_builds", [ + { + command: "cargo", + args: [ + "build", + "-p", + "hello-build", + "--target", + "wasm32-unknown-unknown", + ], + }, + { + command: "cargo", + args: [ + "build", + "-p", + "recovery-build", + "--target", + "wasm32-unknown-unknown", + ], + }, + { + command: "cargo", + args: [ + "build", + "-p", + "runtime-conformance", + "--target", + "wasm32-unknown-unknown", + ], + }, + ]); + runGroup("filtered_public_tree_build_and_tests", [ + { command: path.join(repo, "scripts/verify-public-split.sh") }, + ]); + + const extensionAsset = manifest.assets.find((asset) => + asset.name.endsWith(".vsix") + ); + assert(extensionAsset, "candidate manifest omitted its VSIX"); + const extensionFile = path.resolve( + path.dirname(manifestPath), + extensionAsset.file + ); + assert(fs.existsSync(extensionFile), `missing candidate VSIX ${extensionFile}`); + const extensionDigest = sha256(extensionFile); + assert.strictEqual(extensionDigest, extensionAsset.sha256); + assert.strictEqual( + extensionDigest, + manifest.release_candidate.extension_sha256 + ); + const extractedVsix = fs.mkdtempSync( + path.join(os.tmpdir(), "clusterflux-tested-vsix-") + ); + try { + runGroup( + "vscode_extension_candidate", + [ + { + command: "npm", + args: ["ci", "--ignore-scripts"], + cwd: path.join(repo, "vscode-extension"), + }, + { + command: "unzip", + args: ["-q", extensionFile, "-d", extractedVsix], + }, + { + command: "node", + args: [path.join(repo, "scripts/vscode-extension-smoke.js")], + env: { + CLUSTERFLUX_VSCODE_EXTENSION_ROOT: path.join( + extractedVsix, + "extension" + ), + }, + }, + ], + { + candidate_vsix: { + file: extensionFile, + sha256: extensionDigest, + }, + } + ); + } finally { + fs.rmSync(extractedVsix, { recursive: true, force: true }); + } + + const publicCheckIds = [ + "repository_structure", + "formatting", + "clippy_warnings_denied", + "public_workspace_tests", + "process_lifecycle_regressions", + "wasm_example_builds", + "filtered_public_tree_build_and_tests", + "vscode_extension_candidate", + ]; + evidence.public = { + status: "passed", + duration_ms: publicCheckIds.reduce( + (total, id) => total + evidence.checks[id].duration_ms, + 0 + ), + independent_filtered_tree: true, + }; + evidence.private = { + status: "passed", + duration_ms: + evidence.checks.formatting.duration_ms + + evidence.checks.clippy_warnings_denied.duration_ms + + evidence.checks.private_hosted_policy_locked_tests.duration_ms, + }; + evidence.status = "passed"; + evidence.transcript = transcriptPath; + fs.writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + } finally { + fs.closeSync(transcript); + } + console.log(`Release quality gates passed: ${evidencePath}`); +} + +try { + main(); +} catch (error) { + console.error(error.stack || error.message); + process.exit(1); +} diff --git a/scripts/vscode-extension-smoke.js b/scripts/vscode-extension-smoke.js index 7eddcea..3a98e8d 100755 --- a/scripts/vscode-extension-smoke.js +++ b/scripts/vscode-extension-smoke.js @@ -5,16 +5,20 @@ const os = require("os"); const path = require("path"); const assert = require("assert"); -const extension = require("../vscode-extension/extension"); -const packageJson = require("../vscode-extension/package.json"); +const extensionRoot = path.resolve( + process.env.CLUSTERFLUX_VSCODE_EXTENSION_ROOT || + path.join(__dirname, "../vscode-extension") +); +const extension = require(path.join(extensionRoot, "extension.js")); +const packageJson = require(path.join(extensionRoot, "package.json")); const extensionSource = fs.readFileSync( - path.join(__dirname, "../vscode-extension/extension.js"), + path.join(extensionRoot, "extension.js"), "utf8" ); const repo = path.resolve(__dirname, ".."); assert.strictEqual(packageJson.main, "./extension.js"); -assert(fs.existsSync(path.join(__dirname, "../vscode-extension", packageJson.main))); +assert(fs.existsSync(path.join(extensionRoot, packageJson.main))); assert.deepStrictEqual(packageJson.dependencies || {}, {}); const root = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-vscode-")); fs.mkdirSync(path.join(root, "envs/linux"), { recursive: true }); @@ -157,7 +161,7 @@ assert( "package.json must contribute a Clusterflux activity-bar container" ); assert( - fs.existsSync(path.join(__dirname, "../vscode-extension/resources/clusterflux.svg")), + fs.existsSync(path.join(extensionRoot, "resources/clusterflux.svg")), "Clusterflux activity-bar icon must exist" ); const packageViewIds = packageJson.contributes.views.clusterflux.map((view) => view.id).sort();