diff --git a/CLUSTERFLUX_PUBLIC_TREE.json b/CLUSTERFLUX_PUBLIC_TREE.json index 55b2238..f968739 100644 --- a/CLUSTERFLUX_PUBLIC_TREE.json +++ b/CLUSTERFLUX_PUBLIC_TREE.json @@ -1,7 +1,7 @@ { "kind": "clusterflux-filtered-public-tree", - "source_commit": "e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d", - "release_name": "release-e47f9c27bbeb", + "source_commit": "ea887c8f56cd53985a1179b13e5f1b85c485f584", + "release_name": "release-ea887c8f56cd", "filtered_out": [ "private/**", "internal/**", diff --git a/crates/clusterflux-cli/src/node.rs b/crates/clusterflux-cli/src/node.rs index 917310d..902fa14 100644 --- a/crates/clusterflux-cli/src/node.rs +++ b/crates/clusterflux-cli/src/node.rs @@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result }; let heartbeat_request = json!({ "type": "node_heartbeat", + "tenant": &tenant, + "project": &project, "node": &plan.node, }); let heartbeat_signature = sign_node_request( diff --git a/crates/clusterflux-coordinator/src/durable.rs b/crates/clusterflux-coordinator/src/durable.rs index cd3e3e5..0cc8a66 100644 --- a/crates/clusterflux-coordinator/src/durable.rs +++ b/crates/clusterflux-coordinator/src/durable.rs @@ -33,6 +33,39 @@ pub struct NodeIdentityRecord { pub enrollment_scope: String, } +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct NodeScopeKey { + pub tenant: TenantId, + pub project: ProjectId, + pub node: NodeId, +} + +impl NodeScopeKey { + pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self { + Self { + tenant, + project, + node, + } + } + + pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self { + Self::new(tenant.clone(), project.clone(), node.clone()) + } + + pub fn credential_subject(&self) -> String { + format!( + "node:{}:{}:{}:{}:{}:{}", + self.tenant.as_str().len(), + self.tenant, + self.project.as_str().len(), + self.project, + self.node.as_str().len(), + self.node + ) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CredentialRecord { pub subject: String, @@ -107,7 +140,7 @@ pub struct DurableState { pub tenants: BTreeMap, pub users: BTreeMap, pub projects: BTreeMap, - pub node_identities: BTreeMap, + pub node_identities: BTreeMap, pub credentials: BTreeMap, pub cli_sessions: BTreeMap, pub source_provider_configs: diff --git a/crates/clusterflux-coordinator/src/lib.rs b/crates/clusterflux-coordinator/src/lib.rs index c0252cd..7776307 100644 --- a/crates/clusterflux-coordinator/src/lib.rs +++ b/crates/clusterflux-coordinator/src/lib.rs @@ -13,7 +13,7 @@ pub mod service; mod sessions; pub use durable::{ AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState, - DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, + DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; @@ -128,8 +128,9 @@ impl Coordinator { public_key: impl Into, enrollment_scope: impl Into, ) { + let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone()); self.durable.node_identities.insert( - node.clone(), + key, NodeIdentityRecord { id: node, tenant, @@ -181,10 +182,13 @@ impl Coordinator { public_key, credential.scope.clone(), ); + let subject = + NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node) + .credential_subject(); self.durable.credentials.insert( - format!("node:{node}"), + subject.clone(), CredentialRecord { - subject: format!("node:{node}"), + subject, tenant: credential.tenant.clone(), project: Some(credential.project.clone()), kind: credential.credential_kind.clone(), @@ -356,15 +360,12 @@ impl Coordinator { project: &ProjectId, process: &ProcessId, ) -> Result<(), CoordinatorError> { - let identity = self + if !self .durable .node_identities - .get(node) - .ok_or(CoordinatorError::UnknownNode)?; - if &identity.tenant != tenant || &identity.project != project { - return Err(CoordinatorError::Unauthorized( - "node identity is outside the requested tenant/project scope".to_owned(), - )); + .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) + { + return Err(CoordinatorError::UnknownNode); } if !self .active_processes @@ -379,14 +380,18 @@ impl Coordinator { pub fn reconnect_node( &mut self, + tenant: &TenantId, + project: &ProjectId, node: &NodeId, process: Option<(&ProcessId, u64)>, ) -> Result<(), CoordinatorError> { - let identity = self + if !self .durable .node_identities - .get(node) - .ok_or(CoordinatorError::UnknownNode)?; + .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) + { + return Err(CoordinatorError::UnknownNode); + } if let Some((process_id, stale_epoch)) = process { if stale_epoch != self.coordinator_epoch { @@ -395,11 +400,7 @@ impl Coordinator { current_epoch: self.coordinator_epoch, }); } - let key = ( - identity.tenant.clone(), - identity.project.clone(), - process_id.clone(), - ); + let key = (tenant.clone(), project.clone(), process_id.clone()); if let Some(active) = self.active_processes.get_mut(&key) { active.connected_nodes.insert(node.clone()); } @@ -416,23 +417,27 @@ impl Coordinator { let identity = self .durable .node_identities - .get(node) + .get(&NodeScopeKey::from_refs( + &context.tenant, + &context.project, + node, + )) .ok_or(CoordinatorError::UnknownNode)? .clone(); - if identity.tenant != context.tenant || identity.project != context.project { - return Err(CoordinatorError::Unauthorized( - "node credential is outside the signed-in tenant/project scope".to_owned(), - )); - } if !matches!(context.actor, Actor::User(_)) { return Err(CoordinatorError::Unauthorized( "node credential revocation requires a user identity".to_owned(), )); } - self.durable.node_identities.remove(node); - self.durable.credentials.remove(&format!("node:{node}")); - for active in self.active_processes.values_mut() { - active.connected_nodes.remove(node); + let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node); + self.durable.node_identities.remove(&key); + self.durable.credentials.remove(&key.credential_subject()); + for active in self + .active_processes + .values_mut() + .filter(|active| active.tenant == context.tenant && active.project == context.project) + { + active.connected_nodes.remove(&key.node); } Ok(identity) } @@ -580,8 +585,15 @@ impl Coordinator { .count() } - pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> { - self.durable.node_identities.get(id) + pub fn node_identity( + &self, + tenant: &TenantId, + project: &ProjectId, + id: &NodeId, + ) -> Option<&NodeIdentityRecord> { + self.durable + .node_identities + .get(&NodeScopeKey::from_refs(tenant, project, id)) } pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize { @@ -683,12 +695,24 @@ mod tests { .contains_key(&TenantId::from("tenant"))); assert!(restarted.durable.users.contains_key(&UserId::from("user"))); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted.node_identity(&NodeId::from("node")).is_some()); + assert!(restarted + .node_identity( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ) + .is_some()); + let node_subject = NodeScopeKey::new( + TenantId::from("tenant"), + ProjectId::from("project"), + NodeId::from("node"), + ) + .credential_subject(); assert_eq!( restarted .durable .credentials - .get("node:node") + .get(&node_subject) .map(|credential| &credential.kind), Some(&CredentialKind::NodeCredential) ); @@ -712,7 +736,12 @@ mod tests { ); assert_eq!(rerun.coordinator_epoch, 2); restarted - .reconnect_node(&NodeId::from("node"), Some((&process, 2))) + .reconnect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + Some((&process, 2)), + ) .unwrap(); assert!(restarted .active_process( @@ -725,6 +754,68 @@ mod tests { .contains(&NodeId::from("node"))); } + #[test] + fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() { + let store = InMemoryDurableStore::default(); + let mut coordinator = Coordinator::boot(&store, 1); + let node = NodeId::from("shared-node"); + let scopes = [ + (TenantId::from("tenant-a"), ProjectId::from("project-a")), + (TenantId::from("tenant-b"), ProjectId::from("project-b")), + (TenantId::from("tenant-a"), ProjectId::from("project-c")), + ]; + for (index, (tenant, project)) in scopes.iter().enumerate() { + let mut grant = coordinator.create_node_enrollment_grant( + tenant.clone(), + project.clone(), + format!("grant-{index}"), + "node:attach", + 100, + ); + coordinator + .exchange_node_enrollment_grant( + &mut grant, + node.clone(), + &format!("public-key-{index}"), + "node:attach", + 99, + ) + .unwrap(); + } + + let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node); + let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node); + let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node); + assert_ne!(scope_a.credential_subject(), scope_b.credential_subject()); + assert_ne!(scope_a.credential_subject(), scope_c.credential_subject()); + assert_eq!( + coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node), + coordinator.durable.node_identities.get(&scope_a) + ); + assert_eq!( + coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node), + coordinator.durable.node_identities.get(&scope_b) + ); + assert_eq!( + coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node), + coordinator.durable.node_identities.get(&scope_c) + ); + assert!(coordinator + .durable + .credentials + .contains_key(&scope_a.credential_subject())); + assert!(coordinator + .durable + .credentials + .contains_key(&scope_b.credential_subject())); + assert!(coordinator + .durable + .credentials + .contains_key(&scope_c.credential_subject())); + assert_eq!(coordinator.durable.node_identities.len(), 3); + assert_eq!(coordinator.durable.credentials.len(), 3); + } + #[test] fn identical_process_ids_are_isolated_by_tenant_and_project() { let store = InMemoryDurableStore::default(); @@ -773,11 +864,18 @@ mod tests { let mut restarted = Coordinator::boot(&store, 2); restarted - .reconnect_node(&NodeId::from("node"), None) + .reconnect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + None, + ) .unwrap(); let error = restarted .reconnect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -809,7 +907,13 @@ mod tests { .unwrap(); assert_eq!(credential.credential_kind, CredentialKind::NodeCredential); - assert!(coordinator.node_identity(&NodeId::from("node")).is_some()); + assert!(coordinator + .node_identity( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ) + .is_some()); } #[test] @@ -823,10 +927,16 @@ mod tests { "public-key", "node:attach", ); + let node_subject = NodeScopeKey::new( + TenantId::from("tenant"), + ProjectId::from("project"), + NodeId::from("node"), + ) + .credential_subject(); coordinator.durable.credentials.insert( - "node:node".to_owned(), + node_subject.clone(), CredentialRecord { - subject: "node:node".to_owned(), + subject: node_subject.clone(), tenant: TenantId::from("tenant"), project: Some(ProjectId::from("project")), kind: CredentialKind::NodeCredential, @@ -840,6 +950,8 @@ mod tests { ); coordinator .reconnect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -855,7 +967,7 @@ mod tests { &NodeId::from("node"), ) .unwrap_err(); - assert!(matches!(foreign, CoordinatorError::Unauthorized(_))); + assert!(matches!(foreign, CoordinatorError::UnknownNode)); let revoked = coordinator .revoke_node_credential( @@ -868,8 +980,14 @@ mod tests { ) .unwrap(); assert_eq!(revoked.id, NodeId::from("node")); - assert!(coordinator.node_identity(&NodeId::from("node")).is_none()); - assert!(!coordinator.durable.credentials.contains_key("node:node")); + assert!(coordinator + .node_identity( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ) + .is_none()); + assert!(!coordinator.durable.credentials.contains_key(&node_subject)); assert!(!coordinator .active_process( &TenantId::from("tenant"), @@ -1084,7 +1202,7 @@ mod tests { ) .unwrap_err(); - assert!(matches!(error, CoordinatorError::Unauthorized(_))); + assert!(matches!(error, CoordinatorError::UnknownNode)); } #[test] diff --git a/crates/clusterflux-coordinator/src/postgres_store.rs b/crates/clusterflux-coordinator/src/postgres_store.rs index 4ddfd02..18d5206 100644 --- a/crates/clusterflux-coordinator/src/postgres_store.rs +++ b/crates/clusterflux-coordinator/src/postgres_store.rs @@ -4,8 +4,8 @@ use thiserror::Error; use crate::{ AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord, - DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, - ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, + DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord, + ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; #[derive(Clone, Debug, PartialEq, Eq)] @@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore { state.projects.insert(record.id.clone(), record); } for record in self.query_records::( - "SELECT record FROM clusterflux_node_identities ORDER BY node_id", + "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id", )? { - state.node_identities.insert(record.id.clone(), record); + state.node_identities.insert( + NodeScopeKey::new( + record.tenant.clone(), + record.project.clone(), + record.id.clone(), + ), + record, + ); } for record in self.query_records::( "SELECT record FROM clusterflux_credentials ORDER BY subject", @@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects ( ); CREATE TABLE IF NOT EXISTS clusterflux_node_identities ( - node_id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE, - record JSONB NOT NULL + node_id TEXT NOT NULL, + record JSONB NOT NULL, + PRIMARY KEY (tenant_id, project_id, node_id) ); +DO $clusterflux_node_scope_migration$ +DECLARE + current_primary_key TEXT; + current_primary_key_columns TEXT[]; +BEGIN + IF EXISTS ( + SELECT 1 + FROM clusterflux_node_identities + WHERE record->>'id' IS DISTINCT FROM node_id + OR record->>'tenant' IS DISTINCT FROM tenant_id + OR record->>'project' IS DISTINCT FROM project_id + ) THEN + RAISE EXCEPTION + 'clusterflux node identity migration refused an internally inconsistent legacy row'; + END IF; + + SELECT + constraint_record.conname, + array_agg(attribute.attname ORDER BY key_column.ordinality) + INTO current_primary_key, current_primary_key_columns + FROM pg_constraint AS constraint_record + CROSS JOIN LATERAL unnest(constraint_record.conkey) + WITH ORDINALITY AS key_column(attribute_number, ordinality) + JOIN pg_attribute AS attribute + ON attribute.attrelid = constraint_record.conrelid + AND attribute.attnum = key_column.attribute_number + WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass + AND constraint_record.contype = 'p' + GROUP BY constraint_record.conname; + + IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN + EXECUTE format( + 'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I', + current_primary_key + ); + ALTER TABLE clusterflux_node_identities + ADD PRIMARY KEY (tenant_id, project_id, node_id); + ELSIF current_primary_key_columns + IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[] + THEN + RAISE EXCEPTION + 'clusterflux node identity migration found an unexpected primary key shape'; + END IF; +END +$clusterflux_node_scope_migration$; + CREATE TABLE IF NOT EXISTS clusterflux_credentials ( subject TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials ( record JSONB NOT NULL ); +DO $clusterflux_node_credential_migration$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM clusterflux_credentials + WHERE record->>'subject' IS DISTINCT FROM subject + OR record->>'tenant' IS DISTINCT FROM tenant_id + OR record->>'project' IS DISTINCT FROM project_id + ) THEN + RAISE EXCEPTION + 'clusterflux node credential migration refused an internally inconsistent legacy row'; + END IF; + + UPDATE clusterflux_credentials AS credential + SET subject = format( + 'node:%s:%s:%s:%s:%s:%s', + octet_length(identity.tenant_id), + identity.tenant_id, + octet_length(identity.project_id), + identity.project_id, + octet_length(identity.node_id), + identity.node_id + ), + record = jsonb_set( + credential.record, + '{subject}', + to_jsonb(format( + 'node:%s:%s:%s:%s:%s:%s', + octet_length(identity.tenant_id), + identity.tenant_id, + octet_length(identity.project_id), + identity.project_id, + octet_length(identity.node_id), + identity.node_id + )), + false + ) + FROM clusterflux_node_identities AS identity + WHERE credential.subject = 'node:' || identity.node_id + AND credential.tenant_id = identity.tenant_id + AND credential.project_id = identity.project_id; + + IF EXISTS ( + SELECT 1 + FROM clusterflux_credentials AS credential + WHERE credential.subject LIKE 'node:%' + AND NOT EXISTS ( + SELECT 1 + FROM clusterflux_node_identities AS identity + WHERE credential.tenant_id = identity.tenant_id + AND credential.project_id = identity.project_id + AND credential.subject = format( + 'node:%s:%s:%s:%s:%s:%s', + octet_length(identity.tenant_id), + identity.tenant_id, + octet_length(identity.project_id), + identity.project_id, + octet_length(identity.node_id), + identity.node_id + ) + ) + ) THEN + RAISE EXCEPTION + 'clusterflux node credential migration found an unscoped or orphaned node subject'; + END IF; +END +$clusterflux_node_credential_migration$; + CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions ( session_digest TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -529,7 +651,13 @@ mod tests { let restarted = Coordinator::try_boot(&mut store, 2).unwrap(); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted.node_identity(&NodeId::from("node")).is_some()); + assert!(restarted + .node_identity( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ) + .is_some()); assert_eq!(restarted.active_process_count(), 0); } diff --git a/crates/clusterflux-coordinator/src/service.rs b/crates/clusterflux-coordinator/src/service.rs index abef88d..c1ea1f0 100644 --- a/crates/clusterflux-coordinator/src/service.rs +++ b/crates/clusterflux-coordinator/src/service.rs @@ -13,7 +13,7 @@ use clusterflux_core::{ }; use thiserror::Error; -use crate::{Coordinator, CoordinatorError}; +use crate::{Coordinator, CoordinatorError, NodeScopeKey}; mod admin; mod artifacts; @@ -154,8 +154,8 @@ pub enum CoordinatorServiceError { pub struct CoordinatorService { coordinator: Coordinator, store: RuntimeDurableStore, - node_descriptors: BTreeMap, - node_last_seen_epoch_seconds: BTreeMap, + node_descriptors: BTreeMap, + node_last_seen_epoch_seconds: BTreeMap, node_stale_after_seconds: u64, debug_freeze_timeout: std::time::Duration, enrollment_grants: BTreeMap, @@ -180,7 +180,7 @@ pub struct CoordinatorService { process_cancellations: BTreeSet, process_aborts: BTreeSet, agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>, - node_replay_nonces: BTreeMap<(NodeId, String), u64>, + node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>, panel_snapshots: BTreeMap, stopped_panels: BTreeSet, panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>, @@ -284,15 +284,10 @@ impl CoordinatorService { { let identity = self .coordinator - .node_identity(node) + .node_identity(tenant, project, node) .ok_or(CoordinatorError::UnknownNode)?; - if &identity.tenant != tenant || &identity.project != project { - return Err(CoordinatorError::Unauthorized( - "node process-control request is outside its enrolled tenant/project scope" - .to_owned(), - ) - .into()); - } + debug_assert_eq!(&identity.tenant, tenant); + debug_assert_eq!(&identity.project, project); return Ok(()); } self.coordinator diff --git a/crates/clusterflux-coordinator/src/service/artifacts.rs b/crates/clusterflux-coordinator/src/service/artifacts.rs index c6698f9..1a94c7e 100644 --- a/crates/clusterflux-coordinator/src/service/artifacts.rs +++ b/crates/clusterflux-coordinator/src/service/artifacts.rs @@ -8,7 +8,7 @@ use clusterflux_core::{ }; use sha2::{Digest as _, Sha256}; -use crate::CoordinatorError; +use crate::{CoordinatorError, NodeScopeKey}; use super::relay::RelayFinishReason; use super::{ @@ -52,7 +52,11 @@ impl CoordinatorService { let action = self .artifact_registry .download_action(&context, &artifact, &policy)?; - self.ensure_download_source_connectivity(&action.source)?; + self.ensure_download_source_connectivity( + &context.tenant, + &context.project, + &action.source, + )?; let downloadable_size = self .artifact_registry .downloadable_size(&context, &artifact, &policy)?; @@ -163,7 +167,11 @@ impl CoordinatorService { }, &mut validation_meter, )?; - self.ensure_download_source_connectivity(&stream.link.source)?; + self.ensure_download_source_connectivity( + &stream.link.tenant, + &stream.link.project, + &stream.link.source, + )?; self.expire_artifact_reverse_transfers(now_epoch_seconds)?; if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() { @@ -302,7 +310,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&artifact) + .metadata(&context.tenant, &context.project, &artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let transfer_id = generate_opaque_token("artifact_transfer") .map_err(CoordinatorServiceError::Protocol)?; @@ -410,7 +418,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&artifact) + .metadata(&context.tenant, &context.project, &artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?; let destination = @@ -682,15 +690,10 @@ impl CoordinatorService { ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(node) + .node_identity(tenant, project, node) .ok_or(CoordinatorError::UnknownNode)?; - if &identity.tenant != tenant || &identity.project != project { - return Err(CoordinatorError::Unauthorized( - "artifact reverse transfer node is outside its enrolled tenant/project scope" - .to_owned(), - ) - .into()); - } + debug_assert_eq!(&identity.tenant, tenant); + debug_assert_eq!(&identity.project, project); Ok(()) } @@ -725,15 +728,12 @@ impl CoordinatorService { ) -> Result { let identity = self .coordinator - .node_identity(node) + .node_identity(tenant, project, node) .ok_or(CoordinatorError::UnknownNode)?; - if &identity.tenant != tenant || &identity.project != project { - return Err(CoordinatorError::Unauthorized( - "artifact export node is outside the tenant/project scope".to_owned(), - ) - .into()); - } - let descriptor = self.node_descriptors.get(node).ok_or_else(|| { + debug_assert_eq!(&identity.tenant, tenant); + debug_assert_eq!(&identity.project, project); + let node_scope = NodeScopeKey::from_refs(tenant, project, node); + let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} has not reported export connectivity" )) @@ -744,7 +744,7 @@ impl CoordinatorService { ) .into()); } - if !self.node_is_live(node) { + if !self.node_is_live(&node_scope) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} is offline for artifact export" @@ -769,17 +769,20 @@ impl CoordinatorService { fn ensure_download_source_connectivity( &self, + tenant: &TenantId, + project: &ProjectId, source: &StorageLocation, ) -> Result<(), clusterflux_core::DownloadError> { let StorageLocation::RetainedNode(node) = source else { return Ok(()); }; - let _descriptor = self.node_descriptors.get(node).ok_or_else(|| { + let node_scope = NodeScopeKey::from_refs(tenant, project, node); + let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} has not reported online status for artifact download" )) })?; - if !self.node_is_live(node) { + if !self.node_is_live(&node_scope) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} is offline for artifact download" diff --git a/crates/clusterflux-coordinator/src/service/debug_requests.rs b/crates/clusterflux-coordinator/src/service/debug_requests.rs index 7447b93..343f177 100644 --- a/crates/clusterflux-coordinator/src/service/debug_requests.rs +++ b/crates/clusterflux-coordinator/src/service/debug_requests.rs @@ -226,24 +226,25 @@ impl CoordinatorService { } else if !completed_event_observed { "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned() } else if let Some(checkpoint) = checkpoint { - let vfs_available = - checkpoint - .checkpoint - .vfs_manifest - .objects - .iter() - .all(|(path, object)| { - let artifact = super::keys::artifact_id_from_path(path); - self.artifact_registry - .metadata(&artifact) - .is_some_and(|metadata| { - metadata.tenant == tenant - && metadata.project == project - && metadata.digest == object.digest - && metadata.size == object.size - && !metadata.retaining_nodes.is_empty() - }) - }); + let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold( + true, + |available, (path, object)| { + let artifact = super::keys::artifact_id_from_path(path).map_err(|error| { + CoordinatorServiceError::InvalidArtifactPath(error.to_string()) + })?; + Ok::<_, CoordinatorServiceError>( + available + && self + .artifact_registry + .metadata(&tenant, &project, &artifact) + .is_some_and(|metadata| { + metadata.digest == object.digest + && metadata.size == object.size + && !metadata.retaining_nodes.is_empty() + }), + ) + }, + )?; if !vfs_available { "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned() } else { @@ -479,6 +480,7 @@ impl CoordinatorService { } self.record_task_completion_event(event.clone()); self.notify_coordinator_main_waiters(&event); + self.maybe_retire_terminal_process(&tenant, &project, &process)?; Ok(CoordinatorResponse::TaskFailureResolved { process, task, diff --git a/crates/clusterflux-coordinator/src/service/keys.rs b/crates/clusterflux-coordinator/src/service/keys.rs index 8cc82c5..2e14ba8 100644 --- a/crates/clusterflux-coordinator/src/service/keys.rs +++ b/crates/clusterflux-coordinator/src/service/keys.rs @@ -1,6 +1,7 @@ use clusterflux_core::{ ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath, }; +use thiserror::Error; pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId); pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId); @@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key( (tenant.clone(), project.clone(), grant.to_owned()) } -pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId { +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub(super) enum ArtifactPathError { + #[error("path must start with the exact /vfs/artifacts/ prefix")] + WrongPrefix, + #[error("path must name an artifact after /vfs/artifacts/")] + EmptyArtifact, + #[error("mapped artifact identifier is invalid: {0}")] + InvalidArtifactId(#[from] clusterflux_core::IdParseError), +} + +pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result { let value = path .as_str() .strip_prefix("/vfs/artifacts/") - .unwrap_or(path.as_str()) - .replace('/', ":"); - ArtifactId::new(value) + .ok_or(ArtifactPathError::WrongPrefix)?; + if value.is_empty() { + return Err(ArtifactPathError::EmptyArtifact); + } + ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn artifact_id_conversion_is_fallible_and_structural() { + assert_eq!( + artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(), + ArtifactId::from("build:output") + ); + for path in [ + "/vfs/other/output", + "/vfs/artifacts", + "/vfs/artifacts/bad artifact!", + ] { + let path = VfsPath::new(path).unwrap(); + assert!(artifact_id_from_path(&path).is_err(), "{path:?}"); + } + + let mapped = format!("/vfs/artifacts/{}", "x".repeat(256)); + assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err()); + } } diff --git a/crates/clusterflux-coordinator/src/service/logs.rs b/crates/clusterflux-coordinator/src/service/logs.rs index 0b53641..79e4ade 100644 --- a/crates/clusterflux-coordinator/src/service/logs.rs +++ b/crates/clusterflux-coordinator/src/service/logs.rs @@ -1,5 +1,5 @@ use clusterflux_core::{ - ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, + ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath, }; @@ -85,7 +85,9 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path), + id: artifact_id_from_path(path).map_err(|error| { + CoordinatorServiceError::InvalidArtifactPath(error.to_string()) + })?, tenant, project, process: process.clone(), @@ -203,7 +205,9 @@ impl CoordinatorService { event.placement = self.task_placements.remove(&task_key); if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path), + id: artifact_id_from_path(path).map_err(|error| { + CoordinatorServiceError::InvalidArtifactPath(error.to_string()) + })?, tenant: event.tenant.clone(), project: event.project.clone(), process: event.process.clone(), @@ -217,34 +221,16 @@ impl CoordinatorService { self.task_aborts.remove(&task_key); self.debug_commands.remove(&task_key); self.active_tasks.remove(&task_key); - let no_active_tasks = - !self - .active_tasks - .iter() - .any(|(task_tenant, task_project, task_process, _, _)| { - task_tenant == &event.tenant - && task_project == &event.project - && task_process == &event.process - }); - let completed_after_main = matches!(event.terminal_state, TaskTerminalState::Completed) - && self.task_events.iter().rev().any(|retained| { - retained.tenant == event.tenant - && retained.project == event.project - && retained.process == event.process - && matches!(retained.executor, super::TaskExecutor::CoordinatorMain) - && matches!(retained.terminal_state, TaskTerminalState::Completed) + self.task_assignments.retain(|_, assignments| { + assignments.retain(|assignment| { + assignment.tenant != event.tenant + || assignment.project != event.project + || assignment.process != event.process + || assignment.node != event.node + || assignment.task != event.task }); - if no_active_tasks { - self.process_aborts.remove(&process_key); - if (self.process_cancellations.remove(&process_key) || completed_after_main) - && !self.main_runtime.controls.contains_key(&process_key) - { - self.coordinator - .abort_process(&event.tenant, &event.project, &event.process)?; - self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process); - self.clear_operator_panel_state(&event.tenant, &event.project, &event.process); - } - } + !assignments.is_empty() + }); if process_was_aborted { let checkpoint_key = super::keys::task_restart_key( &event.tenant, @@ -261,6 +247,7 @@ impl CoordinatorService { if !awaiting_operator { self.notify_coordinator_main_waiters(&event); } + self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?; Ok(CoordinatorResponse::TaskRecorded { process: event.process, task: event.task, @@ -553,6 +540,82 @@ impl CoordinatorService { awaiting_operator } + pub(super) fn maybe_retire_terminal_process( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + ) -> Result { + let process_key = process_control_key(tenant, project, process); + if self.main_runtime.controls.contains_key(&process_key) { + return Ok(false); + } + + let has_runnable_remote_work = + self.active_tasks + .iter() + .any(|(task_tenant, task_project, task_process, _, _)| { + task_tenant == tenant && task_project == project && task_process == process + }) + || self.pending_task_launches.iter().any(|pending| { + &pending.tenant == tenant + && &pending.project == project + && &pending.process == process + }) + || self.task_assignments.values().any(|assignments| { + assignments.iter().any(|assignment| { + &assignment.tenant == tenant + && &assignment.project == project + && &assignment.process == process + }) + }) + || self.task_attempts.iter().any( + |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { + attempt_tenant == tenant + && attempt_project == project + && attempt_process == process + && attempts.iter().rev().any(|attempt| { + attempt.current + && matches!( + attempt.state, + TaskAttemptState::Queued + | TaskAttemptState::Running + | TaskAttemptState::FailedAwaitingAction + ) + }) + }, + ); + if has_runnable_remote_work { + return Ok(false); + } + + let main_completed = self.task_events.iter().rev().any(|event| { + &event.tenant == tenant + && &event.project == project + && &event.process == process + && matches!(event.executor, super::TaskExecutor::CoordinatorMain) + && matches!(event.terminal_state, TaskTerminalState::Completed) + }); + let cancellation_completed = self.process_cancellations.contains(&process_key); + if !main_completed && !cancellation_completed { + return Ok(false); + } + + self.process_aborts.remove(&process_key); + self.process_cancellations.remove(&process_key); + if self + .coordinator + .active_process(tenant, project, process) + .is_none() + { + return Ok(false); + } + self.coordinator.abort_process(tenant, project, process)?; + self.clear_debug_state_for_process(tenant, project, process); + self.clear_operator_panel_state(tenant, project, process); + Ok(true) + } + fn flush_artifact_metadata( &mut self, flush: ArtifactFlush, @@ -560,17 +623,25 @@ impl CoordinatorService { let now_epoch_seconds = self.current_epoch_seconds()?; self.artifact_registry .expire_download_links(now_epoch_seconds); - let pinned = self - .task_restart_checkpoints - .values() - .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter()) - .chain( - self.pending_task_launches - .iter() - .flat_map(|pending| pending.task_spec.required_artifacts.iter()), - ) - .cloned() - .collect::>(); + let mut pinned = std::collections::BTreeSet::new(); + for checkpoint in self.task_restart_checkpoints.values() { + for artifact in &checkpoint.assignment.task_spec.required_artifacts { + pinned.insert(ArtifactScopeKey::from_refs( + &checkpoint.assignment.tenant, + &checkpoint.assignment.project, + artifact, + )); + } + } + for pending in &self.pending_task_launches { + for artifact in &pending.task_spec.required_artifacts { + pinned.insert(ArtifactScopeKey::from_refs( + &pending.tenant, + &pending.project, + artifact, + )); + } + } self.artifact_registry .flush_metadata_bounded(flush, &pinned) .map(|_| ()) diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index add2826..735d2f1 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -971,16 +971,10 @@ impl CoordinatorService { } } let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process); - let has_active_children = - self.active_tasks - .iter() - .any(|(task_tenant, task_project, task_process, _, _)| { - task_tenant == &scope.tenant - && task_project == &scope.project - && task_process == &scope.process - }); self.main_runtime.controls.remove(&process_key); - if main_completed && has_active_children { + if main_completed { + let _ = + self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process); return; } for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() { @@ -1169,7 +1163,7 @@ mod tests { assert!(!service.main_runtime.controls.contains_key(&process_key)); assert!(!service.debug_epochs.contains_key(&process_key)); assert!(!service.debug_epoch_runtime.contains_key(&process_key)); - assert!(service.process_aborts.contains(&process_key)); + assert!(!service.process_aborts.contains(&process_key)); } #[test] diff --git a/crates/clusterflux-coordinator/src/service/nodes.rs b/crates/clusterflux-coordinator/src/service/nodes.rs index 9db62c8..b7a7b4e 100644 --- a/crates/clusterflux-coordinator/src/service/nodes.rs +++ b/crates/clusterflux-coordinator/src/service/nodes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ SourceProviderKind, TenantId, UserId, }; -use crate::CoordinatorError; +use crate::{CoordinatorError, NodeScopeKey}; use super::{ bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService, @@ -27,9 +27,9 @@ impl CoordinatorService { unix_timestamp_seconds() } - pub(super) fn node_is_live(&self, node: &NodeId) -> bool { + pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool { self.node_last_seen_epoch_seconds - .get(node) + .get(scope) .is_some_and(|last_seen| { self.liveness_now_epoch_seconds().saturating_sub(*last_seen) <= self.node_stale_after_seconds @@ -41,7 +41,11 @@ impl CoordinatorService { .values() .cloned() .map(|mut descriptor| { - descriptor.online = self.node_is_live(&descriptor.id); + descriptor.online = self.node_is_live(&NodeScopeKey::from_refs( + &descriptor.tenant, + &descriptor.project, + &descriptor.id, + )); descriptor }) .collect() @@ -165,7 +169,11 @@ impl CoordinatorService { !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds }); self.coordinator.ensure_tenant_active(&tenant)?; - if self.coordinator.node_identity(&node).is_none() { + if self + .coordinator + .node_identity(&tenant, &project, &node) + .is_none() + { self.quota.ensure_node_admission( &tenant, self.coordinator.node_identity_count_for_tenant(&tenant), @@ -197,12 +205,21 @@ impl CoordinatorService { pub(super) fn handle_node_heartbeat( &mut self, + tenant: String, + project: String, node: String, node_signature: Option, payload_digest: &Digest, ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); let node = NodeId::new(node); - self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?; + self.authenticate_node_request( + &NodeScopeKey::new(tenant, project, node.clone()), + node_signature, + "node_heartbeat", + payload_digest, + )?; Ok(CoordinatorResponse::NodeHeartbeat { node, epoch: self.coordinator.coordinator_epoch(), @@ -225,16 +242,13 @@ impl CoordinatorService { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let node = NodeId::new(node); + let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); let identity = self .coordinator - .node_identity(&node) + .node_identity(&tenant, &project, &node) .ok_or(CoordinatorError::UnknownNode)?; - if identity.tenant != tenant || identity.project != project { - return Err(CoordinatorError::Unauthorized( - "node capability report is outside the enrolled tenant/project scope".to_owned(), - ) - .into()); - } + debug_assert_eq!(identity.tenant, tenant); + debug_assert_eq!(identity.project, project); capabilities.validate_public_report()?; for (kind, count) in [ ("cached environments", cached_environment_digests.len()), @@ -274,12 +288,16 @@ impl CoordinatorService { .into_iter() .map(ArtifactId::new) .collect::>(); - self.artifact_registry - .reconcile_node_retention(&node, &artifact_locations); + self.artifact_registry.reconcile_node_retention( + &tenant, + &project, + &node, + &artifact_locations, + ); - let online = self.node_is_live(&node); + let online = self.node_is_live(&node_scope); self.node_descriptors.insert( - node.clone(), + node_scope, NodeDescriptor { id: node.clone(), tenant, @@ -333,9 +351,13 @@ impl CoordinatorService { actor: Actor::User(actor.clone()), }; self.coordinator.revoke_node_credential(&context, &node)?; - let descriptor_removed = self.node_descriptors.remove(&node).is_some(); - self.node_last_seen_epoch_seconds.remove(&node); - self.artifact_registry.garbage_collect_node(&node); + let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); + let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some(); + self.node_last_seen_epoch_seconds.remove(&node_scope); + self.node_replay_nonces + .retain(|(retained_scope, _), _| retained_scope != &node_scope); + self.artifact_registry + .garbage_collect_node(&tenant, &project, &node); let queued_assignments_removed = self .task_assignments .remove(&(tenant.clone(), project.clone(), node.clone())) @@ -369,14 +391,14 @@ impl CoordinatorService { pub(super) fn authenticate_node_request( &mut self, - node: &NodeId, + scope: &NodeScopeKey, node_signature: Option, request_kind: &str, payload_digest: &Digest, ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(node) + .node_identity(&scope.tenant, &scope.project, &scope.node) .ok_or(CoordinatorError::UnknownNode)?; let signature = node_signature.ok_or_else(|| { CoordinatorError::Unauthorized( @@ -401,7 +423,7 @@ impl CoordinatorService { ) .into()); } - let replay_key = (node.clone(), signature.nonce.clone()); + let replay_key = (scope.clone(), signature.nonce.clone()); self.node_replay_nonces.retain(|_, accepted_at| { now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS) }); @@ -413,7 +435,7 @@ impl CoordinatorService { } verify_node_request_signature( &identity.public_key, - node, + &scope.node, request_kind, payload_digest, &signature, @@ -422,7 +444,7 @@ impl CoordinatorService { if self .node_replay_nonces .keys() - .filter(|(retained_node, _)| retained_node == node) + .filter(|(retained_scope, _)| retained_scope == scope) .count() >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY { @@ -436,8 +458,8 @@ impl CoordinatorService { .insert(replay_key, now_epoch_seconds); let seen_at = self.liveness_now_epoch_seconds(); self.node_last_seen_epoch_seconds - .insert(node.clone(), seen_at); - if let Some(descriptor) = self.node_descriptors.get_mut(node) { + .insert(scope.clone(), seen_at); + if let Some(descriptor) = self.node_descriptors.get_mut(scope) { descriptor.online = true; } Ok(()) diff --git a/crates/clusterflux-coordinator/src/service/panels.rs b/crates/clusterflux-coordinator/src/service/panels.rs index 4e66042..9dd6a9a 100644 --- a/crates/clusterflux-coordinator/src/service/panels.rs +++ b/crates/clusterflux-coordinator/src/service/panels.rs @@ -254,7 +254,8 @@ impl CoordinatorService { .rev() .find_map(|event| event.artifact_path.as_ref()) { - let artifact = artifact_id_from_path(path); + let artifact = artifact_id_from_path(path) + .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?; let context = clusterflux_core::AuthContext { tenant, project, diff --git a/crates/clusterflux-coordinator/src/service/process_launch.rs b/crates/clusterflux-coordinator/src/service/process_launch.rs index 0a0a155..31fda49 100644 --- a/crates/clusterflux-coordinator/src/service/process_launch.rs +++ b/crates/clusterflux-coordinator/src/service/process_launch.rs @@ -118,14 +118,14 @@ impl CoordinatorService { let mut objects = BTreeMap::new(); let mut missing_required_artifact = false; for artifact in &task_spec.required_artifacts { - let Some(metadata) = self.artifact_registry.metadata(artifact) else { + let Some(metadata) = + self.artifact_registry + .metadata(&assignment.tenant, &assignment.project, artifact) + else { missing_required_artifact = true; continue; }; - if metadata.tenant != assignment.tenant - || metadata.project != assignment.project - || metadata.retaining_nodes.is_empty() - { + if metadata.retaining_nodes.is_empty() { missing_required_artifact = true; continue; } @@ -483,17 +483,14 @@ impl CoordinatorService { .validate() .map_err(CoordinatorServiceError::Protocol)?; for artifact in &task_spec.required_artifacts { - let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| { - CoordinatorError::Unauthorized(format!( - "required artifact {artifact} is unavailable or has expired" - )) - })?; - if metadata.tenant != tenant || metadata.project != project { - return Err(CoordinatorError::Unauthorized(format!( - "required artifact {artifact} is outside the task tenant/project scope" - )) - .into()); - } + let metadata = self + .artifact_registry + .metadata(&tenant, &project, artifact) + .ok_or_else(|| { + CoordinatorError::Unauthorized(format!( + "required artifact {artifact} is unavailable or has expired in this tenant/project scope" + )) + })?; if metadata.retaining_nodes.is_empty() { return Err(CoordinatorError::Unauthorized(format!( "required artifact {artifact} has no retaining node" diff --git a/crates/clusterflux-coordinator/src/service/processes.rs b/crates/clusterflux-coordinator/src/service/processes.rs index 7afaa44..54f2f32 100644 --- a/crates/clusterflux-coordinator/src/service/processes.rs +++ b/crates/clusterflux-coordinator/src/service/processes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId, }; -use crate::CoordinatorError; +use crate::{CoordinatorError, NodeScopeKey}; use super::keys::{process_control_key, task_control_key}; use super::{ @@ -47,14 +47,10 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&node) + .node_identity(&tenant, &project, &node) .ok_or(CoordinatorError::UnknownNode)?; - if identity.tenant != tenant || identity.project != project { - return Err(CoordinatorError::Unauthorized( - "task assignment poll is outside the enrolled tenant/project scope".to_owned(), - ) - .into()); - } + debug_assert_eq!(identity.tenant, tenant); + debug_assert_eq!(identity.project, project); let assignment_key = (tenant.clone(), project.clone(), node.clone()); let assignment = self .task_assignments @@ -77,7 +73,11 @@ impl CoordinatorService { project: &ProjectId, node: &NodeId, ) -> Result, CoordinatorServiceError> { - let Some(descriptor) = self.node_descriptors.get(node).cloned() else { + let Some(descriptor) = self + .node_descriptors + .get(&NodeScopeKey::from_refs(tenant, project, node)) + .cloned() + else { return Ok(None); }; let mut remaining = VecDeque::new(); @@ -233,20 +233,18 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&node) + .node_identity(&tenant, &project, &node) .ok_or(CoordinatorError::UnknownNode)?; - if identity.tenant != tenant || identity.project != project { - return Err(CoordinatorError::Unauthorized( - "source preparation completion is outside the enrolled tenant/project scope" - .to_owned(), - ) - .into()); - } - let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| { - CoordinatorError::Unauthorized( - "source preparation completion requires a node capability report".to_owned(), - ) - })?; + debug_assert_eq!(identity.tenant, tenant); + debug_assert_eq!(identity.project, project); + let descriptor = self + .node_descriptors + .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node)) + .ok_or_else(|| { + CoordinatorError::Unauthorized( + "source preparation completion requires a node capability report".to_owned(), + ) + })?; if !descriptor.source_snapshots.contains(&source_snapshot) && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND { @@ -413,14 +411,18 @@ impl CoordinatorService { pub(super) fn handle_reconnect_node( &mut self, + tenant: String, + project: String, node: String, process: String, epoch: u64, ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); let node = NodeId::new(node); let process = ProcessId::new(process); self.coordinator - .reconnect_node(&node, Some((&process, epoch)))?; + .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?; Ok(CoordinatorResponse::NodeReconnected { node, process }) } diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index 3c6e516..edaa988 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -121,6 +121,8 @@ pub enum CoordinatorRequest { enrollment_grant: String, }, NodeHeartbeat { + tenant: String, + project: String, node: String, #[serde(default)] node_signature: Option, @@ -269,6 +271,8 @@ pub enum CoordinatorRequest { restart: bool, }, ReconnectNode { + tenant: String, + project: String, node: String, process: String, epoch: u64, @@ -675,9 +679,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512) } CoordinatorRequest::NodeHeartbeat { + tenant, + project, node, node_signature, } => { + validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; if let Some(signature) = node_signature { validate_node_signature(signature, &format!("{path}.node_signature"))?; @@ -846,7 +853,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res &format!("{path}.launch_attempt"), ) } - CoordinatorRequest::ReconnectNode { node, process, .. } => { + CoordinatorRequest::ReconnectNode { + tenant, + project, + node, + process, + .. + } => { + validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_process(process, &format!("{path}.process")) } @@ -1567,6 +1581,8 @@ mod external_identifier_tests { restart: false, }, CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "bad node!".to_owned(), node_signature: Some(NodeSignedRequest { nonce: "node-nonce".to_owned(), @@ -1626,6 +1642,8 @@ mod external_identifier_tests { assert!(error.contains("malformed external token request.agent_signature.nonce")); let node_request = CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(NodeSignedRequest { nonce: String::new(), diff --git a/crates/clusterflux-coordinator/src/service/routing.rs b/crates/clusterflux-coordinator/src/service/routing.rs index 34ef214..d57af3f 100644 --- a/crates/clusterflux-coordinator/src/service/routing.rs +++ b/crates/clusterflux-coordinator/src/service/routing.rs @@ -274,9 +274,17 @@ impl CoordinatorService { enrollment_grant, ), CoordinatorRequest::NodeHeartbeat { + tenant, + project, node, node_signature, - } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest), + } => self.handle_node_heartbeat( + tenant, + project, + node, + node_signature, + &request_payload_digest, + ), CoordinatorRequest::SignedNode { node, node_signature, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index 10e18cd..19384df 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -1,6 +1,6 @@ -use clusterflux_core::{NodeId, NodeSignedRequest}; +use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId}; -use crate::CoordinatorError; +use crate::{CoordinatorError, NodeScopeKey}; use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError}; @@ -12,7 +12,7 @@ impl CoordinatorService { request: CoordinatorRequest, ) -> Result { let request_kind = signed_node_request_kind(&request)?; - let request_node = signed_node_request_node(&request)?; + let request_scope = signed_node_request_scope(&request)?; let request_payload = serde_json::to_value(&request).map_err(|error| { CoordinatorServiceError::Protocol(format!( "failed to canonicalize signed node request: {error}" @@ -22,14 +22,14 @@ impl CoordinatorService { let signed_node = NodeId::try_new(signed_node).map_err(|error| { CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}")) })?; - if request_node != signed_node { + if request_scope.node != signed_node { return Err(CoordinatorError::Unauthorized( "signed node request node does not match the wrapped request node".to_owned(), ) .into()); } self.authenticate_node_request( - &signed_node, + &request_scope, Some(node_signature), request_kind, &payload_digest, @@ -160,10 +160,12 @@ impl CoordinatorService { failure_reason, ), CoordinatorRequest::ReconnectNode { + tenant, + project, node, process, epoch, - } => self.handle_reconnect_node(node, process, epoch), + } => self.handle_reconnect_node(tenant, project, node, process, epoch), CoordinatorRequest::PollTaskControl { tenant, project, @@ -353,36 +355,129 @@ fn signed_node_request_kind( } } -fn signed_node_request_node( +fn signed_node_request_scope( request: &CoordinatorRequest, -) -> Result { +) -> Result { match request { - CoordinatorRequest::ReportNodeCapabilities { node, .. } - | CoordinatorRequest::PollTaskAssignment { node, .. } - | CoordinatorRequest::PollArtifactTransfer { node, .. } - | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } - | CoordinatorRequest::FailArtifactTransfer { node, .. } - | CoordinatorRequest::LaunchChildTask { node, .. } - | CoordinatorRequest::JoinChildTask { node, .. } - | CoordinatorRequest::CompleteSourcePreparation { node, .. } - | CoordinatorRequest::ReconnectNode { node, .. } - | CoordinatorRequest::PollTaskControl { node, .. } - | CoordinatorRequest::PollDebugCommand { node, .. } - | CoordinatorRequest::ReportDebugState { node, .. } - | CoordinatorRequest::ReportDebugProbeHit { node, .. } - | CoordinatorRequest::ReportTaskLog { node, .. } - | CoordinatorRequest::ReportVfsMetadata { node, .. } - | CoordinatorRequest::TaskCompleted { node, .. } => { - NodeId::try_new(node.clone()).map_err(|error| { - CoordinatorServiceError::Protocol(format!( - "invalid wrapped node identifier: {error}" - )) - }) + CoordinatorRequest::ReportNodeCapabilities { + tenant, + project, + node, + .. } - CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()), + | CoordinatorRequest::PollTaskAssignment { + tenant, + project, + node, + } + | CoordinatorRequest::PollArtifactTransfer { + tenant, + project, + node, + } + | CoordinatorRequest::UploadArtifactTransferChunk { + tenant, + project, + node, + .. + } + | CoordinatorRequest::FailArtifactTransfer { + tenant, + project, + node, + .. + } + | CoordinatorRequest::LaunchChildTask { + tenant, + project, + node, + .. + } + | CoordinatorRequest::JoinChildTask { + tenant, + project, + node, + .. + } + | CoordinatorRequest::CompleteSourcePreparation { + tenant, + project, + node, + .. + } + | CoordinatorRequest::ReconnectNode { + tenant, + project, + node, + .. + } + | CoordinatorRequest::PollTaskControl { + tenant, + project, + node, + .. + } + | CoordinatorRequest::PollDebugCommand { + tenant, + project, + node, + .. + } + | CoordinatorRequest::ReportDebugState { + tenant, + project, + node, + .. + } + | CoordinatorRequest::ReportDebugProbeHit { + tenant, + project, + node, + .. + } + | CoordinatorRequest::ReportTaskLog { + tenant, + project, + node, + .. + } + | CoordinatorRequest::ReportVfsMetadata { + tenant, + project, + node, + .. + } + | CoordinatorRequest::TaskCompleted { + tenant, + project, + node, + .. + } => node_scope_from_strings(tenant, project, node), + CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new( + scope.tenant.clone(), + scope.project.clone(), + source.node.clone(), + )), _ => Err(CoordinatorError::Unauthorized( "signed_node envelope only accepts node-originated coordinator requests".to_owned(), ) .into()), } } + +fn node_scope_from_strings( + tenant: &str, + project: &str, + node: &str, +) -> Result { + let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| { + CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}")) + })?; + let project = ProjectId::try_new(project.to_owned()).map_err(|error| { + CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}")) + })?; + let node = NodeId::try_new(node.to_owned()).map_err(|error| { + CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}")) + })?; + Ok(NodeScopeKey::new(tenant, project, node)) +} diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index e6e788e..3369b5b 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -49,6 +49,35 @@ fn test_admin_request( ) } +fn enroll_test_node( + service: &mut CoordinatorService, + tenant: &str, + project: &str, + node: &str, + public_key: &str, +) { + let response = service + .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { + tenant: tenant.to_owned(), + project: project.to_owned(), + actor_user: "test-user".to_owned(), + ttl_seconds: 900, + }) + .unwrap(); + let CoordinatorResponse::NodeEnrollmentGrantCreated { grant, .. } = response else { + panic!("expected node enrollment grant"); + }; + service + .handle_request(CoordinatorRequest::ExchangeNodeEnrollmentGrant { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + public_key: public_key.to_owned(), + enrollment_grant: grant, + }) + .unwrap(); +} + #[test] fn runtime_service_uses_memory_only_when_database_url_is_absent() { let service = CoordinatorService::new_with_database_url(1, None).unwrap(); @@ -63,7 +92,7 @@ fn runtime_service_uses_memory_only_when_database_url_is_absent() { } #[test] -fn postgres_backed_runtime_service_survives_restart_without_live_process_state() { +fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() { let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else { return; }; @@ -84,14 +113,21 @@ fn postgres_backed_runtime_service_survives_restart_without_live_process_state() None, ) .unwrap(); - first - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant-pg".to_owned(), - project: "project-pg".to_owned(), - node: "node-pg".to_owned(), - public_key: test_node_public_key("node-pg"), - }) - .unwrap(); + enroll_test_node( + &mut first, + "tenant-pg", + "project-pg", + "node-pg", + &test_node_public_key("node-pg"), + ); + let duplicate_private_key = test_node_private_key("node-pg-other-scope"); + enroll_test_node( + &mut first, + "tenant-pg-other", + "project-pg-other", + "node-pg", + &node_ed25519_public_key_from_private_key(&duplicate_private_key).unwrap(), + ); first .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), @@ -121,8 +157,12 @@ fn postgres_backed_runtime_service_survives_restart_without_live_process_state() )); let heartbeat = restarted .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-pg".to_owned(), + project: "project-pg".to_owned(), node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat( + node_signature: Some(signed_node_heartbeat_in_scope( + "tenant-pg", + "project-pg", "node-pg", "postgres-restart-heartbeat", )), @@ -132,6 +172,24 @@ fn postgres_backed_runtime_service_survives_restart_without_live_process_state() heartbeat, CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } )); + let duplicate_heartbeat = restarted + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-pg-other".to_owned(), + project: "project-pg-other".to_owned(), + node: "node-pg".to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + "tenant-pg-other", + "project-pg-other", + "node-pg", + &duplicate_private_key, + "postgres-restart-heartbeat", + )), + }) + .unwrap(); + assert!(matches!( + duplicate_heartbeat, + CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } + )); let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), @@ -142,6 +200,49 @@ fn postgres_backed_runtime_service_survives_restart_without_live_process_state() panic!("expected process list"); }; assert!(processes.is_empty()); + + restarted + .handle_request(CoordinatorRequest::RevokeNodeCredential { + tenant: "tenant-pg".to_owned(), + project: "project-pg".to_owned(), + actor_user: "user-pg".to_owned(), + node: "node-pg".to_owned(), + }) + .unwrap(); + drop(restarted); + + let mut after_revocation = + CoordinatorService::new_with_database_url(43, Some(&database_url)).unwrap(); + assert!(after_revocation + .coordinator + .node_identity( + &TenantId::from("tenant-pg"), + &ProjectId::from("project-pg"), + &NodeId::from("node-pg"), + ) + .is_none()); + assert!(after_revocation + .coordinator + .node_identity( + &TenantId::from("tenant-pg-other"), + &ProjectId::from("project-pg-other"), + &NodeId::from("node-pg"), + ) + .is_some()); + after_revocation + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-pg-other".to_owned(), + project: "project-pg-other".to_owned(), + node: "node-pg".to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + "tenant-pg-other", + "project-pg-other", + "node-pg", + &duplicate_private_key, + "post-revocation-restart", + )), + }) + .expect("the duplicate node in the other scope must remain authenticated"); } fn linux_capabilities() -> NodeCapabilities { @@ -393,6 +494,177 @@ fn register_test_task_assignment( )); } +fn service_with_completed_main_and_final_child( + failure_policy: clusterflux_core::TaskFailurePolicy, +) -> CoordinatorService { + let mut service = CoordinatorService::new(83); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + let node = NodeId::from("worker"); + let task = TaskInstanceId::from("final-child"); + + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: tenant.to_string(), + project: project.to_string(), + node: node.to_string(), + public_key: test_node_public_key(node.as_str()), + }) + .unwrap(); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: process.to_string(), + restart: false, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: tenant.to_string(), + project: project.to_string(), + node: node.to_string(), + process: process.to_string(), + epoch: 83, + }) + .unwrap(); + + let mut task_spec = test_task_spec_instance( + tenant.as_str(), + project.as_str(), + process.as_str(), + "child-definition", + task.as_str(), + 83, + [], + ); + task_spec.failure_policy = failure_policy; + let assignment = TaskAssignment { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + task: task.clone(), + node: node.clone(), + epoch: 83, + artifact_path: "/vfs/artifacts/final-child.bin".to_owned(), + task_spec: task_spec.clone(), + wasm_module_base64: test_wasm_module_base64(), + }; + service + .capture_task_restart_checkpoint(&assignment) + .unwrap(); + service + .begin_task_attempt( + &task_spec, + Some(node.clone()), + Some(&assignment.artifact_path), + false, + ) + .unwrap(); + service + .active_tasks + .insert(task_control_key(&tenant, &project, &process, &node, &task)); + service + .task_assignments + .entry((tenant.clone(), project.clone(), node.clone())) + .or_default() + .push_back(assignment); + service + .debug_epochs + .insert(process_control_key(&tenant, &project, &process), 11); + service.debug_breakpoints.insert( + process_control_key(&tenant, &project, &process), + super::debug::DebugBreakpointPlan { + actor: UserId::from("user"), + revision: 1, + probe_symbols: BTreeSet::from(["child-probe".to_owned()]), + hit_epoch: None, + hit_task: None, + hit_probe_symbol: None, + }, + ); + service.debug_commands.insert( + task_control_key(&tenant, &project, &process, &node, &task), + super::debug::DebugPendingCommand { + epoch: 11, + command: "continue".to_owned(), + }, + ); + let panel_key = super::keys::panel_stop_key(&tenant, &project, &process); + service.panel_snapshots.insert( + panel_key.clone(), + PanelState { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + widgets: BTreeMap::new(), + program_ui_events_enabled: false, + control_plane_actions: Vec::new(), + }, + ); + service.stopped_panels.insert(panel_key); + service.record_task_completion_event(TaskCompletionEvent { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + node: NodeId::from("coordinator-main"), + executor: TaskExecutor::CoordinatorMain, + task_definition: TaskDefinitionId::from("build"), + task: TaskInstanceId::from("main"), + attempt_id: None, + placement: None, + terminal_state: TaskTerminalState::Completed, + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }); + service + .coordinator + .grant_project_debug(tenant, project, UserId::from("user")); + service +} + +fn complete_terminal_matrix_child( + service: &mut CoordinatorService, + terminal_state: TaskTerminalState, +) { + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "terminal-matrix".to_owned(), + node: "worker".to_owned(), + task: "final-child".to_owned(), + terminal_state: Some(terminal_state), + status_code: Some(1), + stdout_bytes: 0, + stderr_bytes: 4, + stdout_tail: String::new(), + stderr_tail: "boom".to_owned(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }) + .unwrap(); +} + fn test_agent_public_key() -> String { agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap() } @@ -455,7 +727,21 @@ fn test_node_public_key(node: &str) -> String { } fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest { - let payload = json!({"type": "node_heartbeat", "node": node}); + signed_node_heartbeat_in_scope("tenant", "project", node, nonce) +} + +fn signed_node_heartbeat_in_scope( + tenant: &str, + project: &str, + node: &str, + nonce: &str, +) -> NodeSignedRequest { + let payload = json!({ + "type": "node_heartbeat", + "tenant": tenant, + "project": project, + "node": node + }); signed_node_request_with_private_key( node, &test_node_private_key(node), @@ -470,7 +756,22 @@ fn signed_node_heartbeat_with_private_key( private_key: &str, nonce: &str, ) -> NodeSignedRequest { - let payload = json!({"type": "node_heartbeat", "node": node}); + signed_node_heartbeat_in_scope_with_private_key("tenant", "project", node, private_key, nonce) +} + +fn signed_node_heartbeat_in_scope_with_private_key( + tenant: &str, + project: &str, + node: &str, + private_key: &str, + nonce: &str, +) -> NodeSignedRequest { + let payload = json!({ + "type": "node_heartbeat", + "tenant": tenant, + "project": project, + "node": node + }); signed_node_request_with_private_key( node, private_key, @@ -499,6 +800,33 @@ fn signed_node_request_with_private_key( } fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { + let node = match &request { + CoordinatorRequest::ReportNodeCapabilities { node, .. } + | CoordinatorRequest::PollTaskAssignment { node, .. } + | CoordinatorRequest::PollArtifactTransfer { node, .. } + | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } + | CoordinatorRequest::FailArtifactTransfer { node, .. } + | CoordinatorRequest::LaunchChildTask { node, .. } + | CoordinatorRequest::JoinChildTask { node, .. } + | CoordinatorRequest::CompleteSourcePreparation { node, .. } + | CoordinatorRequest::ReconnectNode { node, .. } + | CoordinatorRequest::PollTaskControl { node, .. } + | CoordinatorRequest::PollDebugCommand { node, .. } + | CoordinatorRequest::ReportDebugState { node, .. } + | CoordinatorRequest::ReportDebugProbeHit { node, .. } + | CoordinatorRequest::ReportTaskLog { node, .. } + | CoordinatorRequest::ReportVfsMetadata { node, .. } + | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), + CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), + _ => panic!("test helper only signs node-originated requests"), + }; + signed_node_request_auto_with_private_key(request, &test_node_private_key(&node)) +} + +fn signed_node_request_auto_with_private_key( + request: CoordinatorRequest, + private_key: &str, +) -> CoordinatorRequest { let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap()); let (node, request_kind) = match &request { CoordinatorRequest::ReportNodeCapabilities { node, .. } => { @@ -543,7 +871,7 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { node: node.clone(), node_signature: signed_node_request_with_private_key( &node, - &test_node_private_key(&node), + private_key, request_kind, &payload_digest, &format!("node-request-{nonce}"), @@ -949,6 +1277,8 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), node: "session-node".to_owned(), process: "vp-session".to_owned(), epoch: 7, @@ -2122,6 +2452,8 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2217,6 +2549,8 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")), }) @@ -2231,6 +2565,8 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2415,6 +2751,8 @@ fn service_revokes_node_credentials_and_live_descriptors() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")), }) @@ -2434,6 +2772,201 @@ fn service_revokes_node_credentials_and_live_descriptors() { assert!(descriptors.is_empty()); } +#[test] +fn duplicate_node_ids_are_isolated_across_identity_replay_liveness_and_revocation() { + let mut service = CoordinatorService::new(19); + let node = "shared-node"; + let private_a = test_node_private_key("tenant-a-shared-node"); + let private_b = test_node_private_key("tenant-b-shared-node"); + let private_c = test_node_private_key("tenant-a-other-project-shared-node"); + let public_a = node_ed25519_public_key_from_private_key(&private_a).unwrap(); + let public_b = node_ed25519_public_key_from_private_key(&private_b).unwrap(); + let public_c = node_ed25519_public_key_from_private_key(&private_c).unwrap(); + + for (tenant, project, public_key) in [ + ("tenant-a", "project-a", public_a), + ("tenant-b", "project-b", public_b), + ("tenant-a", "project-c", public_c), + ] { + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + public_key, + }) + .unwrap(); + } + assert_eq!( + service + .coordinator + .node_identity_count_for_tenant(&TenantId::from("tenant-a")), + 2 + ); + assert_eq!( + service + .coordinator + .node_identity_count_for_tenant(&TenantId::from("tenant-b")), + 1 + ); + + for (index, (tenant, project, private_key)) in [ + ("tenant-a", "project-a", private_a.as_str()), + ("tenant-b", "project-b", private_b.as_str()), + ("tenant-a", "project-c", private_c.as_str()), + ] + .into_iter() + .enumerate() + { + service.set_server_time(100 + index as u64); + service + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + tenant, + project, + node, + private_key, + "same-nonce", + )), + }) + .unwrap(); + service + .handle_request(signed_node_request_auto_with_private_key( + CoordinatorRequest::ReportNodeCapabilities { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: vec![Digest::sha256(format!( + "cache-{tenant}-{project}" + ))], + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: Vec::new(), + direct_connectivity: index != 1, + online: true, + }, + private_key, + )) + .unwrap(); + } + + let replay = service + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + node: node.to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + "tenant-a", + "project-a", + node, + &private_a, + "same-nonce", + )), + }) + .unwrap_err(); + assert!(replay.to_string().contains("nonce")); + + let forged = service + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-b".to_owned(), + project: "project-b".to_owned(), + node: node.to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + "tenant-b", + "project-b", + node, + &private_a, + "forged-cross-scope", + )), + }) + .unwrap_err(); + assert!(forged.to_string().contains("signature")); + + let scope_a = crate::NodeScopeKey::new( + TenantId::from("tenant-a"), + ProjectId::from("project-a"), + NodeId::from(node), + ); + let scope_b = crate::NodeScopeKey::new( + TenantId::from("tenant-b"), + ProjectId::from("project-b"), + NodeId::from(node), + ); + let scope_c = crate::NodeScopeKey::new( + TenantId::from("tenant-a"), + ProjectId::from("project-c"), + NodeId::from(node), + ); + assert!(service.node_descriptors.contains_key(&scope_a)); + assert!(service.node_descriptors.contains_key(&scope_b)); + assert!(service.node_descriptors.contains_key(&scope_c)); + assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_a)); + assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_b)); + assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_c)); + assert_eq!(service.node_last_seen_epoch_seconds[&scope_a], 100); + assert_eq!(service.node_last_seen_epoch_seconds[&scope_b], 101); + assert_eq!(service.node_last_seen_epoch_seconds[&scope_c], 102); + assert!(service.node_descriptors[&scope_a].direct_connectivity); + assert!(!service.node_descriptors[&scope_b].direct_connectivity); + assert!(service.node_descriptors[&scope_c].direct_connectivity); + assert!(service + .node_replay_nonces + .contains_key(&(scope_a.clone(), "same-nonce".to_owned()))); + assert!(service + .node_replay_nonces + .contains_key(&(scope_b.clone(), "same-nonce".to_owned()))); + assert!(service + .node_replay_nonces + .contains_key(&(scope_c.clone(), "same-nonce".to_owned()))); + + service + .handle_request(CoordinatorRequest::RevokeNodeCredential { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + actor_user: "user-a".to_owned(), + node: node.to_owned(), + }) + .unwrap(); + assert!(service + .coordinator + .node_identity( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + &NodeId::from(node), + ) + .is_none()); + assert!(service + .coordinator + .node_identity( + &TenantId::from("tenant-b"), + &ProjectId::from("project-b"), + &NodeId::from(node), + ) + .is_some()); + assert!(!service.node_descriptors.contains_key(&scope_a)); + assert!(service.node_descriptors.contains_key(&scope_b)); + assert!(service.node_descriptors.contains_key(&scope_c)); + + service + .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-b".to_owned(), + project: "project-b".to_owned(), + node: node.to_owned(), + node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( + "tenant-b", + "project-b", + node, + &private_b, + "post-other-scope-revocation", + )), + }) + .expect("revoking tenant A's duplicate node must not affect tenant B"); +} + #[test] fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() { let mut service = CoordinatorService::new(7); @@ -2461,6 +2994,8 @@ fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -3343,6 +3878,8 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() { for node in ["node-a", "node-b"] { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: node.to_owned(), process: "process".to_owned(), epoch: 7, @@ -3641,6 +4178,8 @@ fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_ .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: tenant.to_string(), + project: project.to_string(), node: "worker".to_owned(), process: process.to_string(), epoch: 31, @@ -3766,7 +4305,7 @@ fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_ })); let metadata = service .artifact_registry - .metadata(&ArtifactId::from("child-output")) + .metadata(&tenant, &project, &ArtifactId::from("child-output")) .expect("artifact metadata must survive terminal cleanup"); assert_eq!(metadata.process, process); assert_eq!(metadata.digest, Digest::sha256(artifact_bytes)); @@ -3787,6 +4326,302 @@ fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_ assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. })); } +#[test] +fn completed_main_terminal_matrix_retires_after_failed_or_cancelled_final_child() { + for terminal_state in [TaskTerminalState::Failed, TaskTerminalState::Cancelled] { + let mut service = service_with_completed_main_and_final_child( + clusterflux_core::TaskFailurePolicy::FailFast, + ); + complete_terminal_matrix_child(&mut service, terminal_state.clone()); + + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + assert!( + service + .coordinator + .active_process(&tenant, &project, &process) + .is_none(), + "{terminal_state:?} final child left the process slot active" + ); + assert!(!service + .debug_epochs + .contains_key(&process_control_key(&tenant, &project, &process))); + assert!(!service + .debug_breakpoints + .contains_key(&process_control_key(&tenant, &project, &process))); + assert!(service.debug_commands.keys().all( + |(task_tenant, task_project, task_process, _, _)| { + task_tenant != &tenant || task_project != &project || task_process != &process + } + )); + assert!(!service + .panel_snapshots + .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); + assert!(service.active_tasks.iter().all( + |(task_tenant, task_project, task_process, _, _)| { + task_tenant != &tenant || task_project != &project || task_process != &process + } + )); + assert!(!service + .main_runtime + .controls + .contains_key(&process_control_key(&tenant, &project, &process))); + let join = service.task_join_result( + tenant.clone(), + project.clone(), + process.clone(), + TaskInstanceId::from("final-child"), + ); + assert_eq!( + join.state, + match terminal_state { + TaskTerminalState::Failed => TaskJoinState::Failed, + TaskTerminalState::Cancelled => TaskJoinState::Cancelled, + TaskTerminalState::Completed => unreachable!(), + } + ); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "next-after-terminal".to_owned(), + restart: false, + }) + .expect("the terminal outcome must release the one-process project slot"); + } +} + +#[test] +fn completed_main_unpolled_final_assignment_completion_retires_process() { + let mut service = + service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + let node = NodeId::from("worker"); + let assignment_key = (tenant.clone(), project.clone(), node); + assert_eq!( + service + .task_assignments + .get(&assignment_key) + .map_or(0, VecDeque::len), + 1 + ); + + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: tenant.to_string(), + project: project.to_string(), + process: process.to_string(), + node: "worker".to_owned(), + task: "final-child".to_owned(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 2, + stderr_bytes: 0, + stdout_tail: "42".to_owned(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: Some(TaskBoundaryValue::SmallJson(json!(42))), + }) + .unwrap(); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + assert!(!service.task_assignments.contains_key(&assignment_key)); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "next-after-unpolled-completion".to_owned(), + restart: false, + }) + .expect("unpolled terminal completion must release the one-process slot"); +} + +#[test] +fn completed_main_await_operator_blocks_retirement_until_each_resolution() { + for resolution in [ + TaskFailureResolution::AcceptFailure, + TaskFailureResolution::Cancel, + ] { + let mut service = service_with_completed_main_and_final_child( + clusterflux_core::TaskFailurePolicy::AwaitOperator, + ); + complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); + + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + let process_key = process_control_key(&tenant, &project, &process); + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_some()); + assert!(service.debug_epochs.contains_key(&process_key)); + assert!(service.debug_breakpoints.contains_key(&process_key)); + assert!(service + .panel_snapshots + .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); + let attempt = service + .task_attempts + .get(&super::keys::task_restart_key( + &tenant, + &project, + &process, + &TaskInstanceId::from("final-child"), + )) + .and_then(|attempts| attempts.iter().rev().find(|attempt| attempt.current)) + .unwrap(); + assert_eq!(attempt.state, TaskAttemptState::FailedAwaitingAction); + + service + .handle_request(CoordinatorRequest::ResolveTaskFailure { + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: "user".to_owned(), + process: process.to_string(), + task: "final-child".to_owned(), + resolution, + }) + .unwrap(); + + assert!( + service + .coordinator + .active_process(&tenant, &project, &process) + .is_none(), + "{resolution:?} left the process slot active" + ); + assert!(!service.debug_epochs.contains_key(&process_key)); + assert!(!service.debug_breakpoints.contains_key(&process_key)); + assert!(!service + .panel_snapshots + .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); + assert!(service.debug_commands.keys().all( + |(task_tenant, task_project, task_process, _, _)| { + task_tenant != &tenant || task_project != &project || task_process != &process + } + )); + let join = service.task_join_result( + tenant.clone(), + project.clone(), + process.clone(), + TaskInstanceId::from("final-child"), + ); + assert_eq!( + join.state, + match resolution { + TaskFailureResolution::AcceptFailure => TaskJoinState::Failed, + TaskFailureResolution::Cancel => TaskJoinState::Cancelled, + } + ); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "next-after-resolution".to_owned(), + restart: false, + }) + .expect("operator resolution must release the one-process project slot"); + } +} + +#[test] +fn completed_main_failed_child_does_not_abort_another_active_child() { + let mut service = + service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); + register_test_task_assignment( + &mut service, + "tenant", + "project", + "terminal-matrix", + "worker", + "other-child-definition", + "other-child", + 83, + ); + + complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + let other_key = task_control_key( + &tenant, + &project, + &process, + &NodeId::from("worker"), + &TaskInstanceId::from("other-child"), + ); + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_some()); + assert!(service.active_tasks.contains(&other_key)); + assert!(!service.task_aborts.contains(&other_key)); + assert_eq!( + service + .task_join_result( + tenant.clone(), + project.clone(), + process.clone(), + TaskInstanceId::from("final-child"), + ) + .state, + TaskJoinState::Failed + ); + + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: tenant.to_string(), + project: project.to_string(), + process: process.to_string(), + node: "worker".to_owned(), + task: "other-child".to_owned(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }) + .unwrap(); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + assert!(!service.active_tasks.contains(&other_key)); +} + #[test] fn quiescent_cooperative_cancel_releases_slot_immediately() { let mut service = CoordinatorService::new(17); @@ -3918,6 +4753,8 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 17, @@ -4227,7 +5064,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_tenant.to_string().contains("tenant mismatch")); + assert!(cross_tenant.to_string().contains("does not exist")); let cross_project = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { @@ -4239,7 +5076,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_project.to_string().contains("project mismatch")); + assert!(cross_project.to_string().contains("does not exist")); let cross_tenant_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -4252,7 +5089,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_open.to_string().contains("tenant mismatch")); + assert!(cross_tenant_open.to_string().contains("does not exist")); let cross_project_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -4265,7 +5102,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_open.to_string().contains("project mismatch")); + assert!(cross_project_open.to_string().contains("does not exist")); let guessed = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -4521,6 +5358,8 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -4565,7 +5404,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_retry.to_string().contains("tenant mismatch")); + assert!(cross_tenant_retry.to_string().contains("does not exist")); let cross_project_retry = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -4578,7 +5417,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_retry.to_string().contains("project mismatch")); + assert!(cross_project_retry.to_string().contains("does not exist")); } #[test] @@ -5042,6 +5881,8 @@ fn coordinator_side_task_launch_queues_worker_assignment() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "worker-linux".to_owned(), process: "vp-control".to_owned(), epoch, @@ -5363,6 +6204,8 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 41, @@ -5690,6 +6533,8 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant-soak".to_owned(), + project: "project-soak".to_owned(), node: "worker-soak".to_owned(), process: "vp-soak".to_owned(), epoch: 73, @@ -5756,14 +6601,29 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { } for index in 0..2_000 { - service - .node_replay_nonces - .insert((NodeId::from("worker-soak"), format!("expired-{index}")), 0); + service.node_replay_nonces.insert( + ( + crate::NodeScopeKey::new( + TenantId::from("tenant-soak"), + ProjectId::from("project-soak"), + NodeId::from("worker-soak"), + ), + format!("expired-{index}"), + ), + 0, + ); } service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant-soak".to_owned(), + project: "project-soak".to_owned(), node: "worker-soak".to_owned(), - node_signature: Some(signed_node_heartbeat("worker-soak", "post-expiry-prune")), + node_signature: Some(signed_node_heartbeat_in_scope( + "tenant-soak", + "project-soak", + "worker-soak", + "post-expiry-prune", + )), }) .unwrap(); @@ -5816,7 +6676,14 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { let retained_soak_nonces = service .node_replay_nonces .keys() - .filter(|(node, _)| node == &NodeId::from("worker-soak")) + .filter(|(scope, _)| { + scope + == &crate::NodeScopeKey::new( + TenantId::from("tenant-soak"), + ProjectId::from("project-soak"), + NodeId::from("worker-soak"), + ) + }) .count(); assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS); @@ -5887,6 +6754,8 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 12, @@ -6007,6 +6876,8 @@ fn coordinator_rejects_named_environment_without_requirements() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "uncached-worker".to_owned(), process: "vp-environment".to_owned(), epoch, @@ -6317,6 +7188,143 @@ fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_author assert_eq!(plan.destination.node, NodeId::from("node-b")); } +#[test] +fn signed_hostile_artifact_paths_return_errors_and_the_same_service_stays_healthy() { + let mut service = CoordinatorService::new(27); + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: "node".to_owned(), + public_key: test_node_public_key("node"), + }) + .unwrap(); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "hostile-path-process".to_owned(), + restart: false, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: "node".to_owned(), + process: "hostile-path-process".to_owned(), + epoch: 27, + }) + .unwrap(); + + let invalid_metadata = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "hostile-path-process".to_owned(), + node: "node".to_owned(), + task: "metadata-task".to_owned(), + artifact_path: Some("/vfs/artifacts/bad artifact!".to_owned()), + artifact_digest: Some(Digest::sha256("bad")), + artifact_size_bytes: Some(3), + large_bytes_uploaded: false, + }) + .unwrap_err(); + assert!( + invalid_metadata + .to_string() + .contains("invalid VFS artifact path"), + "unexpected error: {invalid_metadata}" + ); + + let valid_metadata = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "hostile-path-process".to_owned(), + node: "node".to_owned(), + task: "metadata-task".to_owned(), + artifact_path: Some("/vfs/artifacts/valid-artifact".to_owned()), + artifact_digest: Some(Digest::sha256("valid")), + artifact_size_bytes: Some(5), + large_bytes_uploaded: false, + }) + .unwrap(); + assert!(matches!( + valid_metadata, + CoordinatorResponse::VfsMetadataRecorded { .. } + )); + + register_test_task_assignment( + &mut service, + "tenant", + "project", + "hostile-path-process", + "node", + "child", + "child-instance", + 27, + ); + let invalid_completion = service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "hostile-path-process".to_owned(), + node: "node".to_owned(), + task: "child-instance".to_owned(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: Some("/vfs/artifacts/repeated//component".to_owned()), + artifact_digest: Some(Digest::sha256("bad-completion")), + artifact_size_bytes: Some(0), + result: None, + }) + .unwrap_err(); + assert!( + invalid_completion + .to_string() + .contains("invalid VFS artifact path"), + "unexpected error: {invalid_completion}" + ); + + let valid_completion = service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "hostile-path-process".to_owned(), + node: "node".to_owned(), + task: "child-instance".to_owned(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: Some("/vfs/artifacts/valid-completion".to_owned()), + artifact_digest: Some(Digest::sha256("valid-completion")), + artifact_size_bytes: Some(0), + result: None, + }) + .unwrap(); + assert!(matches!( + valid_completion, + CoordinatorResponse::TaskRecorded { .. } + )); +} + #[test] fn service_rejects_task_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); @@ -6364,7 +7372,7 @@ fn service_rejects_task_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("outside")); + assert!(error.to_string().contains("not enrolled")); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant-b".to_owned(), @@ -6429,7 +7437,7 @@ fn service_rejects_node_capability_report_outside_enrollment_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("tenant/project scope")); + assert!(error.to_string().contains("not enrolled")); assert!(service.node_descriptors.is_empty()); } @@ -6455,7 +7463,7 @@ fn service_rejects_source_preparation_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("tenant/project scope")); + assert!(error.to_string().contains("not enrolled")); } #[test] @@ -6464,6 +7472,8 @@ fn service_rejects_unknown_node_heartbeat() { let error = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "missing".to_owned(), node_signature: None, }) @@ -6486,6 +7496,8 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let unsigned = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: None, }) @@ -6495,6 +7507,8 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let wrong_private_key = test_node_private_key("other-node"); let wrong_signature = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat_with_private_key( "node", @@ -6508,6 +7522,8 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let signed = signed_node_heartbeat("node", "fresh-node-heartbeat"); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed.clone()), }) @@ -6522,6 +7538,8 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let replay = service .handle_request(CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed), }) @@ -6592,6 +7610,8 @@ fn service_stream_accepts_multiple_requests_on_one_connection() { write_coordinator_wire_request( &mut stream, &CoordinatorRequest::NodeHeartbeat { + tenant: "tenant".to_owned(), + project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")), }, diff --git a/crates/clusterflux-core/src/artifact.rs b/crates/clusterflux-core/src/artifact.rs index f4f5311..240d5b4 100644 --- a/crates/clusterflux-core/src/artifact.rs +++ b/crates/clusterflux-core/src/artifact.rs @@ -134,6 +134,27 @@ pub struct ArtifactMetadata { pub coordinator_has_large_bytes: bool, } +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ArtifactScopeKey { + pub tenant: TenantId, + pub project: ProjectId, + pub artifact: ArtifactId, +} + +impl ArtifactScopeKey { + pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self { + Self { + tenant, + project, + artifact, + } + } + + pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self { + Self::new(tenant.clone(), project.clone(), artifact.clone()) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArtifactFlush { pub id: ArtifactId, @@ -148,7 +169,7 @@ pub struct ArtifactFlush { #[derive(Clone, Debug, Default)] pub struct ArtifactRegistry { - artifacts: BTreeMap, + artifacts: BTreeMap, issued_download_links: BTreeMap, next_epoch: u64, } @@ -162,9 +183,10 @@ impl ArtifactRegistry { pub fn flush_metadata_bounded( &mut self, flush: ArtifactFlush, - pinned: &BTreeSet, + pinned: &BTreeSet, ) -> Result { - let replacing_existing = self.artifacts.contains_key(&flush.id); + let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id); + let replacing_existing = self.artifacts.contains_key(&key); while !replacing_existing && self .artifacts @@ -184,13 +206,25 @@ impl ArtifactRegistry { metadata.tenant == flush.tenant && metadata.project == flush.project && metadata.process == flush.process - && !pinned.contains(&metadata.id) - && !self.issued_download_links.values().any(|issued| { - issued.link.artifact == metadata.id - }) + && !pinned.contains(&ArtifactScopeKey::from_refs( + &metadata.tenant, + &metadata.project, + &metadata.id, + )) + && !self.issued_download_links.values().any(|issued| { + issued.link.tenant == metadata.tenant + && issued.link.project == metadata.project + && issued.link.artifact == metadata.id + }) }) .min_by_key(|metadata| metadata.flushed_epoch) - .map(|metadata| metadata.id.clone()) + .map(|metadata| { + ArtifactScopeKey::from_refs( + &metadata.tenant, + &metadata.project, + &metadata.id, + ) + }) .ok_or_else(|| { "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download" .to_owned() @@ -212,36 +246,47 @@ impl ArtifactRegistry { explicit_locations: Vec::new(), coordinator_has_large_bytes: false, }; - self.artifacts.insert(flush.id, metadata.clone()); + self.artifacts.insert(key, metadata.clone()); Ok(metadata) } pub fn sync_to_explicit_store( &mut self, + tenant: &TenantId, + project: &ProjectId, artifact: &ArtifactId, location: impl Into, ) -> Result<(), ArtifactUnavailable> { let metadata = self .artifacts - .get_mut(artifact) + .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact)) .ok_or(ArtifactUnavailable)?; metadata.explicit_locations.push(location.into()); Ok(()) } - pub fn garbage_collect_node(&mut self, node: &NodeId) { - for metadata in self.artifacts.values_mut() { + pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) { + for metadata in self + .artifacts + .values_mut() + .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) + { metadata.retaining_nodes.remove(node); } } pub fn reconcile_node_retention( &mut self, + tenant: &TenantId, + project: &ProjectId, node: &NodeId, retained_artifacts: &BTreeSet, ) { - for (artifact, metadata) in &mut self.artifacts { - if retained_artifacts.contains(artifact) { + for (key, metadata) in &mut self.artifacts { + if &key.tenant != tenant || &key.project != project { + continue; + } + if retained_artifacts.contains(&key.artifact) { metadata.retaining_nodes.insert(node.clone()); } else { metadata.retaining_nodes.remove(node); @@ -249,8 +294,14 @@ impl ArtifactRegistry { } } - pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> { - self.artifacts.get(artifact) + pub fn metadata( + &self, + tenant: &TenantId, + project: &ProjectId, + artifact: &ArtifactId, + ) -> Option<&ArtifactMetadata> { + self.artifacts + .get(&ArtifactScopeKey::from_refs(tenant, project, artifact)) } pub fn download_action( @@ -261,7 +312,11 @@ impl ArtifactRegistry { ) -> Result { let metadata = self .artifacts - .get(artifact) + .get(&ArtifactScopeKey::from_refs( + &context.tenant, + &context.project, + artifact, + )) .ok_or(DownloadError::NotFound)?; let scope = Scope { tenant: metadata.tenant.clone(), @@ -315,7 +370,11 @@ impl ArtifactRegistry { self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(artifact) + .get(&ArtifactScopeKey::from_refs( + &context.tenant, + &context.project, + artifact, + )) .ok_or(DownloadError::NotFound)?; Ok(metadata.size) } @@ -333,7 +392,11 @@ impl ArtifactRegistry { if self .issued_download_links .values() - .filter(|issued| issued.link.artifact == *artifact) + .filter(|issued| { + issued.link.tenant == context.tenant + && issued.link.project == context.project + && issued.link.artifact == *artifact + }) .count() >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { @@ -345,7 +408,11 @@ impl ArtifactRegistry { let action = self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(artifact) + .get(&ArtifactScopeKey::from_refs( + &context.tenant, + &context.project, + artifact, + )) .ok_or(DownloadError::NotFound)?; let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds); let policy_context_digest = @@ -398,7 +465,11 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.artifact != *artifact || issued.link.actor != context.actor { + if issued.link.tenant != context.tenant + || issued.link.project != context.project + || issued.link.artifact != *artifact + || issued.link.actor != context.actor + { return Err(DownloadError::InvalidToken); } self.download_action( @@ -434,7 +505,9 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.artifact != *artifact + if issued.link.tenant != context.tenant + || issued.link.project != context.project + || issued.link.artifact != *artifact || issued.link.max_bytes != policy.max_bytes || issued.link.actor != context.actor { @@ -452,7 +525,11 @@ impl ArtifactRegistry { } let metadata = self .artifacts - .get(artifact) + .get(&ArtifactScopeKey::from_refs( + &context.tenant, + &context.project, + artifact, + )) .ok_or(DownloadError::NotFound)?; if download_policy_context_digest(metadata, &action.source, policy) != issued.link.policy_context_digest @@ -477,7 +554,11 @@ impl ArtifactRegistry { ) -> Result<(), DownloadError> { let metadata = self .artifacts - .get(&stream.link.artifact) + .get(&ArtifactScopeKey::from_refs( + &stream.link.tenant, + &stream.link.project, + &stream.link.artifact, + )) .ok_or(DownloadError::NotFound)?; if !source_is_available(metadata, &stream.link.source) { return Err(DownloadError::Unavailable); @@ -596,7 +677,13 @@ mod tests { #[test] fn flush_publishes_metadata_without_coordinator_bytes() { let registry = registry_with_artifact(); - let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); + let metadata = registry + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact"), + ) + .unwrap(); assert!(!metadata.coordinator_has_large_bytes); assert_eq!(metadata.id, ArtifactId::from("artifact")); @@ -615,7 +702,11 @@ mod tests { #[test] fn unsynced_node_loss_surfaces_as_unavailable() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node(&NodeId::from("node")); + registry.garbage_collect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -636,9 +727,20 @@ mod tests { #[test] fn signed_node_retention_inventory_removes_garbage_collected_location() { let mut registry = registry_with_artifact(); - registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new()); + registry.reconcile_node_retention( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + &BTreeSet::new(), + ); - let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); + let metadata = registry + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact"), + ) + .unwrap(); assert!(metadata.retaining_nodes.is_empty()); } @@ -648,11 +750,26 @@ mod tests { let mut registry = registry_with_artifact(); let node = NodeId::from("node"); let artifact = ArtifactId::from("artifact"); - registry.garbage_collect_node(&node); + registry.garbage_collect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &node, + ); - registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()])); + registry.reconcile_node_retention( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &node, + &BTreeSet::from([artifact.clone()]), + ); - let metadata = registry.metadata(&artifact).unwrap(); + let metadata = registry + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &artifact, + ) + .unwrap(); assert!(metadata.retaining_nodes.contains(&node)); } @@ -660,9 +777,18 @@ mod tests { fn explicit_user_storage_location_survives_node_retention_loss() { let mut registry = registry_with_artifact(); registry - .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app") + .sync_to_explicit_store( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact"), + "s3://bucket/app", + ) .unwrap(); - registry.garbage_collect_node(&NodeId::from("node")); + registry.garbage_collect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -683,7 +809,11 @@ mod tests { ); assert!( !registry - .metadata(&ArtifactId::from("artifact")) + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact"), + ) .unwrap() .coordinator_has_large_bytes ); @@ -714,7 +844,7 @@ mod tests { ) .unwrap_err(); - assert!(matches!(error, DownloadError::Unauthorized(_))); + assert_eq!(error, DownloadError::NotFound); } #[test] @@ -734,13 +864,206 @@ mod tests { ) .unwrap_err(); - assert!(matches!(error, DownloadError::Unauthorized(_))); + assert_eq!(error, DownloadError::NotFound); + } + + #[test] + fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() { + let artifact = ArtifactId::from("shared-artifact"); + let tenant_a = TenantId::from("tenant-a"); + let project_a = ProjectId::from("project-a"); + let tenant_b = TenantId::from("tenant-b"); + let project_b = ProjectId::from("project-b"); + let project_c = ProjectId::from("project-c"); + let node_a = NodeId::from("node-a"); + let node_b = NodeId::from("node-b"); + let node_c = NodeId::from("node-c"); + let mut registry = ArtifactRegistry::default(); + for (tenant, project, process, node, content, size) in [ + (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17), + (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29), + (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31), + ] { + registry.flush_metadata(ArtifactFlush { + id: artifact.clone(), + tenant: tenant.clone(), + project: project.clone(), + process: ProcessId::from(process), + producer_task: TaskInstanceId::from("producer"), + retaining_node: node.clone(), + digest: Digest::sha256(content), + size, + }); + } + + assert_eq!(registry.artifact_count(), 3); + assert_eq!( + registry + .metadata(&tenant_a, &project_a, &artifact) + .unwrap() + .digest, + Digest::sha256("bytes-a") + ); + assert_eq!( + registry + .metadata(&tenant_b, &project_b, &artifact) + .unwrap() + .digest, + Digest::sha256("bytes-b") + ); + assert_eq!( + registry + .metadata(&tenant_a, &project_c, &artifact) + .unwrap() + .digest, + Digest::sha256("bytes-c") + ); + registry.flush_metadata(ArtifactFlush { + id: artifact.clone(), + tenant: tenant_a.clone(), + project: project_a.clone(), + process: ProcessId::from("process-a"), + producer_task: TaskInstanceId::from("producer-repeat"), + retaining_node: node_a.clone(), + digest: Digest::sha256("bytes-a-repeat"), + size: 18, + }); + assert_eq!(registry.artifact_count(), 3); + assert_eq!( + registry + .metadata(&tenant_a, &project_a, &artifact) + .unwrap() + .digest, + Digest::sha256("bytes-a-repeat") + ); + assert_eq!( + registry + .metadata(&tenant_b, &project_b, &artifact) + .unwrap() + .digest, + Digest::sha256("bytes-b") + ); + + registry + .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a") + .unwrap(); + registry.garbage_collect_node(&tenant_a, &project_a, &node_a); + let context_a = AuthContext { + tenant: tenant_a.clone(), + project: project_a.clone(), + actor: Actor::User(UserId::from("user")), + }; + let context_b = AuthContext { + tenant: tenant_b.clone(), + project: project_b.clone(), + actor: Actor::User(UserId::from("user")), + }; + let policy = DownloadPolicy { max_bytes: 100 }; + assert_eq!( + registry + .download_action(&context_a, &artifact, &policy) + .unwrap() + .source, + StorageLocation::ExplicitStore("store://tenant-a".to_owned()) + ); + assert_eq!( + registry + .download_action(&context_b, &artifact, &policy) + .unwrap() + .source, + StorageLocation::RetainedNode(node_b) + ); + + let first_a = registry + .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60) + .unwrap(); + let first_b = registry + .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60) + .unwrap(); + assert_ne!( + first_a.scoped_token_digest, first_b.scoped_token_digest, + "the full artifact scope must contribute to download tokens" + ); + for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { + registry + .create_download_link( + &context_a, + &artifact, + &policy, + &format!("tenant-a-{index}"), + 10, + 60, + ) + .unwrap(); + } + assert!(matches!( + registry.create_download_link( + &context_a, + &artifact, + &policy, + "tenant-a-over-limit", + 10, + 60, + ), + Err(DownloadError::Usage(_)) + )); + registry + .create_download_link( + &context_b, + &artifact, + &policy, + "tenant-b-still-independent", + 10, + 60, + ) + .unwrap(); + registry + .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest) + .unwrap(); + + let limits = ResourceLimits { + limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]), + }; + let mut meter = ResourceMeter::default(); + registry + .open_download_stream( + DownloadStreamRequest { + context: &context_b, + artifact: &artifact, + policy: &policy, + presented_token_digest: &first_b.scoped_token_digest, + now_epoch_seconds: 11, + limits: &limits, + }, + &mut meter, + ) + .expect("revoking tenant A's link must not revoke tenant B's link"); + assert_eq!( + registry + .open_download_stream( + DownloadStreamRequest { + context: &context_b, + artifact: &artifact, + policy: &policy, + presented_token_digest: &first_a.scoped_token_digest, + now_epoch_seconds: 11, + limits: &limits, + }, + &mut meter, + ) + .unwrap_err(), + DownloadError::InvalidToken + ); } #[test] fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node(&NodeId::from("node")); + registry.garbage_collect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -1019,10 +1342,24 @@ mod tests { #[test] fn artifact_metadata_is_bounded_without_evicting_pins() { let mut registry = ArtifactRegistry::default(); + registry.flush_metadata(ArtifactFlush { + id: ArtifactId::from("artifact-0"), + tenant: TenantId::from("other-tenant"), + project: ProjectId::from("other-project"), + process: ProcessId::from("process"), + producer_task: TaskInstanceId::from("other-task"), + retaining_node: NodeId::from("other-node"), + digest: Digest::sha256("other-content"), + size: 1, + }); let mut pinned = BTreeSet::new(); for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS { let id = ArtifactId::new(format!("artifact-{index}")); - pinned.insert(id.clone()); + pinned.insert(ArtifactScopeKey::new( + TenantId::from("tenant"), + ProjectId::from("project"), + id.clone(), + )); registry .flush_metadata_bounded( ArtifactFlush { @@ -1039,7 +1376,10 @@ mod tests { ) .unwrap(); } - assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); + assert_eq!( + registry.artifact_count(), + MAX_ARTIFACT_METADATA_PER_PROCESS + 1 + ); let next = ArtifactFlush { id: ArtifactId::from("artifact-next"), tenant: TenantId::from("tenant"), @@ -1055,13 +1395,42 @@ mod tests { .unwrap_err() .contains("pinned")); - pinned.remove(&ArtifactId::from("artifact-0")); + pinned.remove(&ArtifactScopeKey::new( + TenantId::from("tenant"), + ProjectId::from("project"), + ArtifactId::from("artifact-0"), + )); registry.flush_metadata_bounded(next, &pinned).unwrap(); - assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); - assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none()); + assert_eq!( + registry.artifact_count(), + MAX_ARTIFACT_METADATA_PER_PROCESS + 1 + ); assert!(registry - .metadata(&ArtifactId::from("artifact-next")) + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact-0"), + ) + .is_none()); + assert!(registry + .metadata( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ArtifactId::from("artifact-next"), + ) .is_some()); + assert_eq!( + registry + .metadata( + &TenantId::from("other-tenant"), + &ProjectId::from("other-project"), + &ArtifactId::from("artifact-0"), + ) + .unwrap() + .digest, + Digest::sha256("other-content"), + "eviction and pins in one scope must not affect an equal ID in another scope" + ); } #[test] @@ -1150,7 +1519,11 @@ mod tests { registry .stream_download_chunk(&mut stream, &limits, &mut meter, 16) .unwrap(); - registry.garbage_collect_node(&NodeId::from("node")); + registry.garbage_collect_node( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &NodeId::from("node"), + ); assert_eq!( registry diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index fe1093b..b2f9188 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -20,8 +20,8 @@ pub mod wire; pub use artifact::{ ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry, - ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy, - DownloadStreamRequest, RetentionPolicy, StorageLocation, + ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, + DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation, }; pub use auth::{ admin_request_proof, admin_request_proof_from_token_digest, diff --git a/crates/clusterflux-core/src/vfs.rs b/crates/clusterflux-core/src/vfs.rs index 9e86e11..9f1b739 100644 --- a/crates/clusterflux-core/src/vfs.rs +++ b/crates/clusterflux-core/src/vfs.rs @@ -1,18 +1,48 @@ use std::collections::BTreeMap; -use serde::{Deserialize, Serialize}; +use serde::{de::Error as _, Deserialize, Deserializer, Serialize}; use thiserror::Error; use crate::{Digest, NodeId, TaskInstanceId}; -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub const MAX_VFS_PATH_BYTES: usize = 4096; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] pub struct VfsPath(String); impl VfsPath { pub fn new(path: impl Into) -> Result { let path = path.into(); if !path.starts_with("/vfs/") { - return Err(VfsError::InvalidPath(path)); + return Err(VfsError::invalid(path, "path must start with /vfs/")); + } + if path.len() > MAX_VFS_PATH_BYTES { + return Err(VfsError::invalid( + path, + "path exceeds the 4096-byte protocol limit", + )); + } + if path.chars().any(char::is_control) { + return Err(VfsError::invalid(path, "control characters are forbidden")); + } + if path.contains('\\') { + return Err(VfsError::invalid(path, "backslashes are forbidden")); + } + let relative = &path["/vfs/".len()..]; + if relative.is_empty() { + return Err(VfsError::invalid( + path, + "path after /vfs/ must not be empty", + )); + } + if relative + .split('/') + .any(|component| component.is_empty() || matches!(component, "." | "..")) + { + return Err(VfsError::invalid( + path, + "empty, '.', and '..' path components are forbidden", + )); } Ok(Self(path)) } @@ -22,6 +52,16 @@ impl VfsPath { } } +impl<'de> Deserialize<'de> for VfsPath { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let path = String::deserialize(deserializer)?; + Self::new(path).map_err(D::Error::custom) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VfsObject { pub path: VfsPath, @@ -63,12 +103,18 @@ pub enum ReuseDecision { #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum VfsError { - #[error("VFS path must start with /vfs/: {0}")] - InvalidPath(String), + #[error("invalid VFS path {path:?}: {reason}")] + InvalidPath { path: String, reason: &'static str }, #[error("path is not visible in the published VFS manifest: {0}")] NotVisible(String), } +impl VfsError { + fn invalid(path: String, reason: &'static str) -> Self { + Self::InvalidPath { path, reason } + } +} + #[derive(Clone, Debug)] pub struct VfsOverlay { task: TaskInstanceId, @@ -238,4 +284,39 @@ mod tests { assert_eq!(overlay.pending_len(), 0); } + + #[test] + fn vfs_path_rejects_the_complete_hostile_protocol_matrix() { + for invalid in [ + "vfs/artifacts/app".to_owned(), + "/vfs/".to_owned(), + "/vfs/artifacts//app".to_owned(), + "/vfs/artifacts/app/".to_owned(), + "/vfs/artifacts/./app".to_owned(), + "/vfs/artifacts/../app".to_owned(), + "/vfs/artifacts\\app".to_owned(), + "/vfs/artifacts/bad\0app".to_owned(), + format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)), + ] { + assert!( + VfsPath::new(&invalid).is_err(), + "hostile VFS path unexpectedly passed: {invalid:?}" + ); + assert!( + serde_json::from_value::(serde_json::json!(invalid)).is_err(), + "hostile VFS path unexpectedly deserialized" + ); + } + } + + #[test] + fn vfs_path_accepts_the_4096_byte_boundary() { + let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len())); + assert_eq!(path.len(), MAX_VFS_PATH_BYTES); + assert_eq!(VfsPath::new(&path).unwrap().as_str(), path); + + let overlong = format!("{path}x"); + assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1); + assert!(VfsPath::new(overlong).is_err()); + } } diff --git a/crates/clusterflux-node/src/daemon.rs b/crates/clusterflux-node/src/daemon.rs index f62c7c5..513fc83 100644 --- a/crates/clusterflux-node/src/daemon.rs +++ b/crates/clusterflux-node/src/daemon.rs @@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box> { let registration = establish_node_identity(&mut session, &args, &node_private_key)?; let heartbeat_request = json!({ "type": "node_heartbeat", + "tenant": &args.tenant, + "project": &args.project, "node": &args.node, }); let heartbeat_signature = sign_node_request( @@ -420,6 +422,8 @@ fn run_runtime_task( "reconnect_node", json!({ "type": "reconnect_node", + "tenant": &args.tenant, + "project": &args.project, "node": &args.node, "process": &task.process, "epoch": epoch, diff --git a/scripts/artifact-download-smoke.js b/scripts/artifact-download-smoke.js index fbb9813..f9e3c55 100755 --- a/scripts/artifact-download-smoke.js +++ b/scripts/artifact-download-smoke.js @@ -194,7 +194,7 @@ function downloadNodeCapabilities() { ttl_seconds: 60, }); assert.strictEqual(crossTenant.type, "error"); - assert.match(crossTenant.message, /tenant mismatch/); + assert.match(crossTenant.message, /artifact does not exist/); const crossProject = await send(addr, { type: "create_artifact_download_link", @@ -206,7 +206,7 @@ function downloadNodeCapabilities() { ttl_seconds: 60, }); assert.strictEqual(crossProject.type, "error"); - assert.match(crossProject.message, /project mismatch/); + assert.match(crossProject.message, /artifact does not exist/); const crossTenantOpen = await send(addr, { type: "open_artifact_download_stream", @@ -219,7 +219,7 @@ function downloadNodeCapabilities() { chunk_bytes: 1, }); assert.strictEqual(crossTenantOpen.type, "error"); - assert.match(crossTenantOpen.message, /tenant mismatch/); + assert.match(crossTenantOpen.message, /artifact does not exist/); const crossProjectOpen = await send(addr, { type: "open_artifact_download_stream", @@ -232,7 +232,7 @@ function downloadNodeCapabilities() { chunk_bytes: 1, }); assert.strictEqual(crossProjectOpen.type, "error"); - assert.match(crossProjectOpen.message, /project mismatch/); + assert.match(crossProjectOpen.message, /artifact does not exist/); const guessed = await send(addr, { type: "open_artifact_download_stream", diff --git a/scripts/artifact-export-smoke.js b/scripts/artifact-export-smoke.js index 8a26668..f33b0ba 100644 --- a/scripts/artifact-export-smoke.js +++ b/scripts/artifact-export-smoke.js @@ -231,7 +231,7 @@ async function reportNode( failure_reason: "", }); assert.strictEqual(crossTenant.type, "error"); - assert.match(crossTenant.message, /tenant mismatch/); + assert.match(crossTenant.message, /artifact does not exist/); await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); const failedDirect = await send(addr, { diff --git a/scripts/cli-happy-path-live-smoke.js b/scripts/cli-happy-path-live-smoke.js index 041baa4..9439f75 100644 --- a/scripts/cli-happy-path-live-smoke.js +++ b/scripts/cli-happy-path-live-smoke.js @@ -81,17 +81,12 @@ function requireEnabled() { const hasSecondTenantSession = Boolean( process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE ); - const hasSingleAccountIsolationTarget = Boolean( - process.env.CLUSTERFLUX_ISOLATION_TARGET_TENANT && - process.env.CLUSTERFLUX_ISOLATION_TARGET_PROJECT - ); if ( strictFullRelease && - !hasSecondTenantSession && - !hasSingleAccountIsolationTarget + !hasSecondTenantSession ) { throw new Error( - "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires either CLUSTERFLUX_SECOND_TENANT_SESSION_FILE or both CLUSTERFLUX_ISOLATION_TARGET_TENANT and CLUSTERFLUX_ISOLATION_TARGET_PROJECT" + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE so the compiled two-tenant Node and artifact collision can run" ); } if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) { @@ -500,8 +495,12 @@ async function runMalformedIdentifierSuite({ send: async (value) => sendHostedControl({ type: "node_heartbeat", + tenant, + project, node: value, node_signature: signedNodeHeartbeat( + tenant, + project, value, securityNode.identity, { @@ -803,8 +802,12 @@ async function runMalformedIdentifierSuite({ send: async (value) => { const request = { type: "node_heartbeat", + tenant, + project, node: securityNode.node, node_signature: signedNodeHeartbeat( + tenant, + project, securityNode.node, securityNode.identity, { nonce: value } @@ -1616,8 +1619,10 @@ async function prepareLiveNodeCredentialSecurity({ const heartbeat = { type: "node_heartbeat", + tenant, + project, node, - node_signature: signedNodeHeartbeat(node, identity, { + node_signature: signedNodeHeartbeat(tenant, project, node, identity, { nonce: `strict-valid-${suffix}`, }), }; @@ -1632,8 +1637,10 @@ async function prepareLiveNodeCredentialSecurity({ const expired = assertDenied( await sendHostedControl({ type: "node_heartbeat", + tenant, + project, node, - node_signature: signedNodeHeartbeat(node, identity, { + node_signature: signedNodeHeartbeat(tenant, project, node, identity, { nonce: `strict-expired-${suffix}`, issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, }), @@ -1646,8 +1653,10 @@ async function prepareLiveNodeCredentialSecurity({ const forged = assertDenied( await sendHostedControl({ type: "node_heartbeat", + tenant, + project, node, - node_signature: signedNodeHeartbeat(node, forgedIdentity, { + node_signature: signedNodeHeartbeat(tenant, project, node, forgedIdentity, { nonce: `strict-forged-${suffix}`, }), }), @@ -2556,6 +2565,12 @@ async function runLiveAgentCredentialSecurity({ } async function runSecondTenantIsolation({ + clusterflux, + clusterfluxNode, + firstScope, + firstHelloBuild, + firstHelloProjectDir, + workRoot, firstSessionSecret, firstTenant, firstProject, @@ -2597,14 +2612,17 @@ async function runSecondTenantIsolation({ "reused isolation evidence must target the exact public binaries" ); return { - ...isolation, - duration_ms: Date.now() - startedAt, - reused_from_immutable_evidence: { - report_sha256: sha256(evidenceBytes), - source_commit: previous.source_commit, - hosted_service_sha256: currentDeployment.hosted_service_sha256, - public_binary_digests: manifest.binary_digests, + evidence: { + ...isolation, + duration_ms: Date.now() - startedAt, + reused_from_immutable_evidence: { + report_sha256: sha256(evidenceBytes), + source_commit: previous.source_commit, + hosted_service_sha256: currentDeployment.hosted_service_sha256, + public_binary_digests: manifest.binary_digests, + }, }, + runtime: null, }; } if (!file && targetTenant && targetProject) { @@ -2702,36 +2720,56 @@ async function runSecondTenantIsolation({ "single-account foreign process control" ); return { - executed: true, - mode: "single_authenticated_account_foreign_project", - second_tenant: targetTenant, - target_project: targetProject, - project, - process_hidden: true, - node_hidden: true, - tasks_and_logs: tasksAndLogs, - debug: { - denied: true, - reason: debugResponse.authorization.reason, - audited: true, - charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, + evidence: { + executed: true, + mode: "single_authenticated_account_foreign_project", + second_tenant: targetTenant, + target_project: targetProject, + project, + process_hidden: true, + node_hidden: true, + tasks_and_logs: tasksAndLogs, + debug: { + denied: true, + reason: debugResponse.authorization.reason, + audited: true, + charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, + }, + artifact_and_download: artifact, + process_control: control, + duration_ms: Date.now() - startedAt, }, - artifact_and_download: artifact, - process_control: control, - duration_ms: Date.now() - startedAt, + runtime: null, }; } - if (!file) return { executed: false, reason: "second tenant session not supplied" }; - const second = readJson(path.resolve(file)); + if (!file) { + return { + evidence: { + executed: false, + reason: "second tenant session not supplied", + duration_ms: Math.max(1, Date.now() - startedAt), + }, + runtime: null, + }; + } + const secondSessionPath = path.resolve(file); + const second = readJson(secondSessionPath); assert.strictEqual(second.coordinator, serviceEndpoint); - assert(second.session_secret, "second tenant session omitted its session secret"); + const secondSessionSecret = + second.cli_session_secret || second.session_secret; + assert(secondSessionSecret, "second tenant session omitted its session secret"); assert.notStrictEqual( second.tenant, firstTenant, "isolation proof requires a genuinely distinct hosted tenant" ); + assert.notStrictEqual( + second.project, + firstProject, + "isolation proof requires a genuinely distinct hosted project" + ); const call = (request) => - sendHostedControl(authenticatedRequest(second.session_secret, request)); + sendHostedControl(authenticatedRequest(secondSessionSecret, request)); const project = assertDenied( await call({ type: "select_project", project: firstProject }), @@ -2778,7 +2816,7 @@ async function runSecondTenantIsolation({ max_bytes: 1024, ttl_seconds: 60, }), - /outside|scope|tenant|project|not found|unavailable/i, + /outside|scope|tenant|project|not found|does not exist|unavailable/i, "cross-tenant artifact download" ); const control = assertDenied( @@ -2786,17 +2824,437 @@ async function runSecondTenantIsolation({ /outside|scope|tenant|project|not active|requires an active|unknown/i, "cross-tenant process control" ); + + const secondCheckout = path.join(workRoot, "second-tenant-public-repo"); + stagePublicCheckout(manifest, secondCheckout); + const secondProjectDir = path.join( + secondCheckout, + "examples", + "hello-build" + ); + const secondControlDir = path.join(secondProjectDir, ".clusterflux"); + ensureDir(secondControlDir); + const secondProjectPath = path.join( + path.dirname(secondSessionPath), + "project.json" + ); + assert( + fs.existsSync(secondProjectPath), + `second tenant session is missing adjacent project.json: ${secondProjectPath}` + ); + fs.copyFileSync( + secondSessionPath, + path.join(secondControlDir, "session.json") + ); + fs.copyFileSync( + secondProjectPath, + path.join(secondControlDir, "project.json") + ); + fs.chmodSync(path.join(secondControlDir, "session.json"), 0o600); + fs.chmodSync(path.join(secondControlDir, "project.json"), 0o644); + const secondScope = [ + "--coordinator", + serviceEndpoint, + "--tenant", + second.tenant, + "--project-id", + second.project, + "--user", + second.user, + "--json", + ]; + const secondProjectList = runJson( + clusterflux, + ["project", "list", ...secondScope], + { cwd: secondProjectDir } + ); + assert(secondProjectList.project_count >= 1); + const secondProjectSelect = runJson( + clusterflux, + ["project", "select", ...secondScope, second.project], + { cwd: secondProjectDir } + ); + assert.strictEqual(secondProjectSelect.command, "project select"); + + const collisionStartedAt = Date.now(); + const secondGrant = runJson( + clusterflux, + ["node", "enroll", ...secondScope], + { cwd: secondProjectDir } + ); + const secondAttach = runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + second.tenant, + "--project-id", + second.project, + "--node", + firstNode, + "--enrollment-grant", + secondGrant.enrollment_grant.grant, + "--json", + ], + { cwd: secondProjectDir } + ); + assert.strictEqual(secondAttach.boundary.used_enrollment_exchange, true); + const secondStored = readNodeCredential(secondProjectDir, firstNode); + const secondCredentialDigest = sha256( + fs.readFileSync(secondStored.absolute) + ); + const secondIdentity = nodeIdentityFromPrivateKey( + secondStored.credential.private_key + ); + const secondWorkerArgs = [ + "--coordinator", + serviceEndpoint, + "--tenant", + second.tenant, + "--project-id", + second.project, + "--node", + firstNode, + "--worker", + "--project-root", + secondProjectDir, + "--assignment-poll-ms", + "500", + "--emit-ready", + ]; + const firstHelloWorker = spawnJsonLines( + clusterfluxNode, + [ + "--coordinator", + serviceEndpoint, + "--tenant", + firstTenant, + "--project-id", + firstProject, + "--node", + firstNode, + "--worker", + "--project-root", + firstHelloProjectDir, + "--assignment-poll-ms", + "500", + "--emit-ready", + ], + { + cwd: firstHelloProjectDir, + env: { ...process.env }, + } + ); + const firstHelloReady = await firstHelloWorker.waitFor( + (value) => value.node_status === "ready", + "first-tenant hello-build artifact worker ready for collision proof" + ); + assert.strictEqual(firstHelloReady.node, firstNode); + const firstCollisionHelloBuild = await runHostedHelloBuild({ + clusterflux, + projectDir: firstHelloProjectDir, + scope: firstScope, + outputFile: path.join(workRoot, "hello-clusterflux-first-tenant-collision"), + }); + assert.strictEqual( + firstCollisionHelloBuild.artifact, + firstHelloBuild.artifact, + "repeated deterministic first-tenant hello-build changed its ArtifactId" + ); + assert.strictEqual(firstCollisionHelloBuild.digest, firstHelloBuild.digest); + assert.strictEqual( + firstCollisionHelloBuild.executable_output, + firstHelloBuild.executable_output + ); + const secondWorker = spawnJsonLines(clusterfluxNode, secondWorkerArgs, { + cwd: secondProjectDir, + env: { ...process.env }, + }); + let secondHelloBuild; + try { + const secondReady = await secondWorker.waitFor( + (value) => value.node_status === "ready", + "same-name second-tenant worker ready" + ); + assert.strictEqual(secondReady.node, firstNode); + secondHelloBuild = await runHostedHelloBuild({ + clusterflux, + projectDir: secondProjectDir, + scope: secondScope, + outputFile: path.join(workRoot, "hello-clusterflux-second-tenant"), + }); + } finally { + await stopChild(secondWorker.child); + } + assert.strictEqual( + secondHelloBuild.artifact, + firstCollisionHelloBuild.artifact, + "deterministic two-tenant hello-build did not collide on ArtifactId" + ); + assert.strictEqual( + secondHelloBuild.digest, + firstCollisionHelloBuild.digest, + "deterministic two-tenant hello-build did not produce identical bytes" + ); + assert.strictEqual( + secondHelloBuild.executable_output, + firstCollisionHelloBuild.executable_output + ); + + const firstArtifactsAfterCollision = runJson( + clusterflux, + [ + "artifact", + "list", + ...firstScope, + "--process", + firstCollisionHelloBuild.process, + ], + { cwd: firstHelloProjectDir } + ); + assert( + firstArtifactsAfterCollision.artifacts.some( + (candidate) => + candidate.artifact === firstCollisionHelloBuild.artifact && + candidate.digest === firstCollisionHelloBuild.digest + ), + "the second tenant's same-ID artifact masked the first tenant's metadata" + ); + const secondCollisionLink = await call({ + type: "create_artifact_download_link", + artifact: secondHelloBuild.artifact, + max_bytes: secondHelloBuild.downloaded_bytes, + ttl_seconds: 60, + }); + assert.strictEqual( + secondCollisionLink.type, + "artifact_download_link", + JSON.stringify(secondCollisionLink) + ); + const secondCollisionLinkRevoked = await call({ + type: "revoke_artifact_download_link", + artifact: secondHelloBuild.artifact, + token_digest: secondCollisionLink.link.scoped_token_digest, + }); + assert.strictEqual( + secondCollisionLinkRevoked.type, + "artifact_download_link_revoked", + JSON.stringify(secondCollisionLinkRevoked) + ); + const firstDownloadAfterSecondLinkRevocation = runJson( + clusterflux, + [ + "artifact", + "download", + ...firstScope, + firstCollisionHelloBuild.artifact, + "--to", + path.join(workRoot, "hello-clusterflux-first-after-second-link-revoke"), + ], + { cwd: firstHelloProjectDir } + ); + assert.strictEqual( + firstDownloadAfterSecondLinkRevocation.local_download.verified_digest, + firstCollisionHelloBuild.digest + ); + await stopChild(firstHelloWorker.child); + return { - executed: true, - second_tenant: second.tenant, + evidence: { + executed: true, + second_tenant: second.tenant, + second_project: second.project, + project, + process_hidden: true, + node_hidden: true, + tasks_and_logs: tasksAndLogs, + debug, + artifact_and_download: artifact, + process_control: control, + scoped_node_and_artifact_collision: { + request: { + node: firstNode, + first_scope: { + tenant: firstTenant, + project: firstProject, + }, + second_scope: { + tenant: second.tenant, + project: second.project, + }, + example: "hello-build", + }, + expected: + "same-name Nodes coexist and deterministic hello-build artifacts share an ArtifactId without metadata, link, or download interference", + observed: { + first_process: firstCollisionHelloBuild.process, + second_process: secondHelloBuild.process, + same_artifact_id: secondHelloBuild.artifact, + first_digest: firstCollisionHelloBuild.digest, + second_digest: secondHelloBuild.digest, + first_downloaded_bytes: firstCollisionHelloBuild.downloaded_bytes, + second_downloaded_bytes: secondHelloBuild.downloaded_bytes, + exact_executable_output: secondHelloBuild.executable_output, + both_metadata_records_visible_to_owner: true, + cross_tenant_unique_artifact_denied: artifact.denied, + second_link_revoked: true, + first_download_survived_second_link_revocation: true, + }, + duration_ms: Math.max(1, Date.now() - collisionStartedAt), + }, + duration_ms: Math.max(1, Date.now() - startedAt), + }, + runtime: { + node: firstNode, + tenant: second.tenant, + project: second.project, + projectDir: secondProjectDir, + scope: secondScope, + workerArgs: secondWorkerArgs, + credentialPath: secondStored.absolute, + credentialDigest: secondCredentialDigest, + identity: secondIdentity, + }, + }; +} + +async function runLiveSignedHostileArtifactPath({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, + securityNode, +}) { + const startedAt = Date.now(); + const runReport = runJson( + clusterflux, + ["run", "park-wake", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const process = runReport.process; + await waitForCli( + "hostile-path probe process to become active", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", process], + { cwd: projectDir } + ), + (status) => status.state !== "not_active", + 120_000 + ); + + const invalidStartedAt = Date.now(); + const invalid = assertDenied( + await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_vfs_metadata", + { + type: "report_vfs_metadata", + tenant, + project, + process, + node: securityNode.node, + task: `hostile-path-${suffix}`, + artifact_path: "/vfs/artifacts/bad artifact!", + artifact_digest: sha256(Buffer.from("hostile-path-invalid")), + artifact_size_bytes: 20, + large_bytes_uploaded: false, + }, + { nonce: `strict-hostile-path-invalid-${suffix}-${Date.now()}` } + ) + ), + /invalid VFS artifact path|invalid artifact path|ArtifactId is invalid/i, + "correctly signed hostile artifact path" + ); + const invalidDurationMs = Math.max(1, Date.now() - invalidStartedAt); + + const validStartedAt = Date.now(); + const validArtifact = `strict-hostile-followup-${suffix}`; + const valid = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_vfs_metadata", + { + type: "report_vfs_metadata", + tenant, + project, + process, + node: securityNode.node, + task: `hostile-path-${suffix}`, + artifact_path: `/vfs/artifacts/${validArtifact}`, + artifact_digest: sha256(Buffer.from("hostile-path-valid")), + artifact_size_bytes: 18, + large_bytes_uploaded: false, + }, + { nonce: `strict-hostile-path-valid-${suffix}-${Date.now()}` } + ) + ); + assert.strictEqual(valid.type, "vfs_metadata_recorded", JSON.stringify(valid)); + const healthyHeartbeat = await sendHostedControl({ + type: "node_heartbeat", + tenant, project, - process_hidden: true, - node_hidden: true, - tasks_and_logs: tasksAndLogs, - debug, - artifact_and_download: artifact, - process_control: control, - duration_ms: Date.now() - startedAt, + node: securityNode.node, + node_signature: signedNodeHeartbeat( + tenant, + project, + securityNode.node, + securityNode.identity, + { nonce: `strict-hostile-path-health-${suffix}-${Date.now()}` } + ), + }); + assert.strictEqual( + healthyHeartbeat.type, + "node_heartbeat", + JSON.stringify(healthyHeartbeat) + ); + const validDurationMs = Math.max(1, Date.now() - validStartedAt); + const aborted = runJson( + clusterflux, + ["process", "abort", ...scope, "--process", process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(aborted.abort_request.accepted, true); + await waitForCli( + "hostile-path probe process cleanup", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + return { + request: { + principal: "enrolled signed Node", + variant: "report_vfs_metadata", + process, + malformed_path: "/vfs/artifacts/bad artifact!", + }, + expected: + "structured InvalidArtifactPath rejection followed immediately by valid signed metadata and heartbeat on the same service instance", + observed: { + rejection: invalid, + valid_metadata_response: valid.type, + valid_artifact: validArtifact, + health_response: healthyHeartbeat.type, + process_cleanup: "not_active", + }, + invalid_duration_ms: invalidDurationMs, + valid_followup_duration_ms: validDurationMs, + duration_ms: Math.max(1, Date.now() - startedAt), }; } @@ -2960,8 +3418,12 @@ async function revokeSecurityNode({ const denied = assertDenied( await sendHostedControl({ type: "node_heartbeat", + tenant: securityNode.capability_body.tenant, + project: securityNode.capability_body.project, node: securityNode.node, node_signature: signedNodeHeartbeat( + securityNode.capability_body.tenant, + securityNode.capability_body.project, securityNode.node, securityNode.identity, { nonce: `strict-node-revoked-${Date.now()}` } @@ -3742,6 +4204,7 @@ async function restartHostedServiceAndResume({ workerNode, credentialPath, credentialDigest, + scopedCollisionRuntime, }) { if (!strictVpsRestart) { return { @@ -3841,6 +4304,79 @@ async function restartHostedServiceAndResume({ assert.strictEqual(ready.node, workerNode); assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + let scopedNodeCollisionAfterRestart = null; + if (scopedCollisionRuntime) { + assert.strictEqual(scopedCollisionRuntime.node, workerNode); + assert.notStrictEqual( + scopedCollisionRuntime.tenant, + readJson(path.join(projectDir, ".clusterflux", "session.json")).tenant + ); + assert.strictEqual( + sha256(fs.readFileSync(scopedCollisionRuntime.credentialPath)), + scopedCollisionRuntime.credentialDigest + ); + const secondWorker = spawnJsonLines( + clusterfluxNode, + scopedCollisionRuntime.workerArgs, + { + cwd: scopedCollisionRuntime.projectDir, + env: { ...process.env }, + } + ); + try { + const secondReady = await secondWorker.waitFor( + (value) => value.node_status === "ready", + "same-name second-tenant worker ready after hosted service restart" + ); + assert.strictEqual(secondReady.node, workerNode); + const firstStatus = await waitForCli( + "first scoped same-name Node online after restart", + () => + runJson( + clusterflux, + ["node", "status", ...scope, "--node", workerNode], + { cwd: projectDir } + ), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30_000 + ); + const secondStatus = await waitForCli( + "second scoped same-name Node online after restart", + () => + runJson( + clusterflux, + [ + "node", + "status", + ...scopedCollisionRuntime.scope, + "--node", + workerNode, + ], + { cwd: scopedCollisionRuntime.projectDir } + ), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30_000 + ); + scopedNodeCollisionAfterRestart = { + node: workerNode, + first_scope_online: + nodeDescriptor(firstStatus, workerNode)?.online === true, + second_scope_online: + nodeDescriptor(secondStatus, workerNode)?.online === true, + both_credentials_reused_without_grants: true, + distinct_credential_digests: + credentialDigest !== scopedCollisionRuntime.credentialDigest, + }; + assert.strictEqual( + scopedNodeCollisionAfterRestart.distinct_credential_digests, + true, + "same-name scoped Nodes unexpectedly reused one credential" + ); + } finally { + await stopChild(secondWorker.child); + } + } + const restartedRun = runJson( clusterflux, ["run", "build", "--project", ".", "--json"], @@ -3873,6 +4409,7 @@ async function restartHostedServiceAndResume({ terminated_ephemeral_process: ephemeralProcess, new_process: restartedProcess, new_process_result: completedEvent.result, + scoped_node_collision_after_restart: scopedNodeCollisionAfterRestart, before, after_first_restart: afterFirstRestart, after, @@ -3880,6 +4417,79 @@ async function restartHostedServiceAndResume({ }; } +async function finishScopedNodeCollisionAfterRestart({ + clusterflux, + projectDir, + scope, + workerNode, + scopedCollisionRuntime, +}) { + const startedAt = Date.now(); + assert(scopedCollisionRuntime, "scoped collision runtime evidence is required"); + const revoked = runJson( + clusterflux, + [ + "node", + "revoke", + ...scopedCollisionRuntime.scope, + "--node", + scopedCollisionRuntime.node, + "--yes", + ], + { cwd: scopedCollisionRuntime.projectDir } + ); + assert.strictEqual(revoked.command, "node revoke"); + const revokedHeartbeat = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + tenant: scopedCollisionRuntime.tenant, + project: scopedCollisionRuntime.project, + node: scopedCollisionRuntime.node, + node_signature: signedNodeHeartbeat( + scopedCollisionRuntime.tenant, + scopedCollisionRuntime.project, + scopedCollisionRuntime.node, + scopedCollisionRuntime.identity, + { + nonce: `strict-scoped-node-revoked-${Date.now()}`, + } + ), + }), + /not enrolled|unknown node|revoked|credential/i, + "revoked second scoped Node" + ); + const firstStatus = await waitForCli( + "first same-name Node remains live after scoped revocation", + () => + runJson( + clusterflux, + ["node", "status", ...scope, "--node", workerNode], + { cwd: projectDir } + ), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30_000 + ); + return { + request: { + revoked_scope: { + tenant: scopedCollisionRuntime.tenant, + project: scopedCollisionRuntime.project, + node: scopedCollisionRuntime.node, + }, + retained_node: workerNode, + }, + expected: + "revocation affects only the selected scoped Node after both same-name credentials survive restart", + observed: { + revoked_command: revoked.command, + revoked_credential_denied: revokedHeartbeat.denied, + first_scope_node_online: + nodeDescriptor(firstStatus, workerNode)?.online === true, + }, + duration_ms: Math.max(1, Date.now() - startedAt), + }; +} + async function finishUserCredentialSecurity({ clusterflux, projectDir, @@ -5713,7 +6323,14 @@ async function main() { scope, }); - const tenantIsolation = await runSecondTenantIsolation({ + await stopChild(worker.child); + const tenantIsolationResult = await runSecondTenantIsolation({ + clusterflux, + clusterfluxNode, + firstScope: scope, + firstHelloBuild: helloBuild, + firstHelloProjectDir: helloProjectDir, + workRoot, firstSessionSecret: sessionSecret, firstTenant: tenant, firstProject: project, @@ -5722,6 +6339,14 @@ async function main() { firstArtifact: releaseArtifact.artifact, manifest, }); + const tenantIsolation = tenantIsolationResult.evidence; + const scopedCollisionRuntime = tenantIsolationResult.runtime; + worker = spawnWorker(); + const collisionRecoveryReady = await worker.waitFor( + (value) => value.node_status === "ready", + "primary worker restored after two-tenant collision proof" + ); + assert.strictEqual(collisionRecoveryReady.node, workerNode); const securityNode = await prepareLiveNodeCredentialSecurity({ clusterflux, @@ -5824,6 +6449,15 @@ async function main() { }, releaseArtifact, }); + const signedHostileArtifactPath = await runLiveSignedHostileArtifactPath({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, + securityNode, + }); const revokedNodeCredential = await revokeSecurityNode({ clusterflux, projectDir, @@ -5844,8 +6478,22 @@ async function main() { workerNode, credentialPath, credentialDigest, + scopedCollisionRuntime, }); if (serviceRestart.worker) worker = serviceRestart.worker; + const scopedNodeRevocationAfterRestart = scopedCollisionRuntime + ? await finishScopedNodeCollisionAfterRestart({ + clusterflux, + projectDir, + scope, + workerNode, + scopedCollisionRuntime, + }) + : { + executed: false, + reason: "second tenant runtime session was not supplied", + duration_ms: 1, + }; const relayAfterRestart = relayDurableState(); assert( relayAfterRestart.ingress_used >= @@ -6298,6 +6946,49 @@ async function main() { evidence: processCancellationLifecycle, durationMs: processCancellationLifecycle.duration_ms, }), + requirement({ + id: "30_scoped_node_and_artifact_collision", + passed: + tenantIsolation.scoped_node_and_artifact_collision?.observed + ?.both_metadata_records_visible_to_owner === true && + tenantIsolation.scoped_node_and_artifact_collision?.observed + ?.first_download_survived_second_link_revocation === true && + tenantIsolation.scoped_node_and_artifact_collision?.observed + ?.same_artifact_id === helloBuild.artifact && + serviceRestart.scoped_node_collision_after_restart + ?.first_scope_online === true && + serviceRestart.scoped_node_collision_after_restart + ?.second_scope_online === true && + serviceRestart.scoped_node_collision_after_restart + ?.both_credentials_reused_without_grants === true && + scopedNodeRevocationAfterRestart.observed + ?.revoked_credential_denied === true && + scopedNodeRevocationAfterRestart.observed + ?.first_scope_node_online === true, + evidence: { + live_collision: + tenantIsolation.scoped_node_and_artifact_collision, + restart: + serviceRestart.scoped_node_collision_after_restart, + scoped_revocation: scopedNodeRevocationAfterRestart, + }, + durationMs: + tenantIsolation.scoped_node_and_artifact_collision?.duration_ms + + scopedNodeRevocationAfterRestart.duration_ms, + }), + requirement({ + id: "31_signed_hostile_artifact_path_preserves_service", + passed: + signedHostileArtifactPath.observed.rejection.denied === true && + signedHostileArtifactPath.observed.valid_metadata_response === + "vfs_metadata_recorded" && + signedHostileArtifactPath.observed.health_response === + "node_heartbeat" && + signedHostileArtifactPath.observed.process_cleanup === + "not_active", + evidence: signedHostileArtifactPath, + durationMs: signedHostileArtifactPath.duration_ms, + }), ]; const fullReleasePassed = strictFullRelease && @@ -6371,6 +7062,8 @@ async function main() { dap_process_aborted: true, process_cancellation_lifecycle: processCancellationLifecycle, tenant_isolation: tenantIsolation, + scoped_node_revocation_after_restart: + scopedNodeRevocationAfterRestart, credential_security: { user: userCredentialSecurity, node: { @@ -6409,6 +7102,7 @@ async function main() { hello_build: helloBuild, recovery_build: recoveryBuild, malformed_identifiers: malformedIdentifiers, + signed_hostile_artifact_path: signedHostileArtifactPath, named_scenarios: namedScenarios, scenario_skips: [], strict_requirement_ledger: strictRequirementLedger, diff --git a/scripts/hostile-input-contract-smoke.js b/scripts/hostile-input-contract-smoke.js index e2c8b25..8d5173a 100644 --- a/scripts/hostile-input-contract-smoke.js +++ b/scripts/hostile-input-contract-smoke.js @@ -42,6 +42,8 @@ const signingInstrumentNode = nodeIdentity( ); assert.strictEqual( signedNodeHeartbeat( + "tenant", + "project", "node-hostile-input-contract", signingInstrumentNode, { nonce: "" } @@ -127,8 +129,7 @@ for (const [name, source, patterns] of [ /const guessed = await send/, /const crossActorOpen = await send/, /token is invalid/, - /tenant mismatch/, - /project mismatch/, + /artifact does not exist/, ], ], [ diff --git a/scripts/node-attach-smoke.js b/scripts/node-attach-smoke.js index d6c4e01..c7c9dc2 100755 --- a/scripts/node-attach-smoke.js +++ b/scripts/node-attach-smoke.js @@ -94,12 +94,12 @@ function nodeSignatureMessage( ); } -function signedNodeHeartbeat(node, identity) { +function signedNodeHeartbeat(tenant, project, node, identity) { const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`; const issuedAt = Math.floor(Date.now() / 1000); const payloadDigest = `sha256:${crypto .createHash("sha256") - .update(JSON.stringify({ node, type: "node_heartbeat" })) + .update(JSON.stringify({ node, project, tenant, type: "node_heartbeat" })) .digest("hex")}`; const signature = crypto.sign( null, @@ -283,8 +283,15 @@ function runAttach(addr, grant) { const heartbeat = await send(addr, { type: "node_heartbeat", + tenant: "tenant", + project: "project", node: "node-attach", - node_signature: signedNodeHeartbeat("node-attach", nodeIdentity("node-attach")), + node_signature: signedNodeHeartbeat( + "tenant", + "project", + "node-attach", + nodeIdentity("node-attach") + ), }); assert.strictEqual(heartbeat.type, "node_heartbeat"); assert.strictEqual(heartbeat.node, "node-attach"); diff --git a/scripts/node-lifecycle-contract-smoke.js b/scripts/node-lifecycle-contract-smoke.js index b597da9..78d4387 100755 --- a/scripts/node-lifecycle-contract-smoke.js +++ b/scripts/node-lifecycle-contract-smoke.js @@ -97,7 +97,7 @@ for (const [name, pattern] of [ for (const [name, pattern] of [ ["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/], - ["reconnect preserves enrolled node identity", /reconnect_node\(&NodeId::from\("node"\), None\)/], + ["reconnect preserves scoped enrolled node identity", /reconnect_node\([\s\S]*&TenantId::from\("tenant"\),[\s\S]*&ProjectId::from\("project"\),[\s\S]*&NodeId::from\("node"\),[\s\S]*None/], ["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/], ]) { expect(coordinatorCore, name, pattern); diff --git a/scripts/node-signing.js b/scripts/node-signing.js index 7a6a4e9..3069840 100644 --- a/scripts/node-signing.js +++ b/scripts/node-signing.js @@ -151,8 +151,8 @@ function signedNodeProof(node, identity, requestKind, request, options = {}) { }; } -function signedNodeHeartbeat(node, identity, options = {}) { - const request = { type: "node_heartbeat", node }; +function signedNodeHeartbeat(tenant, project, node, identity, options = {}) { + const request = { type: "node_heartbeat", tenant, project, node }; return signedNodeProof(node, identity, "node_heartbeat", request, options); } diff --git a/scripts/prepare-manual-github-release.js b/scripts/prepare-manual-github-release.js deleted file mode 100755 index 6c91f8d..0000000 --- a/scripts/prepare-manual-github-release.js +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env node -"use strict"; - -const assert = require("node:assert"); -const crypto = require("node:crypto"); -const fs = require("node:fs"); -const path = require("node:path"); -const { spawnSync } = require("node:child_process"); - -const repo = path.resolve(__dirname, ".."); -const releaseRoot = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/public-release-final") -); -const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); -const resultPath = path.resolve( - process.env.CLUSTERFLUX_FINAL_RESULT || - path.join(repo, "target/acceptance/cli-happy-path-live.json") -); -const transcriptPath = path.resolve( - process.env.CLUSTERFLUX_VALIDATION_TRANSCRIPT || - path.join(repo, "target/acceptance/clusterflux-manual-launch-transcript.log") -); - -function sha256(file) { - const hash = crypto.createHash("sha256"); - hash.update(fs.readFileSync(file)); - return hash.digest("hex"); -} - -function readJson(file) { - return JSON.parse(fs.readFileSync(file, "utf8")); -} - -function write(file, contents) { - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, contents); -} - -function copyVerifiedAsset(asset, destinationRoot) { - const source = path.resolve(releaseRoot, asset.file); - assert(fs.existsSync(source), `missing finalized asset ${source}`); - assert.strictEqual( - sha256(source), - asset.sha256, - `finalized asset digest changed: ${asset.name}` - ); - const destination = path.join(destinationRoot, "assets", asset.name); - fs.copyFileSync(source, destination); - assert.strictEqual(sha256(destination), asset.sha256); - return destination; -} - -function extractBinaries(archive, destinationRoot, expectedDigests) { - const staging = path.join(destinationRoot, ".binary-extract"); - fs.mkdirSync(staging, { recursive: true }); - const extracted = spawnSync("tar", ["-xzf", archive, "-C", staging], { - encoding: "utf8", - }); - if (extracted.status !== 0) { - throw new Error(`failed to extract ${archive}: ${extracted.stderr}`); - } - const binRoot = path.join(staging, "bin"); - assert(fs.existsSync(binRoot), `binary archive omitted bin/: ${archive}`); - const copied = []; - for (const name of fs.readdirSync(binRoot).sort()) { - const source = path.join(binRoot, name); - if (!fs.statSync(source).isFile()) continue; - const destination = path.join(destinationRoot, "binaries", name); - fs.copyFileSync(source, destination); - fs.chmodSync(destination, 0o755); - assert.strictEqual( - `sha256:${sha256(destination)}`, - expectedDigests[name], - `tested binary digest changed: ${name}` - ); - copied.push(destination); - } - fs.rmSync(staging, { recursive: true, force: true }); - assert(copied.length > 0, `binary archive contained no binaries: ${archive}`); - return copied; -} - -function main() { - assert(fs.existsSync(manifestPath), `missing final manifest ${manifestPath}`); - assert(fs.existsSync(resultPath), `missing final result ${resultPath}`); - assert(fs.existsSync(transcriptPath), `missing validation transcript ${transcriptPath}`); - const manifest = readJson(manifestPath); - const result = readJson(resultPath); - assert.strictEqual(result.acceptance_result, "passed"); - assert.strictEqual(result.source_commit, manifest.source_commit); - assert.strictEqual(result.source_tree_digest, manifest.source_tree_digest); - assert.deepStrictEqual(result.binary_digests, manifest.binary_digests); - assert.strictEqual(result.scenario_skips.length, 0); - assert(result.strict_requirement_ledger.length >= 29); - assert( - result.strict_requirement_ledger.every( - (requirement) => - requirement.passed === true && - Number.isFinite(requirement.duration_ms) && - requirement.duration_ms > 0 && - requirement.evidence - ), - "final validation contains a failed launch requirement" - ); - assert.strictEqual( - result.named_scenarios.length, - result.strict_requirement_ledger.length - ); - - const destinationRoot = path.resolve( - process.env.CLUSTERFLUX_GITHUB_RELEASE_DIR || - path.join(repo, "target/github-release", manifest.release_name) - ); - fs.rmSync(destinationRoot, { recursive: true, force: true }); - fs.mkdirSync(path.join(destinationRoot, "assets"), { recursive: true }); - fs.mkdirSync(path.join(destinationRoot, "binaries"), { recursive: true }); - - const copiedAssets = manifest.assets.map((asset) => - copyVerifiedAsset(asset, destinationRoot) - ); - const publicationGuidance = [ - JSON.stringify(manifest.notes || []), - ...copiedAssets - .filter((file) => file.endsWith(".md")) - .map((file) => fs.readFileSync(file, "utf8")), - ].join("\n"); - assert.doesNotMatch( - publicationGuidance, - /(?:download|upload|release downloads?)[^\n]*Forgejo/i, - "manual GitHub package contains Forgejo release-publication instructions" - ); - const binaryArchive = copiedAssets.find((file) => - path.basename(file).startsWith("clusterflux-public-binaries-") - ); - assert(binaryArchive, "final manifest omitted the public binary archive"); - const binaries = extractBinaries( - binaryArchive, - destinationRoot, - manifest.binary_digests - ); - const vsix = copiedAssets.find((file) => file.endsWith(".vsix")); - assert(vsix, "final manifest omitted the tested VSIX"); - assert.strictEqual( - sha256(vsix), - manifest.release_candidate.extension_sha256, - "tested VSIX digest does not match final candidate binding" - ); - - write(path.join(destinationRoot, "SOURCE_REVISION"), `${manifest.source_commit}\n`); - write( - path.join(destinationRoot, "SOURCE_TREE_DIGEST"), - `${manifest.source_tree_digest}\n` - ); - write(path.join(destinationRoot, "VSIX_SHA256"), `${sha256(vsix)} ${path.basename(vsix)}\n`); - fs.copyFileSync(manifestPath, path.join(destinationRoot, "public-release-manifest.json")); - fs.copyFileSync(transcriptPath, path.join(destinationRoot, "VALIDATION_TRANSCRIPT.log")); - write( - path.join(destinationRoot, "RELEASE_NOTES.md"), - `# Clusterflux ${manifest.release_name}\n\n` + - "First manual GitHub launch release of Clusterflux. The release includes the filtered public source, real Linux CLI/node/coordinator binaries, and the exact VSIX used by the final production-shaped validation batch.\n" - ); - write( - path.join(destinationRoot, "KNOWN_LIMITATIONS.md"), - "# Known limitations\n\n" + - "- The launch batch validates one enrolled Linux node with rootless Podman.\n" + - "- Full Windows sandbox validation is deferred.\n" + - "- Providers, billing, teams, managed compute, and operator-panel UI are not part of this release.\n" - ); - const testedUploadAssets = [ - ...copiedAssets, - ...binaries, - ] - .sort() - .map((file) => ({ - file: path.relative(destinationRoot, file), - sha256: `sha256:${sha256(file)}`, - })); - write( - path.join(destinationRoot, "final-result.json"), - `${JSON.stringify( - { - ...result, - manual_release: { - source_revision: manifest.source_commit, - source_tree_digest: manifest.source_tree_digest, - tested_upload_assets: testedUploadAssets, - publication: "manual GitHub upload from this directory", - }, - }, - null, - 2 - )}\n` - ); - - const checksummed = [ - ...copiedAssets, - ...binaries, - path.join(destinationRoot, "SOURCE_REVISION"), - path.join(destinationRoot, "SOURCE_TREE_DIGEST"), - path.join(destinationRoot, "VSIX_SHA256"), - path.join(destinationRoot, "public-release-manifest.json"), - path.join(destinationRoot, "final-result.json"), - path.join(destinationRoot, "VALIDATION_TRANSCRIPT.log"), - path.join(destinationRoot, "RELEASE_NOTES.md"), - path.join(destinationRoot, "KNOWN_LIMITATIONS.md"), - ].sort(); - write( - path.join(destinationRoot, "SHA256SUMS"), - `${checksummed - .map((file) => `${sha256(file)} ${path.relative(destinationRoot, file)}`) - .join("\n")}\n` - ); - console.log(destinationRoot); -} - -main(); diff --git a/scripts/release-quality-gates.js b/scripts/release-quality-gates.js index 5a350da..5ec6343 100755 --- a/scripts/release-quality-gates.js +++ b/scripts/release-quality-gates.js @@ -169,7 +169,7 @@ function main() { "test", "-p", "clusterflux-coordinator", - "completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot", + "completed_main_", ], }, { diff --git a/scripts/scheduler-placement-smoke.js b/scripts/scheduler-placement-smoke.js index 0b35350..2050403 100755 --- a/scripts/scheduler-placement-smoke.js +++ b/scripts/scheduler-placement-smoke.js @@ -243,7 +243,10 @@ async function reportNode(addr, node, identity, locality) { online: true })); assert.strictEqual(crossTenantReport.type, "error"); - assert.match(crossTenantReport.message, /tenant\/project scope/); + assert.match( + crossTenantReport.message, + /tenant\/project scope|node identity is not enrolled/ + ); const placement = await send(addr, { type: "schedule_task", diff --git a/scripts/self-hosted-coordinator-smoke.js b/scripts/self-hosted-coordinator-smoke.js index ca35649..2938eac 100644 --- a/scripts/self-hosted-coordinator-smoke.js +++ b/scripts/self-hosted-coordinator-smoke.js @@ -190,8 +190,8 @@ function nodeSignatureMessage( ); } -function signedNodeHeartbeat(node, identity) { - const request = { type: "node_heartbeat", node }; +function signedNodeHeartbeat(tenant, project, node, identity) { + const request = { type: "node_heartbeat", tenant, project, node }; const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`; const issuedAt = Math.floor(Date.now() / 1000); const signature = crypto.sign( @@ -313,8 +313,10 @@ async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilitie const heartbeat = await send(addr, { type: "node_heartbeat", + tenant: "team", + project: "self-hosted", node, - node_signature: signedNodeHeartbeat(node, identity) + node_signature: signedNodeHeartbeat("team", "self-hosted", node, identity) }); assert.strictEqual(heartbeat.type, "node_heartbeat"); @@ -569,6 +571,8 @@ async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilitie const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", { type: "reconnect_node", + tenant: "team", + project: "self-hosted", node: "team-linux-a", process: "vp-team-build", epoch: started.epoch diff --git a/scripts/source-preparation-smoke.js b/scripts/source-preparation-smoke.js index 9f9acd8..07be55d 100755 --- a/scripts/source-preparation-smoke.js +++ b/scripts/source-preparation-smoke.js @@ -170,7 +170,10 @@ function sourceCapableNode(sourceProviders = ["git"]) { source_snapshot: "sha256:source-prepared" })); assert.strictEqual(crossTenantCompletion.type, "error"); - assert.match(crossTenantCompletion.message, /tenant\/project scope/i); + assert.match( + crossTenantCompletion.message, + /tenant\/project scope|node identity is not enrolled/i + ); const completed = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { type: "complete_source_preparation", diff --git a/scripts/tenant-isolation-contract-smoke.js b/scripts/tenant-isolation-contract-smoke.js index fb93e49..65a1c9d 100644 --- a/scripts/tenant-isolation-contract-smoke.js +++ b/scripts/tenant-isolation-contract-smoke.js @@ -38,6 +38,7 @@ const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); +const liveSmoke = read("scripts/cli-happy-path-live-smoke.js"); for (const [name, pattern] of [ ["auth contexts carry tenant and project", /pub struct AuthContext[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], @@ -79,10 +80,13 @@ for (const [name, pattern] of [ ["project creation rejects foreign tenant reuse", /project id is outside the signed-in tenant scope/], ["project selection rejects foreign tenant", /project is outside the signed-in tenant scope/], ["project listing uses tenant-scoped context", /CoordinatorRequest::ListProjects[\s\S]*list_projects\(&context\)/], - ["node capability reports check enrollment tenant scope", /node capability report is outside the enrolled tenant\/project scope/], + [ + "node capability reports resolve the full enrolled scope", + /NodeScopeKey::from_refs\(&tenant, &project, &node\)[\s\S]*node_identity\(&tenant, &project, &node\)/, + ], ["operator panel stop state is tenant/project/process keyed", /type PanelStopKey = \(TenantId, ProjectId, ProcessId\)/], - ["download service tests tenant mismatch", /cross_tenant\.to_string\(\)\.contains\("tenant mismatch"\)/], - ["download service tests project mismatch", /cross_project\.to_string\(\)\.contains\("project mismatch"\)/], + ["download service hides cross-tenant artifact existence", /cross_tenant\.to_string\(\)\.contains\("does not exist"\)/], + ["download service hides cross-project artifact existence", /cross_project\.to_string\(\)\.contains\("does not exist"\)/], ["node capability test rejects cross-scope report", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], ["source preparation test rejects cross-scope completion", /fn service_rejects_source_preparation_completion_outside_node_scope\(\)/], ]) { @@ -98,8 +102,7 @@ for (const [name, sourceText, patterns] of [ /const crossProject = await send/, /const crossTenantOpen = await send/, /const crossProjectOpen = await send/, - /tenant mismatch/, - /project mismatch/, + /artifact does not exist/, /token is invalid/, ], ], @@ -129,6 +132,19 @@ for (const [name, sourceText, patterns] of [ } } +for (const [name, pattern] of [ + [ + "live same-ID collision refreshes first-tenant metadata before the second build", + /const firstCollisionHelloBuild = await runHostedHelloBuild[\s\S]*secondHelloBuild = await runHostedHelloBuild/, + ], + [ + "live same-ID collision re-reads the fresh first-tenant process", + /"--process",[\s\S]*firstCollisionHelloBuild\.process[\s\S]*candidate\.artifact === firstCollisionHelloBuild\.artifact/, + ], +]) { + expect(liveSmoke, name, pattern); +} + const hostedLibRoot = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); const hostedServiceSource = maybeRead([ "private", diff --git a/scripts/windows-best-effort-smoke.js b/scripts/windows-best-effort-smoke.js index bdb0442..839582a 100755 --- a/scripts/windows-best-effort-smoke.js +++ b/scripts/windows-best-effort-smoke.js @@ -198,6 +198,8 @@ function assertWindowsBackendBoundary() { const reconnected = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "reconnect_node", { type: "reconnect_node", + tenant: "tenant", + project: "project", node: windowsNode, process: "vp-windows", epoch: started.epoch