michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-ea887c8f56cd

Browse source 
Source commit: ea887c8f56cd53985a1179b13e5f1b85c485f584

Public tree identity: sha256:b96a97aacdbc8fd4fc2ea20d0e1450280d46db3f24aabe1475e5fbe20e05f255
This commit is contained in:
Clusterflux release 2026-07-25 02:55:35 +02:00
parent 9223c54939
commit 2a0f7ded04
37 changed files with 3145 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CLUSTERFLUX_PUBLIC_TREE.json
View file

@ -1,7 +1,7 @@
{
"kind": "clusterflux-filtered-public-tree",
"source_commit": "e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d",
"release_name": "release-e47f9c27bbeb",
"source_commit": "ea887c8f56cd53985a1179b13e5f1b85c485f584",
"release_name": "release-ea887c8f56cd",
"filtered_out": [
"private/**",
"internal/**",

2
crates/clusterflux-cli/src/node.rs
View file

@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
};
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &tenant,
"project": &project,
"node": &plan.node,
});
let heartbeat_signature = sign_node_request(

35
crates/clusterflux-coordinator/src/durable.rs
View file

@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
pub enrollment_scope: String,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct NodeScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub node: NodeId,
}
impl NodeScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
Self {
tenant,
project,
node,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
Self::new(tenant.clone(), project.clone(), node.clone())
}
pub fn credential_subject(&self) -> String {
format!(
"node:{}:{}:{}:{}:{}:{}",
self.tenant.as_str().len(),
self.tenant,
self.project.as_str().len(),
self.project,
self.node.as_str().len(),
self.node
)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CredentialRecord {
pub subject: String,
@ -107,7 +140,7 @@ pub struct DurableState {
pub tenants: BTreeMap<TenantId, TenantRecord>,
pub users: BTreeMap<UserId, UserRecord>,
pub projects: BTreeMap<ProjectId, ProjectRecord>,
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
pub credentials: BTreeMap<String, CredentialRecord>,
pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
pub source_provider_configs:

202
crates/clusterflux-coordinator/src/lib.rs
View file

@ -13,7 +13,7 @@ pub mod service;
mod sessions;
pub use durable::{
AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
TenantRecord, UserRecord,
};
@ -128,8 +128,9 @@ impl Coordinator {
public_key: impl Into<String>,
enrollment_scope: impl Into<String>,
) {
let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
self.durable.node_identities.insert(
node.clone(),
key,
NodeIdentityRecord {
id: node,
tenant,
@ -181,10 +182,13 @@ impl Coordinator {
public_key,
credential.scope.clone(),
);
let subject =
NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
.credential_subject();
self.durable.credentials.insert(
format!("node:{node}"),
subject.clone(),
CredentialRecord {
subject: format!("node:{node}"),
subject,
tenant: credential.tenant.clone(),
project: Some(credential.project.clone()),
kind: credential.credential_kind.clone(),
@ -356,15 +360,12 @@ impl Coordinator {
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node identity is outside the requested tenant/project scope".to_owned(),
));
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if !self
.active_processes
@ -379,14 +380,18 @@ impl Coordinator {
pub fn reconnect_node(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
process: Option<(&ProcessId, u64)>,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if let Some((process_id, stale_epoch)) = process {
if stale_epoch != self.coordinator_epoch {
@ -395,11 +400,7 @@ impl Coordinator {
current_epoch: self.coordinator_epoch,
});
}
let key = (
identity.tenant.clone(),
identity.project.clone(),
process_id.clone(),
);
let key = (tenant.clone(), project.clone(), process_id.clone());
if let Some(active) = self.active_processes.get_mut(&key) {
active.connected_nodes.insert(node.clone());
}
@ -416,23 +417,27 @@ impl Coordinator {
let identity = self
.durable
.node_identities
.get(node)
.get(&NodeScopeKey::from_refs(
&context.tenant,
&context.project,
node,
))
.ok_or(CoordinatorError::UnknownNode)?
.clone();
if identity.tenant != context.tenant || identity.project != context.project {
return Err(CoordinatorError::Unauthorized(
"node credential is outside the signed-in tenant/project scope".to_owned(),
));
}
if !matches!(context.actor, Actor::User(_)) {
return Err(CoordinatorError::Unauthorized(
"node credential revocation requires a user identity".to_owned(),
));
}
self.durable.node_identities.remove(node);
self.durable.credentials.remove(&format!("node:{node}"));
for active in self.active_processes.values_mut() {
active.connected_nodes.remove(node);
let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
self.durable.node_identities.remove(&key);
self.durable.credentials.remove(&key.credential_subject());
for active in self
.active_processes
.values_mut()
.filter(|active| active.tenant == context.tenant && active.project == context.project)
{
active.connected_nodes.remove(&key.node);
}
Ok(identity)
}
@ -580,8 +585,15 @@ impl Coordinator {
.count()
}
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
self.durable.node_identities.get(id)
pub fn node_identity(
&self,
tenant: &TenantId,
project: &ProjectId,
id: &NodeId,
) -> Option<&NodeIdentityRecord> {
self.durable
.node_identities
.get(&NodeScopeKey::from_refs(tenant, project, id))
}
pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@ -683,12 +695,24 @@ mod tests {
.contains_key(&TenantId::from("tenant")));
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
assert_eq!(
restarted
.durable
.credentials
.get("node:node")
.get(&node_subject)
.map(|credential| &credential.kind),
Some(&CredentialKind::NodeCredential)
);
@ -712,7 +736,12 @@ mod tests {
);
assert_eq!(rerun.coordinator_epoch, 2);
restarted
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&process, 2)),
)
.unwrap();
assert!(restarted
.active_process(
@ -725,6 +754,68 @@ mod tests {
.contains(&NodeId::from("node")));
}
#[test]
fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
let store = InMemoryDurableStore::default();
let mut coordinator = Coordinator::boot(&store, 1);
let node = NodeId::from("shared-node");
let scopes = [
(TenantId::from("tenant-a"), ProjectId::from("project-a")),
(TenantId::from("tenant-b"), ProjectId::from("project-b")),
(TenantId::from("tenant-a"), ProjectId::from("project-c")),
];
for (index, (tenant, project)) in scopes.iter().enumerate() {
let mut grant = coordinator.create_node_enrollment_grant(
tenant.clone(),
project.clone(),
format!("grant-{index}"),
"node:attach",
100,
);
coordinator
.exchange_node_enrollment_grant(
&mut grant,
node.clone(),
&format!("public-key-{index}"),
"node:attach",
99,
)
.unwrap();
}
let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
assert_eq!(
coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
coordinator.durable.node_identities.get(&scope_a)
);
assert_eq!(
coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
coordinator.durable.node_identities.get(&scope_b)
);
assert_eq!(
coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
coordinator.durable.node_identities.get(&scope_c)
);
assert!(coordinator
.durable
.credentials
.contains_key(&scope_a.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_b.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_c.credential_subject()));
assert_eq!(coordinator.durable.node_identities.len(), 3);
assert_eq!(coordinator.durable.credentials.len(), 3);
}
#[test]
fn identical_process_ids_are_isolated_by_tenant_and_project() {
let store = InMemoryDurableStore::default();
@ -773,11 +864,18 @@ mod tests {
let mut restarted = Coordinator::boot(&store, 2);
restarted
.reconnect_node(&NodeId::from("node"), None)
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
None,
)
.unwrap();
let error = restarted
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -809,7 +907,13 @@ mod tests {
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
}
#[test]
@ -823,10 +927,16 @@ mod tests {
"public-key",
"node:attach",
);
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
coordinator.durable.credentials.insert(
"node:node".to_owned(),
node_subject.clone(),
CredentialRecord {
subject: "node:node".to_owned(),
subject: node_subject.clone(),
tenant: TenantId::from("tenant"),
project: Some(ProjectId::from("project")),
kind: CredentialKind::NodeCredential,
@ -840,6 +950,8 @@ mod tests {
);
coordinator
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -855,7 +967,7 @@ mod tests {
&NodeId::from("node"),
)
.unwrap_err();
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
assert!(matches!(foreign, CoordinatorError::UnknownNode));
let revoked = coordinator
.revoke_node_credential(
@ -868,8 +980,14 @@ mod tests {
)
.unwrap();
assert_eq!(revoked.id, NodeId::from("node"));
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
assert!(!coordinator.durable.credentials.contains_key("node:node"));
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_none());
assert!(!coordinator.durable.credentials.contains_key(&node_subject));
assert!(!coordinator
.active_process(
&TenantId::from("tenant"),
@ -1084,7 +1202,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
assert!(matches!(error, CoordinatorError::UnknownNode));
}
#[test]

142
crates/clusterflux-coordinator/src/postgres_store.rs
View file

@ -4,8 +4,8 @@ use thiserror::Error;
use crate::{
AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
state.projects.insert(record.id.clone(), record);
}
for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM clusterflux_node_identities ORDER BY node_id",
"SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
)? {
state.node_identities.insert(record.id.clone(), record);
state.node_identities.insert(
NodeScopeKey::new(
record.tenant.clone(),
record.project.clone(),
record.id.clone(),
),
record,
);
}
for record in self.query_records::<CredentialRecord>(
"SELECT record FROM clusterflux_credentials ORDER BY subject",
@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
);
CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
node_id TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, node_id)
);
DO $clusterflux_node_scope_migration$
DECLARE
current_primary_key TEXT;
current_primary_key_columns TEXT[];
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_node_identities
WHERE record->>'id' IS DISTINCT FROM node_id
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node identity migration refused an internally inconsistent legacy row';
END IF;
SELECT
constraint_record.conname,
array_agg(attribute.attname ORDER BY key_column.ordinality)
INTO current_primary_key, current_primary_key_columns
FROM pg_constraint AS constraint_record
CROSS JOIN LATERAL unnest(constraint_record.conkey)
WITH ORDINALITY AS key_column(attribute_number, ordinality)
JOIN pg_attribute AS attribute
ON attribute.attrelid = constraint_record.conrelid
AND attribute.attnum = key_column.attribute_number
WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
AND constraint_record.contype = 'p'
GROUP BY constraint_record.conname;
IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
EXECUTE format(
'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
current_primary_key
);
ALTER TABLE clusterflux_node_identities
ADD PRIMARY KEY (tenant_id, project_id, node_id);
ELSIF current_primary_key_columns
IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
THEN
RAISE EXCEPTION
'clusterflux node identity migration found an unexpected primary key shape';
END IF;
END
$clusterflux_node_scope_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
record JSONB NOT NULL
);
DO $clusterflux_node_credential_migration$
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_credentials
WHERE record->>'subject' IS DISTINCT FROM subject
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node credential migration refused an internally inconsistent legacy row';
END IF;
UPDATE clusterflux_credentials AS credential
SET subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
),
record = jsonb_set(
credential.record,
'{subject}',
to_jsonb(format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)),
false
)
FROM clusterflux_node_identities AS identity
WHERE credential.subject = 'node:' || identity.node_id
AND credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id;
IF EXISTS (
SELECT 1
FROM clusterflux_credentials AS credential
WHERE credential.subject LIKE 'node:%'
AND NOT EXISTS (
SELECT 1
FROM clusterflux_node_identities AS identity
WHERE credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id
AND credential.subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)
)
) THEN
RAISE EXCEPTION
'clusterflux node credential migration found an unscoped or orphaned node subject';
END IF;
END
$clusterflux_node_credential_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -529,7 +651,13 @@ mod tests {
let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
assert_eq!(restarted.active_process_count(), 0);
}

19
crates/clusterflux-coordinator/src/service.rs
View file

@ -13,7 +13,7 @@ use clusterflux_core::{
};
use thiserror::Error;
use crate::{Coordinator, CoordinatorError};
use crate::{Coordinator, CoordinatorError, NodeScopeKey};
mod admin;
mod artifacts;
@ -154,8 +154,8 @@ pub enum CoordinatorServiceError {
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
@ -180,7 +180,7 @@ pub struct CoordinatorService {
process_cancellations: BTreeSet<ProcessControlKey>,
process_aborts: BTreeSet<ProcessControlKey>,
agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
node_replay_nonces: BTreeMap<(NodeId, String), u64>,
node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
stopped_panels: BTreeSet<PanelStopKey>,
panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@ -284,15 +284,10 @@ impl CoordinatorService {
{
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node process-control request is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
return Ok(());
}
self.coordinator

51
crates/clusterflux-coordinator/src/service/artifacts.rs
View file

@ -8,7 +8,7 @@ use clusterflux_core::{
};
use sha2::{Digest as _, Sha256};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::relay::RelayFinishReason;
use super::{
@ -52,7 +52,11 @@ impl CoordinatorService {
let action = self
.artifact_registry
.download_action(&context, &artifact, &policy)?;
self.ensure_download_source_connectivity(&action.source)?;
self.ensure_download_source_connectivity(
&context.tenant,
&context.project,
&action.source,
)?;
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
@ -163,7 +167,11 @@ impl CoordinatorService {
},
&mut validation_meter,
)?;
self.ensure_download_source_connectivity(&stream.link.source)?;
self.ensure_download_source_connectivity(
&stream.link.tenant,
&stream.link.project,
&stream.link.source,
)?;
self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@ -302,7 +310,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?;
@ -410,7 +418,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination =
@ -682,15 +690,10 @@ impl CoordinatorService {
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer node is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
Ok(())
}
@ -725,15 +728,12 @@ impl CoordinatorService {
) -> Result<NodeEndpoint, CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact export node is outside the tenant/project scope".to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity"
))
@ -744,7 +744,7 @@ impl CoordinatorService {
)
.into());
}
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export"
@ -769,17 +769,20 @@ impl CoordinatorService {
fn ensure_download_source_connectivity(
&self,
tenant: &TenantId,
project: &ProjectId,
source: &StorageLocation,
) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else {
return Ok(());
};
let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download"
))
})?;
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} is offline for artifact download"

32
crates/clusterflux-coordinator/src/service/debug_requests.rs
View file

@ -226,24 +226,25 @@ impl CoordinatorService {
} else if !completed_event_observed {
"selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
} else if let Some(checkpoint) = checkpoint {
let vfs_available =
checkpoint
.checkpoint
.vfs_manifest
.objects
.iter()
.all(|(path, object)| {
let artifact = super::keys::artifact_id_from_path(path);
self.artifact_registry
.metadata(&artifact)
let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
true,
|available, (path, object)| {
let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?;
Ok::<_, CoordinatorServiceError>(
available
&& self
.artifact_registry
.metadata(&tenant, &project, &artifact)
.is_some_and(|metadata| {
metadata.tenant == tenant
&& metadata.project == project
&& metadata.digest == object.digest
metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
})
});
}),
)
},
)?;
if !vfs_available {
"selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
} else {
@ -479,6 +480,7 @@ impl CoordinatorService {
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
self.maybe_retire_terminal_process(&tenant, &project, &process)?;
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,

45
crates/clusterflux-coordinator/src/service/keys.rs
View file

@ -1,6 +1,7 @@
use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
use thiserror::Error;
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
(tenant.clone(), project.clone(), grant.to_owned())
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub(super) enum ArtifactPathError {
#[error("path must start with the exact /vfs/artifacts/ prefix")]
WrongPrefix,
#[error("path must name an artifact after /vfs/artifacts/")]
EmptyArtifact,
#[error("mapped artifact identifier is invalid: {0}")]
InvalidArtifactId(#[from] clusterflux_core::IdParseError),
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
let value = path
.as_str()
.strip_prefix("/vfs/artifacts/")
.unwrap_or(path.as_str())
.replace('/', ":");
ArtifactId::new(value)
.ok_or(ArtifactPathError::WrongPrefix)?;
if value.is_empty() {
return Err(ArtifactPathError::EmptyArtifact);
}
ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn artifact_id_conversion_is_fallible_and_structural() {
assert_eq!(
artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
ArtifactId::from("build:output")
);
for path in [
"/vfs/other/output",
"/vfs/artifacts",
"/vfs/artifacts/bad artifact!",
] {
let path = VfsPath::new(path).unwrap();
assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
}
let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
}
}

151
crates/clusterflux-coordinator/src/service/logs.rs
View file

@ -1,5 +1,5 @@
use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
};
@ -85,7 +85,9 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant,
project,
process: process.clone(),
@ -203,7 +205,9 @@ impl CoordinatorService {
event.placement = self.task_placements.remove(&task_key);
if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant: event.tenant.clone(),
project: event.project.clone(),
process: event.process.clone(),
@ -217,34 +221,16 @@ impl CoordinatorService {
self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key);
let no_active_tasks =
!self
.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &event.tenant
&& task_project == &event.project
&& task_process == &event.process
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != event.tenant
|| assignment.project != event.project
|| assignment.process != event.process
|| assignment.node != event.node
|| assignment.task != event.task
});
let completed_after_main = matches!(event.terminal_state, TaskTerminalState::Completed)
&& self.task_events.iter().rev().any(|retained| {
retained.tenant == event.tenant
&& retained.project == event.project
&& retained.process == event.process
&& matches!(retained.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(retained.terminal_state, TaskTerminalState::Completed)
!assignments.is_empty()
});
if no_active_tasks {
self.process_aborts.remove(&process_key);
if (self.process_cancellations.remove(&process_key) || completed_after_main)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
}
if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key(
&event.tenant,
@ -261,6 +247,7 @@ impl CoordinatorService {
if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
Ok(CoordinatorResponse::TaskRecorded {
process: event.process,
task: event.task,
@ -553,6 +540,82 @@ impl CoordinatorService {
awaiting_operator
}
pub(super) fn maybe_retire_terminal_process(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<bool, CoordinatorServiceError> {
let process_key = process_control_key(tenant, project, process);
if self.main_runtime.controls.contains_key(&process_key) {
return Ok(false);
}
let has_runnable_remote_work =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == tenant && task_project == project && task_process == process
})
|| self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
})
|| self.task_assignments.values().any(|assignments| {
assignments.iter().any(|assignment| {
&assignment.tenant == tenant
&& &assignment.project == project
&& &assignment.process == process
})
})
|| self.task_attempts.iter().any(
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
attempt_tenant == tenant
&& attempt_project == project
&& attempt_process == process
&& attempts.iter().rev().any(|attempt| {
attempt.current
&& matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
},
);
if has_runnable_remote_work {
return Ok(false);
}
let main_completed = self.task_events.iter().rev().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& matches!(event.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(event.terminal_state, TaskTerminalState::Completed)
});
let cancellation_completed = self.process_cancellations.contains(&process_key);
if !main_completed && !cancellation_completed {
return Ok(false);
}
self.process_aborts.remove(&process_key);
self.process_cancellations.remove(&process_key);
if self
.coordinator
.active_process(tenant, project, process)
.is_none()
{
return Ok(false);
}
self.coordinator.abort_process(tenant, project, process)?;
self.clear_debug_state_for_process(tenant, project, process);
self.clear_operator_panel_state(tenant, project, process);
Ok(true)
}
fn flush_artifact_metadata(
&mut self,
flush: ArtifactFlush,
@ -560,17 +623,25 @@ impl CoordinatorService {
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let pinned = self
.task_restart_checkpoints
.values()
.flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
.chain(
self.pending_task_launches
.iter()
.flat_map(|pending| pending.task_spec.required_artifacts.iter()),
)
.cloned()
.collect::<std::collections::BTreeSet<ArtifactId>>();
let mut pinned = std::collections::BTreeSet::new();
for checkpoint in self.task_restart_checkpoints.values() {
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&checkpoint.assignment.tenant,
&checkpoint.assignment.project,
artifact,
));
}
}
for pending in &self.pending_task_launches {
for artifact in &pending.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&pending.tenant,
&pending.project,
artifact,
));
}
}
self.artifact_registry
.flush_metadata_bounded(flush, &pinned)
.map(|_| ())

14
crates/clusterflux-coordinator/src/service/main_runtime.rs
View file

@ -971,16 +971,10 @@ impl CoordinatorService {
}
}
let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
let has_active_children =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &scope.tenant
&& task_project == &scope.project
&& task_process == &scope.process
});
self.main_runtime.controls.remove(&process_key);
if main_completed && has_active_children {
if main_completed {
let _ =
self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
return;
}
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
@ -1169,7 +1163,7 @@ mod tests {
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
assert!(service.process_aborts.contains(&process_key));
assert!(!service.process_aborts.contains(&process_key));
}
#[test]

76
crates/clusterflux-coordinator/src/service/nodes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
SourceProviderKind, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{
bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@ -27,9 +27,9 @@ impl CoordinatorService {
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.get(scope)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
@ -41,7 +41,11 @@ impl CoordinatorService {
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
&descriptor.tenant,
&descriptor.project,
&descriptor.id,
));
descriptor
})
.collect()
@ -165,7 +169,11 @@ impl CoordinatorService {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
self.coordinator.ensure_tenant_active(&tenant)?;
if self.coordinator.node_identity(&node).is_none() {
if self
.coordinator
.node_identity(&tenant, &project, &node)
.is_none()
{
self.quota.ensure_node_admission(
&tenant,
self.coordinator.node_identity_count_for_tenant(&tenant),
@ -197,12 +205,21 @@ impl CoordinatorService {
pub(super) fn handle_node_heartbeat(
&mut self,
tenant: String,
project: String,
node: String,
node_signature: Option<NodeSignedRequest>,
payload_digest: &Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
self.authenticate_node_request(
&NodeScopeKey::new(tenant, project, node.clone()),
node_signature,
"node_heartbeat",
payload_digest,
)?;
Ok(CoordinatorResponse::NodeHeartbeat {
node,
epoch: self.coordinator.coordinator_epoch(),
@ -225,16 +242,13 @@ impl CoordinatorService {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node capability report is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
capabilities.validate_public_report()?;
for (kind, count) in [
("cached environments", cached_environment_digests.len()),
@ -274,12 +288,16 @@ impl CoordinatorService {
.into_iter()
.map(ArtifactId::new)
.collect::<BTreeSet<_>>();
self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations);
self.artifact_registry.reconcile_node_retention(
&tenant,
&project,
&node,
&artifact_locations,
);
let online = self.node_is_live(&node);
let online = self.node_is_live(&node_scope);
self.node_descriptors.insert(
node.clone(),
node_scope,
NodeDescriptor {
id: node.clone(),
tenant,
@ -333,9 +351,13 @@ impl CoordinatorService {
actor: Actor::User(actor.clone()),
};
self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
self.node_last_seen_epoch_seconds.remove(&node_scope);
self.node_replay_nonces
.retain(|(retained_scope, _), _| retained_scope != &node_scope);
self.artifact_registry
.garbage_collect_node(&tenant, &project, &node);
let queued_assignments_removed = self
.task_assignments
.remove(&(tenant.clone(), project.clone(), node.clone()))
@ -369,14 +391,14 @@ impl CoordinatorService {
pub(super) fn authenticate_node_request(
&mut self,
node: &NodeId,
scope: &NodeScopeKey,
node_signature: Option<NodeSignedRequest>,
request_kind: &str,
payload_digest: &Digest,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(&scope.tenant, &scope.project, &scope.node)
.ok_or(CoordinatorError::UnknownNode)?;
let signature = node_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
@ -401,7 +423,7 @@ impl CoordinatorService {
)
.into());
}
let replay_key = (node.clone(), signature.nonce.clone());
let replay_key = (scope.clone(), signature.nonce.clone());
self.node_replay_nonces.retain(|_, accepted_at| {
now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
});
@ -413,7 +435,7 @@ impl CoordinatorService {
}
verify_node_request_signature(
&identity.public_key,
node,
&scope.node,
request_kind,
payload_digest,
&signature,
@ -422,7 +444,7 @@ impl CoordinatorService {
if self
.node_replay_nonces
.keys()
.filter(|(retained_node, _)| retained_node == node)
.filter(|(retained_scope, _)| retained_scope == scope)
.count()
>= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
{
@ -436,8 +458,8 @@ impl CoordinatorService {
.insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
.insert(scope.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
descriptor.online = true;
}
Ok(())

3
crates/clusterflux-coordinator/src/service/panels.rs
View file

@ -254,7 +254,8 @@ impl CoordinatorService {
.rev()
.find_map(|event| event.artifact_path.as_ref())
{
let artifact = artifact_id_from_path(path);
let artifact = artifact_id_from_path(path)
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
let context = clusterflux_core::AuthContext {
tenant,
project,

23
crates/clusterflux-coordinator/src/service/process_launch.rs
View file

@ -118,14 +118,14 @@ impl CoordinatorService {
let mut objects = BTreeMap::new();
let mut missing_required_artifact = false;
for artifact in &task_spec.required_artifacts {
let Some(metadata) = self.artifact_registry.metadata(artifact) else {
let Some(metadata) =
self.artifact_registry
.metadata(&assignment.tenant, &assignment.project, artifact)
else {
missing_required_artifact = true;
continue;
};
if metadata.tenant != assignment.tenant
|| metadata.project != assignment.project
|| metadata.retaining_nodes.is_empty()
{
if metadata.retaining_nodes.is_empty() {
missing_required_artifact = true;
continue;
}
@ -483,17 +483,14 @@ impl CoordinatorService {
.validate()
.map_err(CoordinatorServiceError::Protocol)?;
for artifact in &task_spec.required_artifacts {
let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
let metadata = self
.artifact_registry
.metadata(&tenant, &project, artifact)
.ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired"
"required artifact {artifact} is unavailable or has expired in this tenant/project scope"
))
})?;
if metadata.tenant != tenant || metadata.project != project {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is outside the task tenant/project scope"
))
.into());
}
if metadata.retaining_nodes.is_empty() {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} has no retaining node"

40
crates/clusterflux-coordinator/src/service/processes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::keys::{process_control_key, task_control_key};
use super::{
@ -47,14 +47,10 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"task assignment poll is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let assignment_key = (tenant.clone(), project.clone(), node.clone());
let assignment = self
.task_assignments
@ -77,7 +73,11 @@ impl CoordinatorService {
project: &ProjectId,
node: &NodeId,
) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
let Some(descriptor) = self
.node_descriptors
.get(&NodeScopeKey::from_refs(tenant, project, node))
.cloned()
else {
return Ok(None);
};
let mut remaining = VecDeque::new();
@ -233,16 +233,14 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"source preparation completion is outside the enrolled tenant/project scope"
.to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let descriptor = self
.node_descriptors
.get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
@ -413,14 +411,18 @@ impl CoordinatorService {
pub(super) fn handle_reconnect_node(
&mut self,
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let process = ProcessId::new(process);
self.coordinator
.reconnect_node(&node, Some((&process, epoch)))?;
.reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
Ok(CoordinatorResponse::NodeReconnected { node, process })
}

20
crates/clusterflux-coordinator/src/service/protocol.rs
View file

@ -121,6 +121,8 @@ pub enum CoordinatorRequest {
enrollment_grant: String,
},
NodeHeartbeat {
tenant: String,
project: String,
node: String,
#[serde(default)]
node_signature: Option<NodeSignedRequest>,
@ -269,6 +271,8 @@ pub enum CoordinatorRequest {
restart: bool,
},
ReconnectNode {
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
@ -675,9 +679,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512)
}
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
if let Some(signature) = node_signature {
validate_node_signature(signature, &format!("{path}.node_signature"))?;
@ -846,7 +853,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
&format!("{path}.launch_attempt"),
)
}
CoordinatorRequest::ReconnectNode { node, process, .. } => {
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
..
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
validate_process(process, &format!("{path}.process"))
}
@ -1567,6 +1581,8 @@ mod external_identifier_tests {
restart: false,
},
CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "bad node!".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: "node-nonce".to_owned(),
@ -1626,6 +1642,8 @@ mod external_identifier_tests {
assert!(error.contains("malformed external token request.agent_signature.nonce"));
let node_request = CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "node".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: String::new(),

10
crates/clusterflux-coordinator/src/service/routing.rs
View file

@ -274,9 +274,17 @@ impl CoordinatorService {
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
} => self.handle_node_heartbeat(
tenant,
project,
node,
node_signature,
&request_payload_digest,
),
CoordinatorRequest::SignedNode {
node,
node_signature,

155
crates/clusterflux-coordinator/src/service/signed_nodes.rs
View file

@ -1,6 +1,6 @@
use clusterflux_core::{NodeId, NodeSignedRequest};
use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
@ -12,7 +12,7 @@ impl CoordinatorService {
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request_kind = signed_node_request_kind(&request)?;
let request_node = signed_node_request_node(&request)?;
let request_scope = signed_node_request_scope(&request)?;
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize signed node request: {error}"
@ -22,14 +22,14 @@ impl CoordinatorService {
let signed_node = NodeId::try_new(signed_node).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
})?;
if request_node != signed_node {
if request_scope.node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
)
.into());
}
self.authenticate_node_request(
&signed_node,
&request_scope,
Some(node_signature),
request_kind,
&payload_digest,
@ -160,10 +160,12 @@ impl CoordinatorService {
failure_reason,
),
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
epoch,
} => self.handle_reconnect_node(node, process, epoch),
} => self.handle_reconnect_node(tenant, project, node, process, epoch),
CoordinatorRequest::PollTaskControl {
tenant,
project,
@ -353,36 +355,129 @@ fn signed_node_request_kind(
}
}
fn signed_node_request_node(
fn signed_node_request_scope(
request: &CoordinatorRequest,
) -> Result<NodeId, CoordinatorServiceError> {
) -> Result<NodeScopeKey, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { node, .. }
| CoordinatorRequest::PollTaskAssignment { node, .. }
| CoordinatorRequest::PollArtifactTransfer { node, .. }
| CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
| CoordinatorRequest::FailArtifactTransfer { node, .. }
| CoordinatorRequest::LaunchChildTask { node, .. }
| CoordinatorRequest::JoinChildTask { node, .. }
| CoordinatorRequest::CompleteSourcePreparation { node, .. }
| CoordinatorRequest::ReconnectNode { node, .. }
| CoordinatorRequest::PollTaskControl { node, .. }
| CoordinatorRequest::PollDebugCommand { node, .. }
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => {
NodeId::try_new(node.clone()).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"invalid wrapped node identifier: {error}"
))
})
CoordinatorRequest::ReportNodeCapabilities {
tenant,
project,
node,
..
}
CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()),
| CoordinatorRequest::PollTaskAssignment {
tenant,
project,
node,
}
| CoordinatorRequest::PollArtifactTransfer {
tenant,
project,
node,
}
| CoordinatorRequest::UploadArtifactTransferChunk {
tenant,
project,
node,
..
}
| CoordinatorRequest::FailArtifactTransfer {
tenant,
project,
node,
..
}
| CoordinatorRequest::LaunchChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::JoinChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::CompleteSourcePreparation {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollTaskControl {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollDebugCommand {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugState {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugProbeHit {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportTaskLog {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
node,
..
}
| CoordinatorRequest::TaskCompleted {
tenant,
project,
node,
..
} => node_scope_from_strings(tenant, project, node),
CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
scope.tenant.clone(),
scope.project.clone(),
source.node.clone(),
)),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}
fn node_scope_from_strings(
tenant: &str,
project: &str,
node: &str,
) -> Result<NodeScopeKey, CoordinatorServiceError> {
let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
})?;
let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
})?;
let node = NodeId::try_new(node.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
})?;
Ok(NodeScopeKey::new(tenant, project, node))
}

1076
crates/clusterflux-coordinator/src/service/tests.rs
View file

File diff suppressed because it is too large Load diff

457
crates/clusterflux-core/src/artifact.rs
View file

@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
pub coordinator_has_large_bytes: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ArtifactScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub artifact: ArtifactId,
}
impl ArtifactScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
Self {
tenant,
project,
artifact,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
Self::new(tenant.clone(), project.clone(), artifact.clone())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactFlush {
pub id: ArtifactId,
@ -148,7 +169,7 @@ pub struct ArtifactFlush {
#[derive(Clone, Debug, Default)]
pub struct ArtifactRegistry {
artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
next_epoch: u64,
}
@ -162,9 +183,10 @@ impl ArtifactRegistry {
pub fn flush_metadata_bounded(
&mut self,
flush: ArtifactFlush,
pinned: &BTreeSet<ArtifactId>,
pinned: &BTreeSet<ArtifactScopeKey>,
) -> Result<ArtifactMetadata, String> {
let replacing_existing = self.artifacts.contains_key(&flush.id);
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
let replacing_existing = self.artifacts.contains_key(&key);
while !replacing_existing
&& self
.artifacts
@ -184,13 +206,25 @@ impl ArtifactRegistry {
metadata.tenant == flush.tenant
&& metadata.project == flush.project
&& metadata.process == flush.process
&& !pinned.contains(&metadata.id)
&& !pinned.contains(&ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
))
&& !self.issued_download_links.values().any(|issued| {
issued.link.artifact == metadata.id
issued.link.tenant == metadata.tenant
&& issued.link.project == metadata.project
&& issued.link.artifact == metadata.id
})
})
.min_by_key(|metadata| metadata.flushed_epoch)
.map(|metadata| metadata.id.clone())
.map(|metadata| {
ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
)
})
.ok_or_else(|| {
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
.to_owned()
@ -212,36 +246,47 @@ impl ArtifactRegistry {
explicit_locations: Vec::new(),
coordinator_has_large_bytes: false,
};
self.artifacts.insert(flush.id, metadata.clone());
self.artifacts.insert(key, metadata.clone());
Ok(metadata)
}
pub fn sync_to_explicit_store(
&mut self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
location: impl Into<String>,
) -> Result<(), ArtifactUnavailable> {
let metadata = self
.artifacts
.get_mut(artifact)
.get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
.ok_or(ArtifactUnavailable)?;
metadata.explicit_locations.push(location.into());
Ok(())
}
pub fn garbage_collect_node(&mut self, node: &NodeId) {
for metadata in self.artifacts.values_mut() {
pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
for metadata in self
.artifacts
.values_mut()
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
{
metadata.retaining_nodes.remove(node);
}
}
pub fn reconcile_node_retention(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
retained_artifacts: &BTreeSet<ArtifactId>,
) {
for (artifact, metadata) in &mut self.artifacts {
if retained_artifacts.contains(artifact) {
for (key, metadata) in &mut self.artifacts {
if &key.tenant != tenant || &key.project != project {
continue;
}
if retained_artifacts.contains(&key.artifact) {
metadata.retaining_nodes.insert(node.clone());
} else {
metadata.retaining_nodes.remove(node);
@ -249,8 +294,14 @@ impl ArtifactRegistry {
}
}
pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
self.artifacts.get(artifact)
pub fn metadata(
&self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
) -> Option<&ArtifactMetadata> {
self.artifacts
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
}
pub fn download_action(
@ -261,7 +312,11 @@ impl ArtifactRegistry {
) -> Result<DownloadAction, DownloadError> {
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let scope = Scope {
tenant: metadata.tenant.clone(),
@ -315,7 +370,11 @@ impl ArtifactRegistry {
self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
Ok(metadata.size)
}
@ -333,7 +392,11 @@ impl ArtifactRegistry {
if self
.issued_download_links
.values()
.filter(|issued| issued.link.artifact == *artifact)
.filter(|issued| {
issued.link.tenant == context.tenant
&& issued.link.project == context.project
&& issued.link.artifact == *artifact
})
.count()
>= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
{
@ -345,7 +408,11 @@ impl ArtifactRegistry {
let action = self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let policy_context_digest =
@ -398,7 +465,11 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact || issued.link.actor != context.actor {
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.actor != context.actor
{
return Err(DownloadError::InvalidToken);
}
self.download_action(
@ -434,7 +505,9 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.max_bytes != policy.max_bytes
|| issued.link.actor != context.actor
{
@ -452,7 +525,11 @@ impl ArtifactRegistry {
}
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
if download_policy_context_digest(metadata, &action.source, policy)
!= issued.link.policy_context_digest
@ -477,7 +554,11 @@ impl ArtifactRegistry {
) -> Result<(), DownloadError> {
let metadata = self
.artifacts
.get(&stream.link.artifact)
.get(&ArtifactScopeKey::from_refs(
&stream.link.tenant,
&stream.link.project,
&stream.link.artifact,
))
.ok_or(DownloadError::NotFound)?;
if !source_is_available(metadata, &stream.link.source) {
return Err(DownloadError::Unavailable);
@ -596,7 +677,13 @@ mod tests {
#[test]
fn flush_publishes_metadata_without_coordinator_bytes() {
let registry = registry_with_artifact();
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(!metadata.coordinator_has_large_bytes);
assert_eq!(metadata.id, ArtifactId::from("artifact"));
@ -615,7 +702,11 @@ mod tests {
#[test]
fn unsynced_node_loss_surfaces_as_unavailable() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -636,9 +727,20 @@ mod tests {
#[test]
fn signed_node_retention_inventory_removes_garbage_collected_location() {
let mut registry = registry_with_artifact();
registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
&BTreeSet::new(),
);
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(metadata.retaining_nodes.is_empty());
}
@ -648,11 +750,26 @@ mod tests {
let mut registry = registry_with_artifact();
let node = NodeId::from("node");
let artifact = ArtifactId::from("artifact");
registry.garbage_collect_node(&node);
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
);
registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()]));
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
&BTreeSet::from([artifact.clone()]),
);
let metadata = registry.metadata(&artifact).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&artifact,
)
.unwrap();
assert!(metadata.retaining_nodes.contains(&node));
}
@ -660,9 +777,18 @@ mod tests {
fn explicit_user_storage_location_survives_node_retention_loss() {
let mut registry = registry_with_artifact();
registry
.sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
.sync_to_explicit_store(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
"s3://bucket/app",
)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -683,7 +809,11 @@ mod tests {
);
assert!(
!registry
.metadata(&ArtifactId::from("artifact"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap()
.coordinator_has_large_bytes
);
@ -714,7 +844,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
@ -734,13 +864,206 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
let artifact = ArtifactId::from("shared-artifact");
let tenant_a = TenantId::from("tenant-a");
let project_a = ProjectId::from("project-a");
let tenant_b = TenantId::from("tenant-b");
let project_b = ProjectId::from("project-b");
let project_c = ProjectId::from("project-c");
let node_a = NodeId::from("node-a");
let node_b = NodeId::from("node-b");
let node_c = NodeId::from("node-c");
let mut registry = ArtifactRegistry::default();
for (tenant, project, process, node, content, size) in [
(&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
(&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
(&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
] {
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::from(process),
producer_task: TaskInstanceId::from("producer"),
retaining_node: node.clone(),
digest: Digest::sha256(content),
size,
});
}
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
assert_eq!(
registry
.metadata(&tenant_a, &project_c, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-c")
);
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant_a.clone(),
project: project_a.clone(),
process: ProcessId::from("process-a"),
producer_task: TaskInstanceId::from("producer-repeat"),
retaining_node: node_a.clone(),
digest: Digest::sha256("bytes-a-repeat"),
size: 18,
});
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a-repeat")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
registry
.sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
.unwrap();
registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
let context_a = AuthContext {
tenant: tenant_a.clone(),
project: project_a.clone(),
actor: Actor::User(UserId::from("user")),
};
let context_b = AuthContext {
tenant: tenant_b.clone(),
project: project_b.clone(),
actor: Actor::User(UserId::from("user")),
};
let policy = DownloadPolicy { max_bytes: 100 };
assert_eq!(
registry
.download_action(&context_a, &artifact, &policy)
.unwrap()
.source,
StorageLocation::ExplicitStore("store://tenant-a".to_owned())
);
assert_eq!(
registry
.download_action(&context_b, &artifact, &policy)
.unwrap()
.source,
StorageLocation::RetainedNode(node_b)
);
let first_a = registry
.create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
let first_b = registry
.create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
assert_ne!(
first_a.scoped_token_digest, first_b.scoped_token_digest,
"the full artifact scope must contribute to download tokens"
);
for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
registry
.create_download_link(
&context_a,
&artifact,
&policy,
&format!("tenant-a-{index}"),
10,
60,
)
.unwrap();
}
assert!(matches!(
registry.create_download_link(
&context_a,
&artifact,
&policy,
"tenant-a-over-limit",
10,
60,
),
Err(DownloadError::Usage(_))
));
registry
.create_download_link(
&context_b,
&artifact,
&policy,
"tenant-b-still-independent",
10,
60,
)
.unwrap();
registry
.revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
.unwrap();
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
};
let mut meter = ResourceMeter::default();
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_b.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.expect("revoking tenant A's link must not revoke tenant B's link");
assert_eq!(
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_a.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.unwrap_err(),
DownloadError::InvalidToken
);
}
#[test]
fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -1019,10 +1342,24 @@ mod tests {
#[test]
fn artifact_metadata_is_bounded_without_evicting_pins() {
let mut registry = ArtifactRegistry::default();
registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("artifact-0"),
tenant: TenantId::from("other-tenant"),
project: ProjectId::from("other-project"),
process: ProcessId::from("process"),
producer_task: TaskInstanceId::from("other-task"),
retaining_node: NodeId::from("other-node"),
digest: Digest::sha256("other-content"),
size: 1,
});
let mut pinned = BTreeSet::new();
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
let id = ArtifactId::new(format!("artifact-{index}"));
pinned.insert(id.clone());
pinned.insert(ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
id.clone(),
));
registry
.flush_metadata_bounded(
ArtifactFlush {
@ -1039,7 +1376,10 @@ mod tests {
)
.unwrap();
}
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
let next = ArtifactFlush {
id: ArtifactId::from("artifact-next"),
tenant: TenantId::from("tenant"),
@ -1055,13 +1395,42 @@ mod tests {
.unwrap_err()
.contains("pinned"));
pinned.remove(&ArtifactId::from("artifact-0"));
pinned.remove(&ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
ArtifactId::from("artifact-0"),
));
registry.flush_metadata_bounded(next, &pinned).unwrap();
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
assert!(registry
.metadata(&ArtifactId::from("artifact-next"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-0"),
)
.is_none());
assert!(registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-next"),
)
.is_some());
assert_eq!(
registry
.metadata(
&TenantId::from("other-tenant"),
&ProjectId::from("other-project"),
&ArtifactId::from("artifact-0"),
)
.unwrap()
.digest,
Digest::sha256("other-content"),
"eviction and pins in one scope must not affect an equal ID in another scope"
);
}
#[test]
@ -1150,7 +1519,11 @@ mod tests {
registry
.stream_download_chunk(&mut stream, &limits, &mut meter, 16)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
assert_eq!(
registry

4
crates/clusterflux-core/src/lib.rs
View file

@ -20,8 +20,8 @@ pub mod wire;
pub use artifact::{
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
DownloadStreamRequest, RetentionPolicy, StorageLocation,
ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
};
pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest,

91
crates/clusterflux-core/src/vfs.rs
View file

@ -1,18 +1,48 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
use thiserror::Error;
use crate::{Digest, NodeId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub const MAX_VFS_PATH_BYTES: usize = 4096;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct VfsPath(String);
impl VfsPath {
pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
let path = path.into();
if !path.starts_with("/vfs/") {
return Err(VfsError::InvalidPath(path));
return Err(VfsError::invalid(path, "path must start with /vfs/"));
}
if path.len() > MAX_VFS_PATH_BYTES {
return Err(VfsError::invalid(
path,
"path exceeds the 4096-byte protocol limit",
));
}
if path.chars().any(char::is_control) {
return Err(VfsError::invalid(path, "control characters are forbidden"));
}
if path.contains('\\') {
return Err(VfsError::invalid(path, "backslashes are forbidden"));
}
let relative = &path["/vfs/".len()..];
if relative.is_empty() {
return Err(VfsError::invalid(
path,
"path after /vfs/ must not be empty",
));
}
if relative
.split('/')
.any(|component| component.is_empty() || matches!(component, "." | ".."))
{
return Err(VfsError::invalid(
path,
"empty, '.', and '..' path components are forbidden",
));
}
Ok(Self(path))
}
@ -22,6 +52,16 @@ impl VfsPath {
}
}
impl<'de> Deserialize<'de> for VfsPath {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let path = String::deserialize(deserializer)?;
Self::new(path).map_err(D::Error::custom)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsObject {
pub path: VfsPath,
@ -63,12 +103,18 @@ pub enum ReuseDecision {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum VfsError {
#[error("VFS path must start with /vfs/: {0}")]
InvalidPath(String),
#[error("invalid VFS path {path:?}: {reason}")]
InvalidPath { path: String, reason: &'static str },
#[error("path is not visible in the published VFS manifest: {0}")]
NotVisible(String),
}
impl VfsError {
fn invalid(path: String, reason: &'static str) -> Self {
Self::InvalidPath { path, reason }
}
}
#[derive(Clone, Debug)]
pub struct VfsOverlay {
task: TaskInstanceId,
@ -238,4 +284,39 @@ mod tests {
assert_eq!(overlay.pending_len(), 0);
}
#[test]
fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
for invalid in [
"vfs/artifacts/app".to_owned(),
"/vfs/".to_owned(),
"/vfs/artifacts//app".to_owned(),
"/vfs/artifacts/app/".to_owned(),
"/vfs/artifacts/./app".to_owned(),
"/vfs/artifacts/../app".to_owned(),
"/vfs/artifacts\\app".to_owned(),
"/vfs/artifacts/bad\0app".to_owned(),
format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
] {
assert!(
VfsPath::new(&invalid).is_err(),
"hostile VFS path unexpectedly passed: {invalid:?}"
);
assert!(
serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
"hostile VFS path unexpectedly deserialized"
);
}
}
#[test]
fn vfs_path_accepts_the_4096_byte_boundary() {
let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
let overlong = format!("{path}x");
assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
assert!(VfsPath::new(overlong).is_err());
}
}

4
crates/clusterflux-node/src/daemon.rs
View file

@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
});
let heartbeat_signature = sign_node_request(
@ -420,6 +422,8 @@ fn run_runtime_task(
"reconnect_node",
json!({
"type": "reconnect_node",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"process": &task.process,
"epoch": epoch,

8
scripts/artifact-download-smoke.js
View file

@ -194,7 +194,7 @@ function downloadNodeCapabilities() {
ttl_seconds: 60,
});
assert.strictEqual(crossTenant.type, "error");
assert.match(crossTenant.message, /tenant mismatch/);
assert.match(crossTenant.message, /artifact does not exist/);
const crossProject = await send(addr, {
type: "create_artifact_download_link",
@ -206,7 +206,7 @@ function downloadNodeCapabilities() {
ttl_seconds: 60,
});
assert.strictEqual(crossProject.type, "error");
assert.match(crossProject.message, /project mismatch/);
assert.match(crossProject.message, /artifact does not exist/);
const crossTenantOpen = await send(addr, {
type: "open_artifact_download_stream",
@ -219,7 +219,7 @@ function downloadNodeCapabilities() {
chunk_bytes: 1,
});
assert.strictEqual(crossTenantOpen.type, "error");
assert.match(crossTenantOpen.message, /tenant mismatch/);
assert.match(crossTenantOpen.message, /artifact does not exist/);
const crossProjectOpen = await send(addr, {
type: "open_artifact_download_stream",
@ -232,7 +232,7 @@ function downloadNodeCapabilities() {
chunk_bytes: 1,
});
assert.strictEqual(crossProjectOpen.type, "error");
assert.match(crossProjectOpen.message, /project mismatch/);
assert.match(crossProjectOpen.message, /artifact does not exist/);
const guessed = await send(addr, {
type: "open_artifact_download_stream",

2
scripts/artifact-export-smoke.js
View file

@ -231,7 +231,7 @@ async function reportNode(
failure_reason: "",
});
assert.strictEqual(crossTenant.type, "error");
assert.match(crossTenant.message, /tenant mismatch/);
assert.match(crossTenant.message, /artifact does not exist/);
await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] });
const failedDirect = await send(addr, {

728
scripts/cli-happy-path-live-smoke.js
View file

@ -81,17 +81,12 @@ function requireEnabled() {
const hasSecondTenantSession = Boolean(
process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE
);
const hasSingleAccountIsolationTarget = Boolean(
process.env.CLUSTERFLUX_ISOLATION_TARGET_TENANT &&
process.env.CLUSTERFLUX_ISOLATION_TARGET_PROJECT
);
if (
strictFullRelease &&
!hasSecondTenantSession &&
!hasSingleAccountIsolationTarget
!hasSecondTenantSession
) {
throw new Error(
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires either CLUSTERFLUX_SECOND_TENANT_SESSION_FILE or both CLUSTERFLUX_ISOLATION_TARGET_TENANT and CLUSTERFLUX_ISOLATION_TARGET_PROJECT"
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE so the compiled two-tenant Node and artifact collision can run"
);
}
if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) {
@ -500,8 +495,12 @@ async function runMalformedIdentifierSuite({
send: async (value) =>
sendHostedControl({
type: "node_heartbeat",
tenant,
project,
node: value,
node_signature: signedNodeHeartbeat(
tenant,
project,
value,
securityNode.identity,
{
@ -803,8 +802,12 @@ async function runMalformedIdentifierSuite({
send: async (value) => {
const request = {
type: "node_heartbeat",
tenant,
project,
node: securityNode.node,
node_signature: signedNodeHeartbeat(
tenant,
project,
securityNode.node,
securityNode.identity,
{ nonce: value }
@ -1616,8 +1619,10 @@ async function prepareLiveNodeCredentialSecurity({
const heartbeat = {
type: "node_heartbeat",
tenant,
project,
node,
node_signature: signedNodeHeartbeat(node, identity, {
node_signature: signedNodeHeartbeat(tenant, project, node, identity, {
nonce: `strict-valid-${suffix}`,
}),
};
@ -1632,8 +1637,10 @@ async function prepareLiveNodeCredentialSecurity({
const expired = assertDenied(
await sendHostedControl({
type: "node_heartbeat",
tenant,
project,
node,
node_signature: signedNodeHeartbeat(node, identity, {
node_signature: signedNodeHeartbeat(tenant, project, node, identity, {
nonce: `strict-expired-${suffix}`,
issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
}),
@ -1646,8 +1653,10 @@ async function prepareLiveNodeCredentialSecurity({
const forged = assertDenied(
await sendHostedControl({
type: "node_heartbeat",
tenant,
project,
node,
node_signature: signedNodeHeartbeat(node, forgedIdentity, {
node_signature: signedNodeHeartbeat(tenant, project, node, forgedIdentity, {
nonce: `strict-forged-${suffix}`,
}),
}),
@ -2556,6 +2565,12 @@ async function runLiveAgentCredentialSecurity({
}
async function runSecondTenantIsolation({
clusterflux,
clusterfluxNode,
firstScope,
firstHelloBuild,
firstHelloProjectDir,
workRoot,
firstSessionSecret,
firstTenant,
firstProject,
@ -2597,6 +2612,7 @@ async function runSecondTenantIsolation({
"reused isolation evidence must target the exact public binaries"
);
return {
evidence: {
...isolation,
duration_ms: Date.now() - startedAt,
reused_from_immutable_evidence: {
@ -2605,6 +2621,8 @@ async function runSecondTenantIsolation({
hosted_service_sha256: currentDeployment.hosted_service_sha256,
public_binary_digests: manifest.binary_digests,
},
},
runtime: null,
};
}
if (!file && targetTenant && targetProject) {
@ -2702,6 +2720,7 @@ async function runSecondTenantIsolation({
"single-account foreign process control"
);
return {
evidence: {
executed: true,
mode: "single_authenticated_account_foreign_project",
second_tenant: targetTenant,
@ -2719,19 +2738,38 @@ async function runSecondTenantIsolation({
artifact_and_download: artifact,
process_control: control,
duration_ms: Date.now() - startedAt,
},
runtime: null,
};
}
if (!file) return { executed: false, reason: "second tenant session not supplied" };
const second = readJson(path.resolve(file));
if (!file) {
return {
evidence: {
executed: false,
reason: "second tenant session not supplied",
duration_ms: Math.max(1, Date.now() - startedAt),
},
runtime: null,
};
}
const secondSessionPath = path.resolve(file);
const second = readJson(secondSessionPath);
assert.strictEqual(second.coordinator, serviceEndpoint);
assert(second.session_secret, "second tenant session omitted its session secret");
const secondSessionSecret =
second.cli_session_secret || second.session_secret;
assert(secondSessionSecret, "second tenant session omitted its session secret");
assert.notStrictEqual(
second.tenant,
firstTenant,
"isolation proof requires a genuinely distinct hosted tenant"
);
assert.notStrictEqual(
second.project,
firstProject,
"isolation proof requires a genuinely distinct hosted project"
);
const call = (request) =>
sendHostedControl(authenticatedRequest(second.session_secret, request));
sendHostedControl(authenticatedRequest(secondSessionSecret, request));
const project = assertDenied(
await call({ type: "select_project", project: firstProject }),
@ -2778,7 +2816,7 @@ async function runSecondTenantIsolation({
max_bytes: 1024,
ttl_seconds: 60,
}),
/outside|scope|tenant|project|not found|unavailable/i,
/outside|scope|tenant|project|not found|does not exist|unavailable/i,
"cross-tenant artifact download"
);
const control = assertDenied(
@ -2786,9 +2824,249 @@ async function runSecondTenantIsolation({
/outside|scope|tenant|project|not active|requires an active|unknown/i,
"cross-tenant process control"
);
const secondCheckout = path.join(workRoot, "second-tenant-public-repo");
stagePublicCheckout(manifest, secondCheckout);
const secondProjectDir = path.join(
secondCheckout,
"examples",
"hello-build"
);
const secondControlDir = path.join(secondProjectDir, ".clusterflux");
ensureDir(secondControlDir);
const secondProjectPath = path.join(
path.dirname(secondSessionPath),
"project.json"
);
assert(
fs.existsSync(secondProjectPath),
`second tenant session is missing adjacent project.json: ${secondProjectPath}`
);
fs.copyFileSync(
secondSessionPath,
path.join(secondControlDir, "session.json")
);
fs.copyFileSync(
secondProjectPath,
path.join(secondControlDir, "project.json")
);
fs.chmodSync(path.join(secondControlDir, "session.json"), 0o600);
fs.chmodSync(path.join(secondControlDir, "project.json"), 0o644);
const secondScope = [
"--coordinator",
serviceEndpoint,
"--tenant",
second.tenant,
"--project-id",
second.project,
"--user",
second.user,
"--json",
];
const secondProjectList = runJson(
clusterflux,
["project", "list", ...secondScope],
{ cwd: secondProjectDir }
);
assert(secondProjectList.project_count >= 1);
const secondProjectSelect = runJson(
clusterflux,
["project", "select", ...secondScope, second.project],
{ cwd: secondProjectDir }
);
assert.strictEqual(secondProjectSelect.command, "project select");
const collisionStartedAt = Date.now();
const secondGrant = runJson(
clusterflux,
["node", "enroll", ...secondScope],
{ cwd: secondProjectDir }
);
const secondAttach = runJson(
clusterflux,
[
"node",
"attach",
"--coordinator",
serviceEndpoint,
"--tenant",
second.tenant,
"--project-id",
second.project,
"--node",
firstNode,
"--enrollment-grant",
secondGrant.enrollment_grant.grant,
"--json",
],
{ cwd: secondProjectDir }
);
assert.strictEqual(secondAttach.boundary.used_enrollment_exchange, true);
const secondStored = readNodeCredential(secondProjectDir, firstNode);
const secondCredentialDigest = sha256(
fs.readFileSync(secondStored.absolute)
);
const secondIdentity = nodeIdentityFromPrivateKey(
secondStored.credential.private_key
);
const secondWorkerArgs = [
"--coordinator",
serviceEndpoint,
"--tenant",
second.tenant,
"--project-id",
second.project,
"--node",
firstNode,
"--worker",
"--project-root",
secondProjectDir,
"--assignment-poll-ms",
"500",
"--emit-ready",
];
const firstHelloWorker = spawnJsonLines(
clusterfluxNode,
[
"--coordinator",
serviceEndpoint,
"--tenant",
firstTenant,
"--project-id",
firstProject,
"--node",
firstNode,
"--worker",
"--project-root",
firstHelloProjectDir,
"--assignment-poll-ms",
"500",
"--emit-ready",
],
{
cwd: firstHelloProjectDir,
env: { ...process.env },
}
);
const firstHelloReady = await firstHelloWorker.waitFor(
(value) => value.node_status === "ready",
"first-tenant hello-build artifact worker ready for collision proof"
);
assert.strictEqual(firstHelloReady.node, firstNode);
const firstCollisionHelloBuild = await runHostedHelloBuild({
clusterflux,
projectDir: firstHelloProjectDir,
scope: firstScope,
outputFile: path.join(workRoot, "hello-clusterflux-first-tenant-collision"),
});
assert.strictEqual(
firstCollisionHelloBuild.artifact,
firstHelloBuild.artifact,
"repeated deterministic first-tenant hello-build changed its ArtifactId"
);
assert.strictEqual(firstCollisionHelloBuild.digest, firstHelloBuild.digest);
assert.strictEqual(
firstCollisionHelloBuild.executable_output,
firstHelloBuild.executable_output
);
const secondWorker = spawnJsonLines(clusterfluxNode, secondWorkerArgs, {
cwd: secondProjectDir,
env: { ...process.env },
});
let secondHelloBuild;
try {
const secondReady = await secondWorker.waitFor(
(value) => value.node_status === "ready",
"same-name second-tenant worker ready"
);
assert.strictEqual(secondReady.node, firstNode);
secondHelloBuild = await runHostedHelloBuild({
clusterflux,
projectDir: secondProjectDir,
scope: secondScope,
outputFile: path.join(workRoot, "hello-clusterflux-second-tenant"),
});
} finally {
await stopChild(secondWorker.child);
}
assert.strictEqual(
secondHelloBuild.artifact,
firstCollisionHelloBuild.artifact,
"deterministic two-tenant hello-build did not collide on ArtifactId"
);
assert.strictEqual(
secondHelloBuild.digest,
firstCollisionHelloBuild.digest,
"deterministic two-tenant hello-build did not produce identical bytes"
);
assert.strictEqual(
secondHelloBuild.executable_output,
firstCollisionHelloBuild.executable_output
);
const firstArtifactsAfterCollision = runJson(
clusterflux,
[
"artifact",
"list",
...firstScope,
"--process",
firstCollisionHelloBuild.process,
],
{ cwd: firstHelloProjectDir }
);
assert(
firstArtifactsAfterCollision.artifacts.some(
(candidate) =>
candidate.artifact === firstCollisionHelloBuild.artifact &&
candidate.digest === firstCollisionHelloBuild.digest
),
"the second tenant's same-ID artifact masked the first tenant's metadata"
);
const secondCollisionLink = await call({
type: "create_artifact_download_link",
artifact: secondHelloBuild.artifact,
max_bytes: secondHelloBuild.downloaded_bytes,
ttl_seconds: 60,
});
assert.strictEqual(
secondCollisionLink.type,
"artifact_download_link",
JSON.stringify(secondCollisionLink)
);
const secondCollisionLinkRevoked = await call({
type: "revoke_artifact_download_link",
artifact: secondHelloBuild.artifact,
token_digest: secondCollisionLink.link.scoped_token_digest,
});
assert.strictEqual(
secondCollisionLinkRevoked.type,
"artifact_download_link_revoked",
JSON.stringify(secondCollisionLinkRevoked)
);
const firstDownloadAfterSecondLinkRevocation = runJson(
clusterflux,
[
"artifact",
"download",
...firstScope,
firstCollisionHelloBuild.artifact,
"--to",
path.join(workRoot, "hello-clusterflux-first-after-second-link-revoke"),
],
{ cwd: firstHelloProjectDir }
);
assert.strictEqual(
firstDownloadAfterSecondLinkRevocation.local_download.verified_digest,
firstCollisionHelloBuild.digest
);
await stopChild(firstHelloWorker.child);
return {
evidence: {
executed: true,
second_tenant: second.tenant,
second_project: second.project,
project,
process_hidden: true,
node_hidden: true,
@ -2796,7 +3074,187 @@ async function runSecondTenantIsolation({
debug,
artifact_and_download: artifact,
process_control: control,
duration_ms: Date.now() - startedAt,
scoped_node_and_artifact_collision: {
request: {
node: firstNode,
first_scope: {
tenant: firstTenant,
project: firstProject,
},
second_scope: {
tenant: second.tenant,
project: second.project,
},
example: "hello-build",
},
expected:
"same-name Nodes coexist and deterministic hello-build artifacts share an ArtifactId without metadata, link, or download interference",
observed: {
first_process: firstCollisionHelloBuild.process,
second_process: secondHelloBuild.process,
same_artifact_id: secondHelloBuild.artifact,
first_digest: firstCollisionHelloBuild.digest,
second_digest: secondHelloBuild.digest,
first_downloaded_bytes: firstCollisionHelloBuild.downloaded_bytes,
second_downloaded_bytes: secondHelloBuild.downloaded_bytes,
exact_executable_output: secondHelloBuild.executable_output,
both_metadata_records_visible_to_owner: true,
cross_tenant_unique_artifact_denied: artifact.denied,
second_link_revoked: true,
first_download_survived_second_link_revocation: true,
},
duration_ms: Math.max(1, Date.now() - collisionStartedAt),
},
duration_ms: Math.max(1, Date.now() - startedAt),
},
runtime: {
node: firstNode,
tenant: second.tenant,
project: second.project,
projectDir: secondProjectDir,
scope: secondScope,
workerArgs: secondWorkerArgs,
credentialPath: secondStored.absolute,
credentialDigest: secondCredentialDigest,
identity: secondIdentity,
},
};
}
async function runLiveSignedHostileArtifactPath({
clusterflux,
projectDir,
scope,
tenant,
project,
suffix,
securityNode,
}) {
const startedAt = Date.now();
const runReport = runJson(
clusterflux,
["run", "park-wake", "--project", ".", "--json"],
{ cwd: projectDir }
);
assert.strictEqual(runReport.status, "main_launched");
const process = runReport.process;
await waitForCli(
"hostile-path probe process to become active",
() =>
runJson(
clusterflux,
["process", "status", ...scope, "--process", process],
{ cwd: projectDir }
),
(status) => status.state !== "not_active",
120_000
);
const invalidStartedAt = Date.now();
const invalid = assertDenied(
await sendHostedControl(
signedNodeRequest(
securityNode.node,
securityNode.identity,
"report_vfs_metadata",
{
type: "report_vfs_metadata",
tenant,
project,
process,
node: securityNode.node,
task: `hostile-path-${suffix}`,
artifact_path: "/vfs/artifacts/bad artifact!",
artifact_digest: sha256(Buffer.from("hostile-path-invalid")),
artifact_size_bytes: 20,
large_bytes_uploaded: false,
},
{ nonce: `strict-hostile-path-invalid-${suffix}-${Date.now()}` }
)
),
/invalid VFS artifact path|invalid artifact path|ArtifactId is invalid/i,
"correctly signed hostile artifact path"
);
const invalidDurationMs = Math.max(1, Date.now() - invalidStartedAt);
const validStartedAt = Date.now();
const validArtifact = `strict-hostile-followup-${suffix}`;
const valid = await sendHostedControl(
signedNodeRequest(
securityNode.node,
securityNode.identity,
"report_vfs_metadata",
{
type: "report_vfs_metadata",
tenant,
project,
process,
node: securityNode.node,
task: `hostile-path-${suffix}`,
artifact_path: `/vfs/artifacts/${validArtifact}`,
artifact_digest: sha256(Buffer.from("hostile-path-valid")),
artifact_size_bytes: 18,
large_bytes_uploaded: false,
},
{ nonce: `strict-hostile-path-valid-${suffix}-${Date.now()}` }
)
);
assert.strictEqual(valid.type, "vfs_metadata_recorded", JSON.stringify(valid));
const healthyHeartbeat = await sendHostedControl({
type: "node_heartbeat",
tenant,
project,
node: securityNode.node,
node_signature: signedNodeHeartbeat(
tenant,
project,
securityNode.node,
securityNode.identity,
{ nonce: `strict-hostile-path-health-${suffix}-${Date.now()}` }
),
});
assert.strictEqual(
healthyHeartbeat.type,
"node_heartbeat",
JSON.stringify(healthyHeartbeat)
);
const validDurationMs = Math.max(1, Date.now() - validStartedAt);
const aborted = runJson(
clusterflux,
["process", "abort", ...scope, "--process", process, "--yes"],
{ cwd: projectDir }
);
assert.strictEqual(aborted.abort_request.accepted, true);
await waitForCli(
"hostile-path probe process cleanup",
() =>
runJson(
clusterflux,
["process", "status", ...scope, "--process", process],
{ cwd: projectDir }
),
(status) => status.state === "not_active",
120_000
);
return {
request: {
principal: "enrolled signed Node",
variant: "report_vfs_metadata",
process,
malformed_path: "/vfs/artifacts/bad artifact!",
},
expected:
"structured InvalidArtifactPath rejection followed immediately by valid signed metadata and heartbeat on the same service instance",
observed: {
rejection: invalid,
valid_metadata_response: valid.type,
valid_artifact: validArtifact,
health_response: healthyHeartbeat.type,
process_cleanup: "not_active",
},
invalid_duration_ms: invalidDurationMs,
valid_followup_duration_ms: validDurationMs,
duration_ms: Math.max(1, Date.now() - startedAt),
};
}
@ -2960,8 +3418,12 @@ async function revokeSecurityNode({
const denied = assertDenied(
await sendHostedControl({
type: "node_heartbeat",
tenant: securityNode.capability_body.tenant,
project: securityNode.capability_body.project,
node: securityNode.node,
node_signature: signedNodeHeartbeat(
securityNode.capability_body.tenant,
securityNode.capability_body.project,
securityNode.node,
securityNode.identity,
{ nonce: `strict-node-revoked-${Date.now()}` }
@ -3742,6 +4204,7 @@ async function restartHostedServiceAndResume({
workerNode,
credentialPath,
credentialDigest,
scopedCollisionRuntime,
}) {
if (!strictVpsRestart) {
return {
@ -3841,6 +4304,79 @@ async function restartHostedServiceAndResume({
assert.strictEqual(ready.node, workerNode);
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
let scopedNodeCollisionAfterRestart = null;
if (scopedCollisionRuntime) {
assert.strictEqual(scopedCollisionRuntime.node, workerNode);
assert.notStrictEqual(
scopedCollisionRuntime.tenant,
readJson(path.join(projectDir, ".clusterflux", "session.json")).tenant
);
assert.strictEqual(
sha256(fs.readFileSync(scopedCollisionRuntime.credentialPath)),
scopedCollisionRuntime.credentialDigest
);
const secondWorker = spawnJsonLines(
clusterfluxNode,
scopedCollisionRuntime.workerArgs,
{
cwd: scopedCollisionRuntime.projectDir,
env: { ...process.env },
}
);
try {
const secondReady = await secondWorker.waitFor(
(value) => value.node_status === "ready",
"same-name second-tenant worker ready after hosted service restart"
);
assert.strictEqual(secondReady.node, workerNode);
const firstStatus = await waitForCli(
"first scoped same-name Node online after restart",
() =>
runJson(
clusterflux,
["node", "status", ...scope, "--node", workerNode],
{ cwd: projectDir }
),
(status) => nodeDescriptor(status, workerNode)?.online === true,
30_000
);
const secondStatus = await waitForCli(
"second scoped same-name Node online after restart",
() =>
runJson(
clusterflux,
[
"node",
"status",
...scopedCollisionRuntime.scope,
"--node",
workerNode,
],
{ cwd: scopedCollisionRuntime.projectDir }
),
(status) => nodeDescriptor(status, workerNode)?.online === true,
30_000
);
scopedNodeCollisionAfterRestart = {
node: workerNode,
first_scope_online:
nodeDescriptor(firstStatus, workerNode)?.online === true,
second_scope_online:
nodeDescriptor(secondStatus, workerNode)?.online === true,
both_credentials_reused_without_grants: true,
distinct_credential_digests:
credentialDigest !== scopedCollisionRuntime.credentialDigest,
};
assert.strictEqual(
scopedNodeCollisionAfterRestart.distinct_credential_digests,
true,
"same-name scoped Nodes unexpectedly reused one credential"
);
} finally {
await stopChild(secondWorker.child);
}
}
const restartedRun = runJson(
clusterflux,
["run", "build", "--project", ".", "--json"],
@ -3873,6 +4409,7 @@ async function restartHostedServiceAndResume({
terminated_ephemeral_process: ephemeralProcess,
new_process: restartedProcess,
new_process_result: completedEvent.result,
scoped_node_collision_after_restart: scopedNodeCollisionAfterRestart,
before,
after_first_restart: afterFirstRestart,
after,
@ -3880,6 +4417,79 @@ async function restartHostedServiceAndResume({
};
}
async function finishScopedNodeCollisionAfterRestart({
clusterflux,
projectDir,
scope,
workerNode,
scopedCollisionRuntime,
}) {
const startedAt = Date.now();
assert(scopedCollisionRuntime, "scoped collision runtime evidence is required");
const revoked = runJson(
clusterflux,
[
"node",
"revoke",
...scopedCollisionRuntime.scope,
"--node",
scopedCollisionRuntime.node,
"--yes",
],
{ cwd: scopedCollisionRuntime.projectDir }
);
assert.strictEqual(revoked.command, "node revoke");
const revokedHeartbeat = assertDenied(
await sendHostedControl({
type: "node_heartbeat",
tenant: scopedCollisionRuntime.tenant,
project: scopedCollisionRuntime.project,
node: scopedCollisionRuntime.node,
node_signature: signedNodeHeartbeat(
scopedCollisionRuntime.tenant,
scopedCollisionRuntime.project,
scopedCollisionRuntime.node,
scopedCollisionRuntime.identity,
{
nonce: `strict-scoped-node-revoked-${Date.now()}`,
}
),
}),
/not enrolled|unknown node|revoked|credential/i,
"revoked second scoped Node"
);
const firstStatus = await waitForCli(
"first same-name Node remains live after scoped revocation",
() =>
runJson(
clusterflux,
["node", "status", ...scope, "--node", workerNode],
{ cwd: projectDir }
),
(status) => nodeDescriptor(status, workerNode)?.online === true,
30_000
);
return {
request: {
revoked_scope: {
tenant: scopedCollisionRuntime.tenant,
project: scopedCollisionRuntime.project,
node: scopedCollisionRuntime.node,
},
retained_node: workerNode,
},
expected:
"revocation affects only the selected scoped Node after both same-name credentials survive restart",
observed: {
revoked_command: revoked.command,
revoked_credential_denied: revokedHeartbeat.denied,
first_scope_node_online:
nodeDescriptor(firstStatus, workerNode)?.online === true,
},
duration_ms: Math.max(1, Date.now() - startedAt),
};
}
async function finishUserCredentialSecurity({
clusterflux,
projectDir,
@ -5713,7 +6323,14 @@ async function main() {
scope,
});
const tenantIsolation = await runSecondTenantIsolation({
await stopChild(worker.child);
const tenantIsolationResult = await runSecondTenantIsolation({
clusterflux,
clusterfluxNode,
firstScope: scope,
firstHelloBuild: helloBuild,
firstHelloProjectDir: helloProjectDir,
workRoot,
firstSessionSecret: sessionSecret,
firstTenant: tenant,
firstProject: project,
@ -5722,6 +6339,14 @@ async function main() {
firstArtifact: releaseArtifact.artifact,
manifest,
});
const tenantIsolation = tenantIsolationResult.evidence;
const scopedCollisionRuntime = tenantIsolationResult.runtime;
worker = spawnWorker();
const collisionRecoveryReady = await worker.waitFor(
(value) => value.node_status === "ready",
"primary worker restored after two-tenant collision proof"
);
assert.strictEqual(collisionRecoveryReady.node, workerNode);
const securityNode = await prepareLiveNodeCredentialSecurity({
clusterflux,
@ -5824,6 +6449,15 @@ async function main() {
},
releaseArtifact,
});
const signedHostileArtifactPath = await runLiveSignedHostileArtifactPath({
clusterflux,
projectDir,
scope,
tenant,
project,
suffix,
securityNode,
});
const revokedNodeCredential = await revokeSecurityNode({
clusterflux,
projectDir,
@ -5844,8 +6478,22 @@ async function main() {
workerNode,
credentialPath,
credentialDigest,
scopedCollisionRuntime,
});
if (serviceRestart.worker) worker = serviceRestart.worker;
const scopedNodeRevocationAfterRestart = scopedCollisionRuntime
? await finishScopedNodeCollisionAfterRestart({
clusterflux,
projectDir,
scope,
workerNode,
scopedCollisionRuntime,
})
: {
executed: false,
reason: "second tenant runtime session was not supplied",
duration_ms: 1,
};
const relayAfterRestart = relayDurableState();
assert(
relayAfterRestart.ingress_used >=
@ -6298,6 +6946,49 @@ async function main() {
evidence: processCancellationLifecycle,
durationMs: processCancellationLifecycle.duration_ms,
}),
requirement({
id: "30_scoped_node_and_artifact_collision",
passed:
tenantIsolation.scoped_node_and_artifact_collision?.observed
?.both_metadata_records_visible_to_owner === true &&
tenantIsolation.scoped_node_and_artifact_collision?.observed
?.first_download_survived_second_link_revocation === true &&
tenantIsolation.scoped_node_and_artifact_collision?.observed
?.same_artifact_id === helloBuild.artifact &&
serviceRestart.scoped_node_collision_after_restart
?.first_scope_online === true &&
serviceRestart.scoped_node_collision_after_restart
?.second_scope_online === true &&
serviceRestart.scoped_node_collision_after_restart
?.both_credentials_reused_without_grants === true &&
scopedNodeRevocationAfterRestart.observed
?.revoked_credential_denied === true &&
scopedNodeRevocationAfterRestart.observed
?.first_scope_node_online === true,
evidence: {
live_collision:
tenantIsolation.scoped_node_and_artifact_collision,
restart:
serviceRestart.scoped_node_collision_after_restart,
scoped_revocation: scopedNodeRevocationAfterRestart,
},
durationMs:
tenantIsolation.scoped_node_and_artifact_collision?.duration_ms +
scopedNodeRevocationAfterRestart.duration_ms,
}),
requirement({
id: "31_signed_hostile_artifact_path_preserves_service",
passed:
signedHostileArtifactPath.observed.rejection.denied === true &&
signedHostileArtifactPath.observed.valid_metadata_response ===
"vfs_metadata_recorded" &&
signedHostileArtifactPath.observed.health_response ===
"node_heartbeat" &&
signedHostileArtifactPath.observed.process_cleanup ===
"not_active",
evidence: signedHostileArtifactPath,
durationMs: signedHostileArtifactPath.duration_ms,
}),
];
const fullReleasePassed =
strictFullRelease &&
@ -6371,6 +7062,8 @@ async function main() {
dap_process_aborted: true,
process_cancellation_lifecycle: processCancellationLifecycle,
tenant_isolation: tenantIsolation,
scoped_node_revocation_after_restart:
scopedNodeRevocationAfterRestart,
credential_security: {
user: userCredentialSecurity,
node: {
@ -6409,6 +7102,7 @@ async function main() {
hello_build: helloBuild,
recovery_build: recoveryBuild,
malformed_identifiers: malformedIdentifiers,
signed_hostile_artifact_path: signedHostileArtifactPath,
named_scenarios: namedScenarios,
scenario_skips: [],
strict_requirement_ledger: strictRequirementLedger,

5
scripts/hostile-input-contract-smoke.js
View file

@ -42,6 +42,8 @@ const signingInstrumentNode = nodeIdentity(
);
assert.strictEqual(
signedNodeHeartbeat(
"tenant",
"project",
"node-hostile-input-contract",
signingInstrumentNode,
{ nonce: "" }
@ -127,8 +129,7 @@ for (const [name, source, patterns] of [
/const guessed = await send/,
/const crossActorOpen = await send/,
/token is invalid/,
/tenant mismatch/,
/project mismatch/,
/artifact does not exist/,
],
],
[

13
scripts/node-attach-smoke.js
View file

@ -94,12 +94,12 @@ function nodeSignatureMessage(
);
}
function signedNodeHeartbeat(node, identity) {
function signedNodeHeartbeat(tenant, project, node, identity) {
const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`;
const issuedAt = Math.floor(Date.now() / 1000);
const payloadDigest = `sha256:${crypto
.createHash("sha256")
.update(JSON.stringify({ node, type: "node_heartbeat" }))
.update(JSON.stringify({ node, project, tenant, type: "node_heartbeat" }))
.digest("hex")}`;
const signature = crypto.sign(
null,
@ -283,8 +283,15 @@ function runAttach(addr, grant) {
const heartbeat = await send(addr, {
type: "node_heartbeat",
tenant: "tenant",
project: "project",
node: "node-attach",
node_signature: signedNodeHeartbeat("node-attach", nodeIdentity("node-attach")),
node_signature: signedNodeHeartbeat(
"tenant",
"project",
"node-attach",
nodeIdentity("node-attach")
),
});
assert.strictEqual(heartbeat.type, "node_heartbeat");
assert.strictEqual(heartbeat.node, "node-attach");

2
scripts/node-lifecycle-contract-smoke.js
View file

@ -97,7 +97,7 @@ for (const [name, pattern] of [
for (const [name, pattern] of [
["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/],
["reconnect preserves enrolled node identity", /reconnect_node\(&NodeId::from\("node"\), None\)/],
["reconnect preserves scoped enrolled node identity", /reconnect_node\([\s\S]*&TenantId::from\("tenant"\),[\s\S]*&ProjectId::from\("project"\),[\s\S]*&NodeId::from\("node"\),[\s\S]*None/],
["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/],
]) {
expect(coordinatorCore, name, pattern);

4
scripts/node-signing.js
View file

@ -151,8 +151,8 @@ function signedNodeProof(node, identity, requestKind, request, options = {}) {
};
}
function signedNodeHeartbeat(node, identity, options = {}) {
const request = { type: "node_heartbeat", node };
function signedNodeHeartbeat(tenant, project, node, identity, options = {}) {
const request = { type: "node_heartbeat", tenant, project, node };
return signedNodeProof(node, identity, "node_heartbeat", request, options);
}

217
scripts/prepare-manual-github-release.js
View file

@ -1,217 +0,0 @@
#!/usr/bin/env node
"use strict";
const assert = require("node:assert");
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const { spawnSync } = require("node:child_process");
const repo = path.resolve(__dirname, "..");
const releaseRoot = path.resolve(
process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR ||
path.join(repo, "target/public-release-final")
);
const manifestPath = path.join(releaseRoot, "public-release-manifest.json");
const resultPath = path.resolve(
process.env.CLUSTERFLUX_FINAL_RESULT ||
path.join(repo, "target/acceptance/cli-happy-path-live.json")
);
const transcriptPath = path.resolve(
process.env.CLUSTERFLUX_VALIDATION_TRANSCRIPT ||
path.join(repo, "target/acceptance/clusterflux-manual-launch-transcript.log")
);
function sha256(file) {
const hash = crypto.createHash("sha256");
hash.update(fs.readFileSync(file));
return hash.digest("hex");
}
function readJson(file) {
return JSON.parse(fs.readFileSync(file, "utf8"));
}
function write(file, contents) {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, contents);
}
function copyVerifiedAsset(asset, destinationRoot) {
const source = path.resolve(releaseRoot, asset.file);
assert(fs.existsSync(source), `missing finalized asset ${source}`);
assert.strictEqual(
sha256(source),
asset.sha256,
`finalized asset digest changed: ${asset.name}`
);
const destination = path.join(destinationRoot, "assets", asset.name);
fs.copyFileSync(source, destination);
assert.strictEqual(sha256(destination), asset.sha256);
return destination;
}
function extractBinaries(archive, destinationRoot, expectedDigests) {
const staging = path.join(destinationRoot, ".binary-extract");
fs.mkdirSync(staging, { recursive: true });
const extracted = spawnSync("tar", ["-xzf", archive, "-C", staging], {
encoding: "utf8",
});
if (extracted.status !== 0) {
throw new Error(`failed to extract ${archive}: ${extracted.stderr}`);
}
const binRoot = path.join(staging, "bin");
assert(fs.existsSync(binRoot), `binary archive omitted bin/: ${archive}`);
const copied = [];
for (const name of fs.readdirSync(binRoot).sort()) {
const source = path.join(binRoot, name);
if (!fs.statSync(source).isFile()) continue;
const destination = path.join(destinationRoot, "binaries", name);
fs.copyFileSync(source, destination);
fs.chmodSync(destination, 0o755);
assert.strictEqual(
`sha256:${sha256(destination)}`,
expectedDigests[name],
`tested binary digest changed: ${name}`
);
copied.push(destination);
}
fs.rmSync(staging, { recursive: true, force: true });
assert(copied.length > 0, `binary archive contained no binaries: ${archive}`);
return copied;
}
function main() {
assert(fs.existsSync(manifestPath), `missing final manifest ${manifestPath}`);
assert(fs.existsSync(resultPath), `missing final result ${resultPath}`);
assert(fs.existsSync(transcriptPath), `missing validation transcript ${transcriptPath}`);
const manifest = readJson(manifestPath);
const result = readJson(resultPath);
assert.strictEqual(result.acceptance_result, "passed");
assert.strictEqual(result.source_commit, manifest.source_commit);
assert.strictEqual(result.source_tree_digest, manifest.source_tree_digest);
assert.deepStrictEqual(result.binary_digests, manifest.binary_digests);
assert.strictEqual(result.scenario_skips.length, 0);
assert(result.strict_requirement_ledger.length >= 29);
assert(
result.strict_requirement_ledger.every(
(requirement) =>
requirement.passed === true &&
Number.isFinite(requirement.duration_ms) &&
requirement.duration_ms > 0 &&
requirement.evidence
),
"final validation contains a failed launch requirement"
);
assert.strictEqual(
result.named_scenarios.length,
result.strict_requirement_ledger.length
);
const destinationRoot = path.resolve(
process.env.CLUSTERFLUX_GITHUB_RELEASE_DIR ||
path.join(repo, "target/github-release", manifest.release_name)
);
fs.rmSync(destinationRoot, { recursive: true, force: true });
fs.mkdirSync(path.join(destinationRoot, "assets"), { recursive: true });
fs.mkdirSync(path.join(destinationRoot, "binaries"), { recursive: true });
const copiedAssets = manifest.assets.map((asset) =>
copyVerifiedAsset(asset, destinationRoot)
);
const publicationGuidance = [
JSON.stringify(manifest.notes || []),
...copiedAssets
.filter((file) => file.endsWith(".md"))
.map((file) => fs.readFileSync(file, "utf8")),
].join("\n");
assert.doesNotMatch(
publicationGuidance,
/(?:download|upload|release downloads?)[^\n]*Forgejo/i,
"manual GitHub package contains Forgejo release-publication instructions"
);
const binaryArchive = copiedAssets.find((file) =>
path.basename(file).startsWith("clusterflux-public-binaries-")
);
assert(binaryArchive, "final manifest omitted the public binary archive");
const binaries = extractBinaries(
binaryArchive,
destinationRoot,
manifest.binary_digests
);
const vsix = copiedAssets.find((file) => file.endsWith(".vsix"));
assert(vsix, "final manifest omitted the tested VSIX");
assert.strictEqual(
sha256(vsix),
manifest.release_candidate.extension_sha256,
"tested VSIX digest does not match final candidate binding"
);
write(path.join(destinationRoot, "SOURCE_REVISION"), `${manifest.source_commit}\n`);
write(
path.join(destinationRoot, "SOURCE_TREE_DIGEST"),
`${manifest.source_tree_digest}\n`
);
write(path.join(destinationRoot, "VSIX_SHA256"), `${sha256(vsix)} ${path.basename(vsix)}\n`);
fs.copyFileSync(manifestPath, path.join(destinationRoot, "public-release-manifest.json"));
fs.copyFileSync(transcriptPath, path.join(destinationRoot, "VALIDATION_TRANSCRIPT.log"));
write(
path.join(destinationRoot, "RELEASE_NOTES.md"),
`# Clusterflux ${manifest.release_name}\n\n` +
"First manual GitHub launch release of Clusterflux. The release includes the filtered public source, real Linux CLI/node/coordinator binaries, and the exact VSIX used by the final production-shaped validation batch.\n"
);
write(
path.join(destinationRoot, "KNOWN_LIMITATIONS.md"),
"# Known limitations\n\n" +
"- The launch batch validates one enrolled Linux node with rootless Podman.\n" +
"- Full Windows sandbox validation is deferred.\n" +
"- Providers, billing, teams, managed compute, and operator-panel UI are not part of this release.\n"
);
const testedUploadAssets = [
...copiedAssets,
...binaries,
]
.sort()
.map((file) => ({
file: path.relative(destinationRoot, file),
sha256: `sha256:${sha256(file)}`,
}));
write(
path.join(destinationRoot, "final-result.json"),
`${JSON.stringify(
{
...result,
manual_release: {
source_revision: manifest.source_commit,
source_tree_digest: manifest.source_tree_digest,
tested_upload_assets: testedUploadAssets,
publication: "manual GitHub upload from this directory",
},
},
null,
2
)}\n`
);
const checksummed = [
...copiedAssets,
...binaries,
path.join(destinationRoot, "SOURCE_REVISION"),
path.join(destinationRoot, "SOURCE_TREE_DIGEST"),
path.join(destinationRoot, "VSIX_SHA256"),
path.join(destinationRoot, "public-release-manifest.json"),
path.join(destinationRoot, "final-result.json"),
path.join(destinationRoot, "VALIDATION_TRANSCRIPT.log"),
path.join(destinationRoot, "RELEASE_NOTES.md"),
path.join(destinationRoot, "KNOWN_LIMITATIONS.md"),
].sort();
write(
path.join(destinationRoot, "SHA256SUMS"),
`${checksummed
.map((file) => `${sha256(file)} ${path.relative(destinationRoot, file)}`)
.join("\n")}\n`
);
console.log(destinationRoot);
}
main();

2
scripts/release-quality-gates.js
View file

@ -169,7 +169,7 @@ function main() {
"test",
"-p",
"clusterflux-coordinator",
"completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot",
"completed_main_",
],
},
{

5
scripts/scheduler-placement-smoke.js
View file

@ -243,7 +243,10 @@ async function reportNode(addr, node, identity, locality) {
online: true
}));
assert.strictEqual(crossTenantReport.type, "error");
assert.match(crossTenantReport.message, /tenant\/project scope/);
assert.match(
crossTenantReport.message,
/tenant\/project scope|node identity is not enrolled/
);
const placement = await send(addr, {
type: "schedule_task",

10
scripts/self-hosted-coordinator-smoke.js
View file

@ -190,8 +190,8 @@ function nodeSignatureMessage(
);
}
function signedNodeHeartbeat(node, identity) {
const request = { type: "node_heartbeat", node };
function signedNodeHeartbeat(tenant, project, node, identity) {
const request = { type: "node_heartbeat", tenant, project, node };
const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`;
const issuedAt = Math.floor(Date.now() / 1000);
const signature = crypto.sign(
@ -313,8 +313,10 @@ async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilitie
const heartbeat = await send(addr, {
type: "node_heartbeat",
tenant: "team",
project: "self-hosted",
node,
node_signature: signedNodeHeartbeat(node, identity)
node_signature: signedNodeHeartbeat("team", "self-hosted", node, identity)
});
assert.strictEqual(heartbeat.type, "node_heartbeat");
@ -569,6 +571,8 @@ async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilitie
const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", {
type: "reconnect_node",
tenant: "team",
project: "self-hosted",
node: "team-linux-a",
process: "vp-team-build",
epoch: started.epoch

5
scripts/source-preparation-smoke.js
View file

@ -170,7 +170,10 @@ function sourceCapableNode(sourceProviders = ["git"]) {
source_snapshot: "sha256:source-prepared"
}));
assert.strictEqual(crossTenantCompletion.type, "error");
assert.match(crossTenantCompletion.message, /tenant\/project scope/i);
assert.match(
crossTenantCompletion.message,
/tenant\/project scope|node identity is not enrolled/i
);
const completed = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", {
type: "complete_source_preparation",

26
scripts/tenant-isolation-contract-smoke.js
View file

@ -38,6 +38,7 @@ const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js");
const operatorPanelSmoke = read("scripts/operator-panel-smoke.js");
const schedulerSmoke = read("scripts/scheduler-placement-smoke.js");
const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js");
const liveSmoke = read("scripts/cli-happy-path-live-smoke.js");
for (const [name, pattern] of [
["auth contexts carry tenant and project", /pub struct AuthContext[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/],
@ -79,10 +80,13 @@ for (const [name, pattern] of [
["project creation rejects foreign tenant reuse", /project id is outside the signed-in tenant scope/],
["project selection rejects foreign tenant", /project is outside the signed-in tenant scope/],
["project listing uses tenant-scoped context", /CoordinatorRequest::ListProjects[\s\S]*list_projects\(&context\)/],
["node capability reports check enrollment tenant scope", /node capability report is outside the enrolled tenant\/project scope/],
[
"node capability reports resolve the full enrolled scope",
/NodeScopeKey::from_refs\(&tenant, &project, &node\)[\s\S]*node_identity\(&tenant, &project, &node\)/,
],
["operator panel stop state is tenant/project/process keyed", /type PanelStopKey = \(TenantId, ProjectId, ProcessId\)/],
["download service tests tenant mismatch", /cross_tenant\.to_string\(\)\.contains\("tenant mismatch"\)/],
["download service tests project mismatch", /cross_project\.to_string\(\)\.contains\("project mismatch"\)/],
["download service hides cross-tenant artifact existence", /cross_tenant\.to_string\(\)\.contains\("does not exist"\)/],
["download service hides cross-project artifact existence", /cross_project\.to_string\(\)\.contains\("does not exist"\)/],
["node capability test rejects cross-scope report", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/],
["source preparation test rejects cross-scope completion", /fn service_rejects_source_preparation_completion_outside_node_scope\(\)/],
]) {
@ -98,8 +102,7 @@ for (const [name, sourceText, patterns] of [
/const crossProject = await send/,
/const crossTenantOpen = await send/,
/const crossProjectOpen = await send/,
/tenant mismatch/,
/project mismatch/,
/artifact does not exist/,
/token is invalid/,
],
],
@ -129,6 +132,19 @@ for (const [name, sourceText, patterns] of [
}
}
for (const [name, pattern] of [
[
"live same-ID collision refreshes first-tenant metadata before the second build",
/const firstCollisionHelloBuild = await runHostedHelloBuild[\s\S]*secondHelloBuild = await runHostedHelloBuild/,
],
[
"live same-ID collision re-reads the fresh first-tenant process",
/"--process",[\s\S]*firstCollisionHelloBuild\.process[\s\S]*candidate\.artifact === firstCollisionHelloBuild\.artifact/,
],
]) {
expect(liveSmoke, name, pattern);
}
const hostedLibRoot = maybeRead(["private", "hosted-policy", "src", "lib.rs"]);
const hostedServiceSource = maybeRead([
"private",

2
scripts/windows-best-effort-smoke.js
View file

@ -198,6 +198,8 @@ function assertWindowsBackendBoundary() {
const reconnected = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "reconnect_node", {
type: "reconnect_node",
tenant: "tenant",
project: "project",
node: windowsNode,
process: "vp-windows",
epoch: started.epoch