michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-ea887c8f56cd

Browse source 
Source commit: ea887c8f56cd53985a1179b13e5f1b85c485f584

Public tree identity: sha256:b96a97aacdbc8fd4fc2ea20d0e1450280d46db3f24aabe1475e5fbe20e05f255
This commit is contained in:
Clusterflux release 2026-07-25 02:55:35 +02:00
parent 9223c54939
commit 2a0f7ded04
37 changed files with 3145 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/clusterflux-cli/src/node.rs
View file

@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
};
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &tenant,
"project": &project,
"node": &plan.node,
});
let heartbeat_signature = sign_node_request(

35
crates/clusterflux-coordinator/src/durable.rs
View file

@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
pub enrollment_scope: String,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct NodeScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub node: NodeId,
}
impl NodeScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
Self {
tenant,
project,
node,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
Self::new(tenant.clone(), project.clone(), node.clone())
}
pub fn credential_subject(&self) -> String {
format!(
"node:{}:{}:{}:{}:{}:{}",
self.tenant.as_str().len(),
self.tenant,
self.project.as_str().len(),
self.project,
self.node.as_str().len(),
self.node
)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CredentialRecord {
pub subject: String,
@ -107,7 +140,7 @@ pub struct DurableState {
pub tenants: BTreeMap<TenantId, TenantRecord>,
pub users: BTreeMap<UserId, UserRecord>,
pub projects: BTreeMap<ProjectId, ProjectRecord>,
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
pub credentials: BTreeMap<String, CredentialRecord>,
pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
pub source_provider_configs:

202
crates/clusterflux-coordinator/src/lib.rs
View file

@ -13,7 +13,7 @@ pub mod service;
mod sessions;
pub use durable::{
AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
TenantRecord, UserRecord,
};
@ -128,8 +128,9 @@ impl Coordinator {
public_key: impl Into<String>,
enrollment_scope: impl Into<String>,
) {
let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
self.durable.node_identities.insert(
node.clone(),
key,
NodeIdentityRecord {
id: node,
tenant,
@ -181,10 +182,13 @@ impl Coordinator {
public_key,
credential.scope.clone(),
);
let subject =
NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
.credential_subject();
self.durable.credentials.insert(
format!("node:{node}"),
subject.clone(),
CredentialRecord {
subject: format!("node:{node}"),
subject,
tenant: credential.tenant.clone(),
project: Some(credential.project.clone()),
kind: credential.credential_kind.clone(),
@ -356,15 +360,12 @@ impl Coordinator {
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node identity is outside the requested tenant/project scope".to_owned(),
));
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if !self
.active_processes
@ -379,14 +380,18 @@ impl Coordinator {
pub fn reconnect_node(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
process: Option<(&ProcessId, u64)>,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if let Some((process_id, stale_epoch)) = process {
if stale_epoch != self.coordinator_epoch {
@ -395,11 +400,7 @@ impl Coordinator {
current_epoch: self.coordinator_epoch,
});
}
let key = (
identity.tenant.clone(),
identity.project.clone(),
process_id.clone(),
);
let key = (tenant.clone(), project.clone(), process_id.clone());
if let Some(active) = self.active_processes.get_mut(&key) {
active.connected_nodes.insert(node.clone());
}
@ -416,23 +417,27 @@ impl Coordinator {
let identity = self
.durable
.node_identities
.get(node)
.get(&NodeScopeKey::from_refs(
&context.tenant,
&context.project,
node,
))
.ok_or(CoordinatorError::UnknownNode)?
.clone();
if identity.tenant != context.tenant || identity.project != context.project {
return Err(CoordinatorError::Unauthorized(
"node credential is outside the signed-in tenant/project scope".to_owned(),
));
}
if !matches!(context.actor, Actor::User(_)) {
return Err(CoordinatorError::Unauthorized(
"node credential revocation requires a user identity".to_owned(),
));
}
self.durable.node_identities.remove(node);
self.durable.credentials.remove(&format!("node:{node}"));
for active in self.active_processes.values_mut() {
active.connected_nodes.remove(node);
let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
self.durable.node_identities.remove(&key);
self.durable.credentials.remove(&key.credential_subject());
for active in self
.active_processes
.values_mut()
.filter(|active| active.tenant == context.tenant && active.project == context.project)
{
active.connected_nodes.remove(&key.node);
}
Ok(identity)
}
@ -580,8 +585,15 @@ impl Coordinator {
.count()
}
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
self.durable.node_identities.get(id)
pub fn node_identity(
&self,
tenant: &TenantId,
project: &ProjectId,
id: &NodeId,
) -> Option<&NodeIdentityRecord> {
self.durable
.node_identities
.get(&NodeScopeKey::from_refs(tenant, project, id))
}
pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@ -683,12 +695,24 @@ mod tests {
.contains_key(&TenantId::from("tenant")));
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
assert_eq!(
restarted
.durable
.credentials
.get("node:node")
.get(&node_subject)
.map(|credential| &credential.kind),
Some(&CredentialKind::NodeCredential)
);
@ -712,7 +736,12 @@ mod tests {
);
assert_eq!(rerun.coordinator_epoch, 2);
restarted
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&process, 2)),
)
.unwrap();
assert!(restarted
.active_process(
@ -725,6 +754,68 @@ mod tests {
.contains(&NodeId::from("node")));
}
#[test]
fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
let store = InMemoryDurableStore::default();
let mut coordinator = Coordinator::boot(&store, 1);
let node = NodeId::from("shared-node");
let scopes = [
(TenantId::from("tenant-a"), ProjectId::from("project-a")),
(TenantId::from("tenant-b"), ProjectId::from("project-b")),
(TenantId::from("tenant-a"), ProjectId::from("project-c")),
];
for (index, (tenant, project)) in scopes.iter().enumerate() {
let mut grant = coordinator.create_node_enrollment_grant(
tenant.clone(),
project.clone(),
format!("grant-{index}"),
"node:attach",
100,
);
coordinator
.exchange_node_enrollment_grant(
&mut grant,
node.clone(),
&format!("public-key-{index}"),
"node:attach",
99,
)
.unwrap();
}
let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
assert_eq!(
coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
coordinator.durable.node_identities.get(&scope_a)
);
assert_eq!(
coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
coordinator.durable.node_identities.get(&scope_b)
);
assert_eq!(
coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
coordinator.durable.node_identities.get(&scope_c)
);
assert!(coordinator
.durable
.credentials
.contains_key(&scope_a.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_b.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_c.credential_subject()));
assert_eq!(coordinator.durable.node_identities.len(), 3);
assert_eq!(coordinator.durable.credentials.len(), 3);
}
#[test]
fn identical_process_ids_are_isolated_by_tenant_and_project() {
let store = InMemoryDurableStore::default();
@ -773,11 +864,18 @@ mod tests {
let mut restarted = Coordinator::boot(&store, 2);
restarted
.reconnect_node(&NodeId::from("node"), None)
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
None,
)
.unwrap();
let error = restarted
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -809,7 +907,13 @@ mod tests {
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
}
#[test]
@ -823,10 +927,16 @@ mod tests {
"public-key",
"node:attach",
);
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
coordinator.durable.credentials.insert(
"node:node".to_owned(),
node_subject.clone(),
CredentialRecord {
subject: "node:node".to_owned(),
subject: node_subject.clone(),
tenant: TenantId::from("tenant"),
project: Some(ProjectId::from("project")),
kind: CredentialKind::NodeCredential,
@ -840,6 +950,8 @@ mod tests {
);
coordinator
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -855,7 +967,7 @@ mod tests {
&NodeId::from("node"),
)
.unwrap_err();
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
assert!(matches!(foreign, CoordinatorError::UnknownNode));
let revoked = coordinator
.revoke_node_credential(
@ -868,8 +980,14 @@ mod tests {
)
.unwrap();
assert_eq!(revoked.id, NodeId::from("node"));
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
assert!(!coordinator.durable.credentials.contains_key("node:node"));
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_none());
assert!(!coordinator.durable.credentials.contains_key(&node_subject));
assert!(!coordinator
.active_process(
&TenantId::from("tenant"),
@ -1084,7 +1202,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
assert!(matches!(error, CoordinatorError::UnknownNode));
}
#[test]

142
crates/clusterflux-coordinator/src/postgres_store.rs
View file

@ -4,8 +4,8 @@ use thiserror::Error;
use crate::{
AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
state.projects.insert(record.id.clone(), record);
}
for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM clusterflux_node_identities ORDER BY node_id",
"SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
)? {
state.node_identities.insert(record.id.clone(), record);
state.node_identities.insert(
NodeScopeKey::new(
record.tenant.clone(),
record.project.clone(),
record.id.clone(),
),
record,
);
}
for record in self.query_records::<CredentialRecord>(
"SELECT record FROM clusterflux_credentials ORDER BY subject",
@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
);
CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
node_id TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, node_id)
);
DO $clusterflux_node_scope_migration$
DECLARE
current_primary_key TEXT;
current_primary_key_columns TEXT[];
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_node_identities
WHERE record->>'id' IS DISTINCT FROM node_id
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node identity migration refused an internally inconsistent legacy row';
END IF;
SELECT
constraint_record.conname,
array_agg(attribute.attname ORDER BY key_column.ordinality)
INTO current_primary_key, current_primary_key_columns
FROM pg_constraint AS constraint_record
CROSS JOIN LATERAL unnest(constraint_record.conkey)
WITH ORDINALITY AS key_column(attribute_number, ordinality)
JOIN pg_attribute AS attribute
ON attribute.attrelid = constraint_record.conrelid
AND attribute.attnum = key_column.attribute_number
WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
AND constraint_record.contype = 'p'
GROUP BY constraint_record.conname;
IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
EXECUTE format(
'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
current_primary_key
);
ALTER TABLE clusterflux_node_identities
ADD PRIMARY KEY (tenant_id, project_id, node_id);
ELSIF current_primary_key_columns
IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
THEN
RAISE EXCEPTION
'clusterflux node identity migration found an unexpected primary key shape';
END IF;
END
$clusterflux_node_scope_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
record JSONB NOT NULL
);
DO $clusterflux_node_credential_migration$
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_credentials
WHERE record->>'subject' IS DISTINCT FROM subject
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node credential migration refused an internally inconsistent legacy row';
END IF;
UPDATE clusterflux_credentials AS credential
SET subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
),
record = jsonb_set(
credential.record,
'{subject}',
to_jsonb(format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)),
false
)
FROM clusterflux_node_identities AS identity
WHERE credential.subject = 'node:' || identity.node_id
AND credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id;
IF EXISTS (
SELECT 1
FROM clusterflux_credentials AS credential
WHERE credential.subject LIKE 'node:%'
AND NOT EXISTS (
SELECT 1
FROM clusterflux_node_identities AS identity
WHERE credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id
AND credential.subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)
)
) THEN
RAISE EXCEPTION
'clusterflux node credential migration found an unscoped or orphaned node subject';
END IF;
END
$clusterflux_node_credential_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -529,7 +651,13 @@ mod tests {
let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
assert_eq!(restarted.active_process_count(), 0);
}

19
crates/clusterflux-coordinator/src/service.rs
View file

@ -13,7 +13,7 @@ use clusterflux_core::{
};
use thiserror::Error;
use crate::{Coordinator, CoordinatorError};
use crate::{Coordinator, CoordinatorError, NodeScopeKey};
mod admin;
mod artifacts;
@ -154,8 +154,8 @@ pub enum CoordinatorServiceError {
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
@ -180,7 +180,7 @@ pub struct CoordinatorService {
process_cancellations: BTreeSet<ProcessControlKey>,
process_aborts: BTreeSet<ProcessControlKey>,
agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
node_replay_nonces: BTreeMap<(NodeId, String), u64>,
node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
stopped_panels: BTreeSet<PanelStopKey>,
panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@ -284,15 +284,10 @@ impl CoordinatorService {
{
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node process-control request is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
return Ok(());
}
self.coordinator

51
crates/clusterflux-coordinator/src/service/artifacts.rs
View file

@ -8,7 +8,7 @@ use clusterflux_core::{
};
use sha2::{Digest as _, Sha256};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::relay::RelayFinishReason;
use super::{
@ -52,7 +52,11 @@ impl CoordinatorService {
let action = self
.artifact_registry
.download_action(&context, &artifact, &policy)?;
self.ensure_download_source_connectivity(&action.source)?;
self.ensure_download_source_connectivity(
&context.tenant,
&context.project,
&action.source,
)?;
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
@ -163,7 +167,11 @@ impl CoordinatorService {
},
&mut validation_meter,
)?;
self.ensure_download_source_connectivity(&stream.link.source)?;
self.ensure_download_source_connectivity(
&stream.link.tenant,
&stream.link.project,
&stream.link.source,
)?;
self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@ -302,7 +310,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?;
@ -410,7 +418,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination =
@ -682,15 +690,10 @@ impl CoordinatorService {
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer node is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
Ok(())
}
@ -725,15 +728,12 @@ impl CoordinatorService {
) -> Result<NodeEndpoint, CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact export node is outside the tenant/project scope".to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity"
))
@ -744,7 +744,7 @@ impl CoordinatorService {
)
.into());
}
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export"
@ -769,17 +769,20 @@ impl CoordinatorService {
fn ensure_download_source_connectivity(
&self,
tenant: &TenantId,
project: &ProjectId,
source: &StorageLocation,
) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else {
return Ok(());
};
let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download"
))
})?;
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} is offline for artifact download"

38
crates/clusterflux-coordinator/src/service/debug_requests.rs
View file

@ -226,24 +226,25 @@ impl CoordinatorService {
} else if !completed_event_observed {
"selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
} else if let Some(checkpoint) = checkpoint {
let vfs_available =
checkpoint
.checkpoint
.vfs_manifest
.objects
.iter()
.all(|(path, object)| {
let artifact = super::keys::artifact_id_from_path(path);
self.artifact_registry
.metadata(&artifact)
.is_some_and(|metadata| {
metadata.tenant == tenant
&& metadata.project == project
&& metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
})
});
let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
true,
|available, (path, object)| {
let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?;
Ok::<_, CoordinatorServiceError>(
available
&& self
.artifact_registry
.metadata(&tenant, &project, &artifact)
.is_some_and(|metadata| {
metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
}),
)
},
)?;
if !vfs_available {
"selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
} else {
@ -479,6 +480,7 @@ impl CoordinatorService {
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
self.maybe_retire_terminal_process(&tenant, &project, &process)?;
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,

45
crates/clusterflux-coordinator/src/service/keys.rs
View file

@ -1,6 +1,7 @@
use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
use thiserror::Error;
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
(tenant.clone(), project.clone(), grant.to_owned())
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub(super) enum ArtifactPathError {
#[error("path must start with the exact /vfs/artifacts/ prefix")]
WrongPrefix,
#[error("path must name an artifact after /vfs/artifacts/")]
EmptyArtifact,
#[error("mapped artifact identifier is invalid: {0}")]
InvalidArtifactId(#[from] clusterflux_core::IdParseError),
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
let value = path
.as_str()
.strip_prefix("/vfs/artifacts/")
.unwrap_or(path.as_str())
.replace('/', ":");
ArtifactId::new(value)
.ok_or(ArtifactPathError::WrongPrefix)?;
if value.is_empty() {
return Err(ArtifactPathError::EmptyArtifact);
}
ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn artifact_id_conversion_is_fallible_and_structural() {
assert_eq!(
artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
ArtifactId::from("build:output")
);
for path in [
"/vfs/other/output",
"/vfs/artifacts",
"/vfs/artifacts/bad artifact!",
] {
let path = VfsPath::new(path).unwrap();
assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
}
let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
}
}

153
crates/clusterflux-coordinator/src/service/logs.rs
View file

@ -1,5 +1,5 @@
use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
};
@ -85,7 +85,9 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant,
project,
process: process.clone(),
@ -203,7 +205,9 @@ impl CoordinatorService {
event.placement = self.task_placements.remove(&task_key);
if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant: event.tenant.clone(),
project: event.project.clone(),
process: event.process.clone(),
@ -217,34 +221,16 @@ impl CoordinatorService {
self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key);
let no_active_tasks =
!self
.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &event.tenant
&& task_project == &event.project
&& task_process == &event.process
});
let completed_after_main = matches!(event.terminal_state, TaskTerminalState::Completed)
&& self.task_events.iter().rev().any(|retained| {
retained.tenant == event.tenant
&& retained.project == event.project
&& retained.process == event.process
&& matches!(retained.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(retained.terminal_state, TaskTerminalState::Completed)
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != event.tenant
|| assignment.project != event.project
|| assignment.process != event.process
|| assignment.node != event.node
|| assignment.task != event.task
});
if no_active_tasks {
self.process_aborts.remove(&process_key);
if (self.process_cancellations.remove(&process_key) || completed_after_main)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
}
!assignments.is_empty()
});
if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key(
&event.tenant,
@ -261,6 +247,7 @@ impl CoordinatorService {
if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
Ok(CoordinatorResponse::TaskRecorded {
process: event.process,
task: event.task,
@ -553,6 +540,82 @@ impl CoordinatorService {
awaiting_operator
}
pub(super) fn maybe_retire_terminal_process(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<bool, CoordinatorServiceError> {
let process_key = process_control_key(tenant, project, process);
if self.main_runtime.controls.contains_key(&process_key) {
return Ok(false);
}
let has_runnable_remote_work =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == tenant && task_project == project && task_process == process
})
|| self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
})
|| self.task_assignments.values().any(|assignments| {
assignments.iter().any(|assignment| {
&assignment.tenant == tenant
&& &assignment.project == project
&& &assignment.process == process
})
})
|| self.task_attempts.iter().any(
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
attempt_tenant == tenant
&& attempt_project == project
&& attempt_process == process
&& attempts.iter().rev().any(|attempt| {
attempt.current
&& matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
},
);
if has_runnable_remote_work {
return Ok(false);
}
let main_completed = self.task_events.iter().rev().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& matches!(event.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(event.terminal_state, TaskTerminalState::Completed)
});
let cancellation_completed = self.process_cancellations.contains(&process_key);
if !main_completed && !cancellation_completed {
return Ok(false);
}
self.process_aborts.remove(&process_key);
self.process_cancellations.remove(&process_key);
if self
.coordinator
.active_process(tenant, project, process)
.is_none()
{
return Ok(false);
}
self.coordinator.abort_process(tenant, project, process)?;
self.clear_debug_state_for_process(tenant, project, process);
self.clear_operator_panel_state(tenant, project, process);
Ok(true)
}
fn flush_artifact_metadata(
&mut self,
flush: ArtifactFlush,
@ -560,17 +623,25 @@ impl CoordinatorService {
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let pinned = self
.task_restart_checkpoints
.values()
.flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
.chain(
self.pending_task_launches
.iter()
.flat_map(|pending| pending.task_spec.required_artifacts.iter()),
)
.cloned()
.collect::<std::collections::BTreeSet<ArtifactId>>();
let mut pinned = std::collections::BTreeSet::new();
for checkpoint in self.task_restart_checkpoints.values() {
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&checkpoint.assignment.tenant,
&checkpoint.assignment.project,
artifact,
));
}
}
for pending in &self.pending_task_launches {
for artifact in &pending.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&pending.tenant,
&pending.project,
artifact,
));
}
}
self.artifact_registry
.flush_metadata_bounded(flush, &pinned)
.map(|_| ())

14
crates/clusterflux-coordinator/src/service/main_runtime.rs
View file

@ -971,16 +971,10 @@ impl CoordinatorService {
}
}
let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
let has_active_children =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &scope.tenant
&& task_project == &scope.project
&& task_process == &scope.process
});
self.main_runtime.controls.remove(&process_key);
if main_completed && has_active_children {
if main_completed {
let _ =
self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
return;
}
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
@ -1169,7 +1163,7 @@ mod tests {
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
assert!(service.process_aborts.contains(&process_key));
assert!(!service.process_aborts.contains(&process_key));
}
#[test]

76
crates/clusterflux-coordinator/src/service/nodes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
SourceProviderKind, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{
bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@ -27,9 +27,9 @@ impl CoordinatorService {
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.get(scope)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
@ -41,7 +41,11 @@ impl CoordinatorService {
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
&descriptor.tenant,
&descriptor.project,
&descriptor.id,
));
descriptor
})
.collect()
@ -165,7 +169,11 @@ impl CoordinatorService {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
self.coordinator.ensure_tenant_active(&tenant)?;
if self.coordinator.node_identity(&node).is_none() {
if self
.coordinator
.node_identity(&tenant, &project, &node)
.is_none()
{
self.quota.ensure_node_admission(
&tenant,
self.coordinator.node_identity_count_for_tenant(&tenant),
@ -197,12 +205,21 @@ impl CoordinatorService {
pub(super) fn handle_node_heartbeat(
&mut self,
tenant: String,
project: String,
node: String,
node_signature: Option<NodeSignedRequest>,
payload_digest: &Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
self.authenticate_node_request(
&NodeScopeKey::new(tenant, project, node.clone()),
node_signature,
"node_heartbeat",
payload_digest,
)?;
Ok(CoordinatorResponse::NodeHeartbeat {
node,
epoch: self.coordinator.coordinator_epoch(),
@ -225,16 +242,13 @@ impl CoordinatorService {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node capability report is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
capabilities.validate_public_report()?;
for (kind, count) in [
("cached environments", cached_environment_digests.len()),
@ -274,12 +288,16 @@ impl CoordinatorService {
.into_iter()
.map(ArtifactId::new)
.collect::<BTreeSet<_>>();
self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations);
self.artifact_registry.reconcile_node_retention(
&tenant,
&project,
&node,
&artifact_locations,
);
let online = self.node_is_live(&node);
let online = self.node_is_live(&node_scope);
self.node_descriptors.insert(
node.clone(),
node_scope,
NodeDescriptor {
id: node.clone(),
tenant,
@ -333,9 +351,13 @@ impl CoordinatorService {
actor: Actor::User(actor.clone()),
};
self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
self.node_last_seen_epoch_seconds.remove(&node_scope);
self.node_replay_nonces
.retain(|(retained_scope, _), _| retained_scope != &node_scope);
self.artifact_registry
.garbage_collect_node(&tenant, &project, &node);
let queued_assignments_removed = self
.task_assignments
.remove(&(tenant.clone(), project.clone(), node.clone()))
@ -369,14 +391,14 @@ impl CoordinatorService {
pub(super) fn authenticate_node_request(
&mut self,
node: &NodeId,
scope: &NodeScopeKey,
node_signature: Option<NodeSignedRequest>,
request_kind: &str,
payload_digest: &Digest,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(&scope.tenant, &scope.project, &scope.node)
.ok_or(CoordinatorError::UnknownNode)?;
let signature = node_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
@ -401,7 +423,7 @@ impl CoordinatorService {
)
.into());
}
let replay_key = (node.clone(), signature.nonce.clone());
let replay_key = (scope.clone(), signature.nonce.clone());
self.node_replay_nonces.retain(|_, accepted_at| {
now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
});
@ -413,7 +435,7 @@ impl CoordinatorService {
}
verify_node_request_signature(
&identity.public_key,
node,
&scope.node,
request_kind,
payload_digest,
&signature,
@ -422,7 +444,7 @@ impl CoordinatorService {
if self
.node_replay_nonces
.keys()
.filter(|(retained_node, _)| retained_node == node)
.filter(|(retained_scope, _)| retained_scope == scope)
.count()
>= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
{
@ -436,8 +458,8 @@ impl CoordinatorService {
.insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
.insert(scope.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
descriptor.online = true;
}
Ok(())

3
crates/clusterflux-coordinator/src/service/panels.rs
View file

@ -254,7 +254,8 @@ impl CoordinatorService {
.rev()
.find_map(|event| event.artifact_path.as_ref())
{
let artifact = artifact_id_from_path(path);
let artifact = artifact_id_from_path(path)
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
let context = clusterflux_core::AuthContext {
tenant,
project,

29
crates/clusterflux-coordinator/src/service/process_launch.rs
View file

@ -118,14 +118,14 @@ impl CoordinatorService {
let mut objects = BTreeMap::new();
let mut missing_required_artifact = false;
for artifact in &task_spec.required_artifacts {
let Some(metadata) = self.artifact_registry.metadata(artifact) else {
let Some(metadata) =
self.artifact_registry
.metadata(&assignment.tenant, &assignment.project, artifact)
else {
missing_required_artifact = true;
continue;
};
if metadata.tenant != assignment.tenant
|| metadata.project != assignment.project
|| metadata.retaining_nodes.is_empty()
{
if metadata.retaining_nodes.is_empty() {
missing_required_artifact = true;
continue;
}
@ -483,17 +483,14 @@ impl CoordinatorService {
.validate()
.map_err(CoordinatorServiceError::Protocol)?;
for artifact in &task_spec.required_artifacts {
let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired"
))
})?;
if metadata.tenant != tenant || metadata.project != project {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is outside the task tenant/project scope"
))
.into());
}
let metadata = self
.artifact_registry
.metadata(&tenant, &project, artifact)
.ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired in this tenant/project scope"
))
})?;
if metadata.retaining_nodes.is_empty() {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} has no retaining node"

48
crates/clusterflux-coordinator/src/service/processes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::keys::{process_control_key, task_control_key};
use super::{
@ -47,14 +47,10 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"task assignment poll is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let assignment_key = (tenant.clone(), project.clone(), node.clone());
let assignment = self
.task_assignments
@ -77,7 +73,11 @@ impl CoordinatorService {
project: &ProjectId,
node: &NodeId,
) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
let Some(descriptor) = self
.node_descriptors
.get(&NodeScopeKey::from_refs(tenant, project, node))
.cloned()
else {
return Ok(None);
};
let mut remaining = VecDeque::new();
@ -233,20 +233,18 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"source preparation completion is outside the enrolled tenant/project scope"
.to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let descriptor = self
.node_descriptors
.get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
if !descriptor.source_snapshots.contains(&source_snapshot)
&& descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
{
@ -413,14 +411,18 @@ impl CoordinatorService {
pub(super) fn handle_reconnect_node(
&mut self,
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let process = ProcessId::new(process);
self.coordinator
.reconnect_node(&node, Some((&process, epoch)))?;
.reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
Ok(CoordinatorResponse::NodeReconnected { node, process })
}

20
crates/clusterflux-coordinator/src/service/protocol.rs
View file

@ -121,6 +121,8 @@ pub enum CoordinatorRequest {
enrollment_grant: String,
},
NodeHeartbeat {
tenant: String,
project: String,
node: String,
#[serde(default)]
node_signature: Option<NodeSignedRequest>,
@ -269,6 +271,8 @@ pub enum CoordinatorRequest {
restart: bool,
},
ReconnectNode {
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
@ -675,9 +679,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512)
}
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
if let Some(signature) = node_signature {
validate_node_signature(signature, &format!("{path}.node_signature"))?;
@ -846,7 +853,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
&format!("{path}.launch_attempt"),
)
}
CoordinatorRequest::ReconnectNode { node, process, .. } => {
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
..
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
validate_process(process, &format!("{path}.process"))
}
@ -1567,6 +1581,8 @@ mod external_identifier_tests {
restart: false,
},
CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "bad node!".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: "node-nonce".to_owned(),
@ -1626,6 +1642,8 @@ mod external_identifier_tests {
assert!(error.contains("malformed external token request.agent_signature.nonce"));
let node_request = CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "node".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: String::new(),

10
crates/clusterflux-coordinator/src/service/routing.rs
View file

@ -274,9 +274,17 @@ impl CoordinatorService {
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
} => self.handle_node_heartbeat(
tenant,
project,
node,
node_signature,
&request_payload_digest,
),
CoordinatorRequest::SignedNode {
node,
node_signature,

155
crates/clusterflux-coordinator/src/service/signed_nodes.rs
View file

@ -1,6 +1,6 @@
use clusterflux_core::{NodeId, NodeSignedRequest};
use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
@ -12,7 +12,7 @@ impl CoordinatorService {
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request_kind = signed_node_request_kind(&request)?;
let request_node = signed_node_request_node(&request)?;
let request_scope = signed_node_request_scope(&request)?;
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize signed node request: {error}"
@ -22,14 +22,14 @@ impl CoordinatorService {
let signed_node = NodeId::try_new(signed_node).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
})?;
if request_node != signed_node {
if request_scope.node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
)
.into());
}
self.authenticate_node_request(
&signed_node,
&request_scope,
Some(node_signature),
request_kind,
&payload_digest,
@ -160,10 +160,12 @@ impl CoordinatorService {
failure_reason,
),
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
epoch,
} => self.handle_reconnect_node(node, process, epoch),
} => self.handle_reconnect_node(tenant, project, node, process, epoch),
CoordinatorRequest::PollTaskControl {
tenant,
project,
@ -353,36 +355,129 @@ fn signed_node_request_kind(
}
}
fn signed_node_request_node(
fn signed_node_request_scope(
request: &CoordinatorRequest,
) -> Result<NodeId, CoordinatorServiceError> {
) -> Result<NodeScopeKey, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { node, .. }
| CoordinatorRequest::PollTaskAssignment { node, .. }
| CoordinatorRequest::PollArtifactTransfer { node, .. }
| CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
| CoordinatorRequest::FailArtifactTransfer { node, .. }
| CoordinatorRequest::LaunchChildTask { node, .. }
| CoordinatorRequest::JoinChildTask { node, .. }
| CoordinatorRequest::CompleteSourcePreparation { node, .. }
| CoordinatorRequest::ReconnectNode { node, .. }
| CoordinatorRequest::PollTaskControl { node, .. }
| CoordinatorRequest::PollDebugCommand { node, .. }
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => {
NodeId::try_new(node.clone()).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"invalid wrapped node identifier: {error}"
))
})
CoordinatorRequest::ReportNodeCapabilities {
tenant,
project,
node,
..
}
CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()),
| CoordinatorRequest::PollTaskAssignment {
tenant,
project,
node,
}
| CoordinatorRequest::PollArtifactTransfer {
tenant,
project,
node,
}
| CoordinatorRequest::UploadArtifactTransferChunk {
tenant,
project,
node,
..
}
| CoordinatorRequest::FailArtifactTransfer {
tenant,
project,
node,
..
}
| CoordinatorRequest::LaunchChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::JoinChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::CompleteSourcePreparation {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollTaskControl {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollDebugCommand {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugState {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugProbeHit {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportTaskLog {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
node,
..
}
| CoordinatorRequest::TaskCompleted {
tenant,
project,
node,
..
} => node_scope_from_strings(tenant, project, node),
CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
scope.tenant.clone(),
scope.project.clone(),
source.node.clone(),
)),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}
fn node_scope_from_strings(
tenant: &str,
project: &str,
node: &str,
) -> Result<NodeScopeKey, CoordinatorServiceError> {
let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
})?;
let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
})?;
let node = NodeId::try_new(node.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
})?;
Ok(NodeScopeKey::new(tenant, project, node))
}

1076
crates/clusterflux-coordinator/src/service/tests.rs
View file

File diff suppressed because it is too large Load diff

461
crates/clusterflux-core/src/artifact.rs
View file

@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
pub coordinator_has_large_bytes: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ArtifactScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub artifact: ArtifactId,
}
impl ArtifactScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
Self {
tenant,
project,
artifact,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
Self::new(tenant.clone(), project.clone(), artifact.clone())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactFlush {
pub id: ArtifactId,
@ -148,7 +169,7 @@ pub struct ArtifactFlush {
#[derive(Clone, Debug, Default)]
pub struct ArtifactRegistry {
artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
next_epoch: u64,
}
@ -162,9 +183,10 @@ impl ArtifactRegistry {
pub fn flush_metadata_bounded(
&mut self,
flush: ArtifactFlush,
pinned: &BTreeSet<ArtifactId>,
pinned: &BTreeSet<ArtifactScopeKey>,
) -> Result<ArtifactMetadata, String> {
let replacing_existing = self.artifacts.contains_key(&flush.id);
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
let replacing_existing = self.artifacts.contains_key(&key);
while !replacing_existing
&& self
.artifacts
@ -184,13 +206,25 @@ impl ArtifactRegistry {
metadata.tenant == flush.tenant
&& metadata.project == flush.project
&& metadata.process == flush.process
&& !pinned.contains(&metadata.id)
&& !self.issued_download_links.values().any(|issued| {
issued.link.artifact == metadata.id
})
&& !pinned.contains(&ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
))
&& !self.issued_download_links.values().any(|issued| {
issued.link.tenant == metadata.tenant
&& issued.link.project == metadata.project
&& issued.link.artifact == metadata.id
})
})
.min_by_key(|metadata| metadata.flushed_epoch)
.map(|metadata| metadata.id.clone())
.map(|metadata| {
ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
)
})
.ok_or_else(|| {
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
.to_owned()
@ -212,36 +246,47 @@ impl ArtifactRegistry {
explicit_locations: Vec::new(),
coordinator_has_large_bytes: false,
};
self.artifacts.insert(flush.id, metadata.clone());
self.artifacts.insert(key, metadata.clone());
Ok(metadata)
}
pub fn sync_to_explicit_store(
&mut self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
location: impl Into<String>,
) -> Result<(), ArtifactUnavailable> {
let metadata = self
.artifacts
.get_mut(artifact)
.get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
.ok_or(ArtifactUnavailable)?;
metadata.explicit_locations.push(location.into());
Ok(())
}
pub fn garbage_collect_node(&mut self, node: &NodeId) {
for metadata in self.artifacts.values_mut() {
pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
for metadata in self
.artifacts
.values_mut()
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
{
metadata.retaining_nodes.remove(node);
}
}
pub fn reconcile_node_retention(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
retained_artifacts: &BTreeSet<ArtifactId>,
) {
for (artifact, metadata) in &mut self.artifacts {
if retained_artifacts.contains(artifact) {
for (key, metadata) in &mut self.artifacts {
if &key.tenant != tenant || &key.project != project {
continue;
}
if retained_artifacts.contains(&key.artifact) {
metadata.retaining_nodes.insert(node.clone());
} else {
metadata.retaining_nodes.remove(node);
@ -249,8 +294,14 @@ impl ArtifactRegistry {
}
}
pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
self.artifacts.get(artifact)
pub fn metadata(
&self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
) -> Option<&ArtifactMetadata> {
self.artifacts
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
}
pub fn download_action(
@ -261,7 +312,11 @@ impl ArtifactRegistry {
) -> Result<DownloadAction, DownloadError> {
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let scope = Scope {
tenant: metadata.tenant.clone(),
@ -315,7 +370,11 @@ impl ArtifactRegistry {
self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
Ok(metadata.size)
}
@ -333,7 +392,11 @@ impl ArtifactRegistry {
if self
.issued_download_links
.values()
.filter(|issued| issued.link.artifact == *artifact)
.filter(|issued| {
issued.link.tenant == context.tenant
&& issued.link.project == context.project
&& issued.link.artifact == *artifact
})
.count()
>= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
{
@ -345,7 +408,11 @@ impl ArtifactRegistry {
let action = self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let policy_context_digest =
@ -398,7 +465,11 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact || issued.link.actor != context.actor {
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.actor != context.actor
{
return Err(DownloadError::InvalidToken);
}
self.download_action(
@ -434,7 +505,9 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.max_bytes != policy.max_bytes
|| issued.link.actor != context.actor
{
@ -452,7 +525,11 @@ impl ArtifactRegistry {
}
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
if download_policy_context_digest(metadata, &action.source, policy)
!= issued.link.policy_context_digest
@ -477,7 +554,11 @@ impl ArtifactRegistry {
) -> Result<(), DownloadError> {
let metadata = self
.artifacts
.get(&stream.link.artifact)
.get(&ArtifactScopeKey::from_refs(
&stream.link.tenant,
&stream.link.project,
&stream.link.artifact,
))
.ok_or(DownloadError::NotFound)?;
if !source_is_available(metadata, &stream.link.source) {
return Err(DownloadError::Unavailable);
@ -596,7 +677,13 @@ mod tests {
#[test]
fn flush_publishes_metadata_without_coordinator_bytes() {
let registry = registry_with_artifact();
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(!metadata.coordinator_has_large_bytes);
assert_eq!(metadata.id, ArtifactId::from("artifact"));
@ -615,7 +702,11 @@ mod tests {
#[test]
fn unsynced_node_loss_surfaces_as_unavailable() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -636,9 +727,20 @@ mod tests {
#[test]
fn signed_node_retention_inventory_removes_garbage_collected_location() {
let mut registry = registry_with_artifact();
registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
&BTreeSet::new(),
);
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(metadata.retaining_nodes.is_empty());
}
@ -648,11 +750,26 @@ mod tests {
let mut registry = registry_with_artifact();
let node = NodeId::from("node");
let artifact = ArtifactId::from("artifact");
registry.garbage_collect_node(&node);
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
);
registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()]));
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
&BTreeSet::from([artifact.clone()]),
);
let metadata = registry.metadata(&artifact).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&artifact,
)
.unwrap();
assert!(metadata.retaining_nodes.contains(&node));
}
@ -660,9 +777,18 @@ mod tests {
fn explicit_user_storage_location_survives_node_retention_loss() {
let mut registry = registry_with_artifact();
registry
.sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
.sync_to_explicit_store(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
"s3://bucket/app",
)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -683,7 +809,11 @@ mod tests {
);
assert!(
!registry
.metadata(&ArtifactId::from("artifact"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap()
.coordinator_has_large_bytes
);
@ -714,7 +844,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
@ -734,13 +864,206 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
let artifact = ArtifactId::from("shared-artifact");
let tenant_a = TenantId::from("tenant-a");
let project_a = ProjectId::from("project-a");
let tenant_b = TenantId::from("tenant-b");
let project_b = ProjectId::from("project-b");
let project_c = ProjectId::from("project-c");
let node_a = NodeId::from("node-a");
let node_b = NodeId::from("node-b");
let node_c = NodeId::from("node-c");
let mut registry = ArtifactRegistry::default();
for (tenant, project, process, node, content, size) in [
(&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
(&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
(&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
] {
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::from(process),
producer_task: TaskInstanceId::from("producer"),
retaining_node: node.clone(),
digest: Digest::sha256(content),
size,
});
}
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
assert_eq!(
registry
.metadata(&tenant_a, &project_c, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-c")
);
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant_a.clone(),
project: project_a.clone(),
process: ProcessId::from("process-a"),
producer_task: TaskInstanceId::from("producer-repeat"),
retaining_node: node_a.clone(),
digest: Digest::sha256("bytes-a-repeat"),
size: 18,
});
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a-repeat")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
registry
.sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
.unwrap();
registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
let context_a = AuthContext {
tenant: tenant_a.clone(),
project: project_a.clone(),
actor: Actor::User(UserId::from("user")),
};
let context_b = AuthContext {
tenant: tenant_b.clone(),
project: project_b.clone(),
actor: Actor::User(UserId::from("user")),
};
let policy = DownloadPolicy { max_bytes: 100 };
assert_eq!(
registry
.download_action(&context_a, &artifact, &policy)
.unwrap()
.source,
StorageLocation::ExplicitStore("store://tenant-a".to_owned())
);
assert_eq!(
registry
.download_action(&context_b, &artifact, &policy)
.unwrap()
.source,
StorageLocation::RetainedNode(node_b)
);
let first_a = registry
.create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
let first_b = registry
.create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
assert_ne!(
first_a.scoped_token_digest, first_b.scoped_token_digest,
"the full artifact scope must contribute to download tokens"
);
for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
registry
.create_download_link(
&context_a,
&artifact,
&policy,
&format!("tenant-a-{index}"),
10,
60,
)
.unwrap();
}
assert!(matches!(
registry.create_download_link(
&context_a,
&artifact,
&policy,
"tenant-a-over-limit",
10,
60,
),
Err(DownloadError::Usage(_))
));
registry
.create_download_link(
&context_b,
&artifact,
&policy,
"tenant-b-still-independent",
10,
60,
)
.unwrap();
registry
.revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
.unwrap();
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
};
let mut meter = ResourceMeter::default();
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_b.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.expect("revoking tenant A's link must not revoke tenant B's link");
assert_eq!(
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_a.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.unwrap_err(),
DownloadError::InvalidToken
);
}
#[test]
fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -1019,10 +1342,24 @@ mod tests {
#[test]
fn artifact_metadata_is_bounded_without_evicting_pins() {
let mut registry = ArtifactRegistry::default();
registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("artifact-0"),
tenant: TenantId::from("other-tenant"),
project: ProjectId::from("other-project"),
process: ProcessId::from("process"),
producer_task: TaskInstanceId::from("other-task"),
retaining_node: NodeId::from("other-node"),
digest: Digest::sha256("other-content"),
size: 1,
});
let mut pinned = BTreeSet::new();
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
let id = ArtifactId::new(format!("artifact-{index}"));
pinned.insert(id.clone());
pinned.insert(ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
id.clone(),
));
registry
.flush_metadata_bounded(
ArtifactFlush {
@ -1039,7 +1376,10 @@ mod tests {
)
.unwrap();
}
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
let next = ArtifactFlush {
id: ArtifactId::from("artifact-next"),
tenant: TenantId::from("tenant"),
@ -1055,13 +1395,42 @@ mod tests {
.unwrap_err()
.contains("pinned"));
pinned.remove(&ArtifactId::from("artifact-0"));
pinned.remove(&ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
ArtifactId::from("artifact-0"),
));
registry.flush_metadata_bounded(next, &pinned).unwrap();
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
assert!(registry
.metadata(&ArtifactId::from("artifact-next"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-0"),
)
.is_none());
assert!(registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-next"),
)
.is_some());
assert_eq!(
registry
.metadata(
&TenantId::from("other-tenant"),
&ProjectId::from("other-project"),
&ArtifactId::from("artifact-0"),
)
.unwrap()
.digest,
Digest::sha256("other-content"),
"eviction and pins in one scope must not affect an equal ID in another scope"
);
}
#[test]
@ -1150,7 +1519,11 @@ mod tests {
registry
.stream_download_chunk(&mut stream, &limits, &mut meter, 16)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
assert_eq!(
registry

4
crates/clusterflux-core/src/lib.rs
View file

@ -20,8 +20,8 @@ pub mod wire;
pub use artifact::{
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
DownloadStreamRequest, RetentionPolicy, StorageLocation,
ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
};
pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest,

91
crates/clusterflux-core/src/vfs.rs
View file

@ -1,18 +1,48 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
use thiserror::Error;
use crate::{Digest, NodeId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub const MAX_VFS_PATH_BYTES: usize = 4096;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct VfsPath(String);
impl VfsPath {
pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
let path = path.into();
if !path.starts_with("/vfs/") {
return Err(VfsError::InvalidPath(path));
return Err(VfsError::invalid(path, "path must start with /vfs/"));
}
if path.len() > MAX_VFS_PATH_BYTES {
return Err(VfsError::invalid(
path,
"path exceeds the 4096-byte protocol limit",
));
}
if path.chars().any(char::is_control) {
return Err(VfsError::invalid(path, "control characters are forbidden"));
}
if path.contains('\\') {
return Err(VfsError::invalid(path, "backslashes are forbidden"));
}
let relative = &path["/vfs/".len()..];
if relative.is_empty() {
return Err(VfsError::invalid(
path,
"path after /vfs/ must not be empty",
));
}
if relative
.split('/')
.any(|component| component.is_empty() || matches!(component, "." | ".."))
{
return Err(VfsError::invalid(
path,
"empty, '.', and '..' path components are forbidden",
));
}
Ok(Self(path))
}
@ -22,6 +52,16 @@ impl VfsPath {
}
}
impl<'de> Deserialize<'de> for VfsPath {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let path = String::deserialize(deserializer)?;
Self::new(path).map_err(D::Error::custom)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsObject {
pub path: VfsPath,
@ -63,12 +103,18 @@ pub enum ReuseDecision {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum VfsError {
#[error("VFS path must start with /vfs/: {0}")]
InvalidPath(String),
#[error("invalid VFS path {path:?}: {reason}")]
InvalidPath { path: String, reason: &'static str },
#[error("path is not visible in the published VFS manifest: {0}")]
NotVisible(String),
}
impl VfsError {
fn invalid(path: String, reason: &'static str) -> Self {
Self::InvalidPath { path, reason }
}
}
#[derive(Clone, Debug)]
pub struct VfsOverlay {
task: TaskInstanceId,
@ -238,4 +284,39 @@ mod tests {
assert_eq!(overlay.pending_len(), 0);
}
#[test]
fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
for invalid in [
"vfs/artifacts/app".to_owned(),
"/vfs/".to_owned(),
"/vfs/artifacts//app".to_owned(),
"/vfs/artifacts/app/".to_owned(),
"/vfs/artifacts/./app".to_owned(),
"/vfs/artifacts/../app".to_owned(),
"/vfs/artifacts\\app".to_owned(),
"/vfs/artifacts/bad\0app".to_owned(),
format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
] {
assert!(
VfsPath::new(&invalid).is_err(),
"hostile VFS path unexpectedly passed: {invalid:?}"
);
assert!(
serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
"hostile VFS path unexpectedly deserialized"
);
}
}
#[test]
fn vfs_path_accepts_the_4096_byte_boundary() {
let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
let overlong = format!("{path}x");
assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
assert!(VfsPath::new(overlong).is_err());
}
}

4
crates/clusterflux-node/src/daemon.rs
View file

@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
});
let heartbeat_signature = sign_node_request(
@ -420,6 +422,8 @@ fn run_runtime_task(
"reconnect_node",
json!({
"type": "reconnect_node",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"process": &task.process,
"epoch": epoch,