michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-ea887c8f56cd

Browse source 
Source commit: ea887c8f56cd53985a1179b13e5f1b85c485f584

Public tree identity: sha256:b96a97aacdbc8fd4fc2ea20d0e1450280d46db3f24aabe1475e5fbe20e05f255
This commit is contained in:
Clusterflux release 2026-07-25 02:55:35 +02:00
parent 9223c54939
commit 2a0f7ded04
37 changed files with 3145 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

35
crates/clusterflux-coordinator/src/durable.rs
View file

@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
pub enrollment_scope: String,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct NodeScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub node: NodeId,
}
impl NodeScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
Self {
tenant,
project,
node,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
Self::new(tenant.clone(), project.clone(), node.clone())
}
pub fn credential_subject(&self) -> String {
format!(
"node:{}:{}:{}:{}:{}:{}",
self.tenant.as_str().len(),
self.tenant,
self.project.as_str().len(),
self.project,
self.node.as_str().len(),
self.node
)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CredentialRecord {
pub subject: String,
@ -107,7 +140,7 @@ pub struct DurableState {
pub tenants: BTreeMap<TenantId, TenantRecord>,
pub users: BTreeMap<UserId, UserRecord>,
pub projects: BTreeMap<ProjectId, ProjectRecord>,
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
pub credentials: BTreeMap<String, CredentialRecord>,
pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
pub source_provider_configs:

202
crates/clusterflux-coordinator/src/lib.rs
View file

@ -13,7 +13,7 @@ pub mod service;
mod sessions;
pub use durable::{
AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
TenantRecord, UserRecord,
};
@ -128,8 +128,9 @@ impl Coordinator {
public_key: impl Into<String>,
enrollment_scope: impl Into<String>,
) {
let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
self.durable.node_identities.insert(
node.clone(),
key,
NodeIdentityRecord {
id: node,
tenant,
@ -181,10 +182,13 @@ impl Coordinator {
public_key,
credential.scope.clone(),
);
let subject =
NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
.credential_subject();
self.durable.credentials.insert(
format!("node:{node}"),
subject.clone(),
CredentialRecord {
subject: format!("node:{node}"),
subject,
tenant: credential.tenant.clone(),
project: Some(credential.project.clone()),
kind: credential.credential_kind.clone(),
@ -356,15 +360,12 @@ impl Coordinator {
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node identity is outside the requested tenant/project scope".to_owned(),
));
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if !self
.active_processes
@ -379,14 +380,18 @@ impl Coordinator {
pub fn reconnect_node(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
process: Option<(&ProcessId, u64)>,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if let Some((process_id, stale_epoch)) = process {
if stale_epoch != self.coordinator_epoch {
@ -395,11 +400,7 @@ impl Coordinator {
current_epoch: self.coordinator_epoch,
});
}
let key = (
identity.tenant.clone(),
identity.project.clone(),
process_id.clone(),
);
let key = (tenant.clone(), project.clone(), process_id.clone());
if let Some(active) = self.active_processes.get_mut(&key) {
active.connected_nodes.insert(node.clone());
}
@ -416,23 +417,27 @@ impl Coordinator {
let identity = self
.durable
.node_identities
.get(node)
.get(&NodeScopeKey::from_refs(
&context.tenant,
&context.project,
node,
))
.ok_or(CoordinatorError::UnknownNode)?
.clone();
if identity.tenant != context.tenant || identity.project != context.project {
return Err(CoordinatorError::Unauthorized(
"node credential is outside the signed-in tenant/project scope".to_owned(),
));
}
if !matches!(context.actor, Actor::User(_)) {
return Err(CoordinatorError::Unauthorized(
"node credential revocation requires a user identity".to_owned(),
));
}
self.durable.node_identities.remove(node);
self.durable.credentials.remove(&format!("node:{node}"));
for active in self.active_processes.values_mut() {
active.connected_nodes.remove(node);
let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
self.durable.node_identities.remove(&key);
self.durable.credentials.remove(&key.credential_subject());
for active in self
.active_processes
.values_mut()
.filter(|active| active.tenant == context.tenant && active.project == context.project)
{
active.connected_nodes.remove(&key.node);
}
Ok(identity)
}
@ -580,8 +585,15 @@ impl Coordinator {
.count()
}
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
self.durable.node_identities.get(id)
pub fn node_identity(
&self,
tenant: &TenantId,
project: &ProjectId,
id: &NodeId,
) -> Option<&NodeIdentityRecord> {
self.durable
.node_identities
.get(&NodeScopeKey::from_refs(tenant, project, id))
}
pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@ -683,12 +695,24 @@ mod tests {
.contains_key(&TenantId::from("tenant")));
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
assert_eq!(
restarted
.durable
.credentials
.get("node:node")
.get(&node_subject)
.map(|credential| &credential.kind),
Some(&CredentialKind::NodeCredential)
);
@ -712,7 +736,12 @@ mod tests {
);
assert_eq!(rerun.coordinator_epoch, 2);
restarted
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&process, 2)),
)
.unwrap();
assert!(restarted
.active_process(
@ -725,6 +754,68 @@ mod tests {
.contains(&NodeId::from("node")));
}
#[test]
fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
let store = InMemoryDurableStore::default();
let mut coordinator = Coordinator::boot(&store, 1);
let node = NodeId::from("shared-node");
let scopes = [
(TenantId::from("tenant-a"), ProjectId::from("project-a")),
(TenantId::from("tenant-b"), ProjectId::from("project-b")),
(TenantId::from("tenant-a"), ProjectId::from("project-c")),
];
for (index, (tenant, project)) in scopes.iter().enumerate() {
let mut grant = coordinator.create_node_enrollment_grant(
tenant.clone(),
project.clone(),
format!("grant-{index}"),
"node:attach",
100,
);
coordinator
.exchange_node_enrollment_grant(
&mut grant,
node.clone(),
&format!("public-key-{index}"),
"node:attach",
99,
)
.unwrap();
}
let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
assert_eq!(
coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
coordinator.durable.node_identities.get(&scope_a)
);
assert_eq!(
coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
coordinator.durable.node_identities.get(&scope_b)
);
assert_eq!(
coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
coordinator.durable.node_identities.get(&scope_c)
);
assert!(coordinator
.durable
.credentials
.contains_key(&scope_a.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_b.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_c.credential_subject()));
assert_eq!(coordinator.durable.node_identities.len(), 3);
assert_eq!(coordinator.durable.credentials.len(), 3);
}
#[test]
fn identical_process_ids_are_isolated_by_tenant_and_project() {
let store = InMemoryDurableStore::default();
@ -773,11 +864,18 @@ mod tests {
let mut restarted = Coordinator::boot(&store, 2);
restarted
.reconnect_node(&NodeId::from("node"), None)
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
None,
)
.unwrap();
let error = restarted
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -809,7 +907,13 @@ mod tests {
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
}
#[test]
@ -823,10 +927,16 @@ mod tests {
"public-key",
"node:attach",
);
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
coordinator.durable.credentials.insert(
"node:node".to_owned(),
node_subject.clone(),
CredentialRecord {
subject: "node:node".to_owned(),
subject: node_subject.clone(),
tenant: TenantId::from("tenant"),
project: Some(ProjectId::from("project")),
kind: CredentialKind::NodeCredential,
@ -840,6 +950,8 @@ mod tests {
);
coordinator
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -855,7 +967,7 @@ mod tests {
&NodeId::from("node"),
)
.unwrap_err();
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
assert!(matches!(foreign, CoordinatorError::UnknownNode));
let revoked = coordinator
.revoke_node_credential(
@ -868,8 +980,14 @@ mod tests {
)
.unwrap();
assert_eq!(revoked.id, NodeId::from("node"));
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
assert!(!coordinator.durable.credentials.contains_key("node:node"));
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_none());
assert!(!coordinator.durable.credentials.contains_key(&node_subject));
assert!(!coordinator
.active_process(
&TenantId::from("tenant"),
@ -1084,7 +1202,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
assert!(matches!(error, CoordinatorError::UnknownNode));
}
#[test]

142
crates/clusterflux-coordinator/src/postgres_store.rs
View file

@ -4,8 +4,8 @@ use thiserror::Error;
use crate::{
AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
state.projects.insert(record.id.clone(), record);
}
for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM clusterflux_node_identities ORDER BY node_id",
"SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
)? {
state.node_identities.insert(record.id.clone(), record);
state.node_identities.insert(
NodeScopeKey::new(
record.tenant.clone(),
record.project.clone(),
record.id.clone(),
),
record,
);
}
for record in self.query_records::<CredentialRecord>(
"SELECT record FROM clusterflux_credentials ORDER BY subject",
@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
);
CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
node_id TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, node_id)
);
DO $clusterflux_node_scope_migration$
DECLARE
current_primary_key TEXT;
current_primary_key_columns TEXT[];
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_node_identities
WHERE record->>'id' IS DISTINCT FROM node_id
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node identity migration refused an internally inconsistent legacy row';
END IF;
SELECT
constraint_record.conname,
array_agg(attribute.attname ORDER BY key_column.ordinality)
INTO current_primary_key, current_primary_key_columns
FROM pg_constraint AS constraint_record
CROSS JOIN LATERAL unnest(constraint_record.conkey)
WITH ORDINALITY AS key_column(attribute_number, ordinality)
JOIN pg_attribute AS attribute
ON attribute.attrelid = constraint_record.conrelid
AND attribute.attnum = key_column.attribute_number
WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
AND constraint_record.contype = 'p'
GROUP BY constraint_record.conname;
IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
EXECUTE format(
'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
current_primary_key
);
ALTER TABLE clusterflux_node_identities
ADD PRIMARY KEY (tenant_id, project_id, node_id);
ELSIF current_primary_key_columns
IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
THEN
RAISE EXCEPTION
'clusterflux node identity migration found an unexpected primary key shape';
END IF;
END
$clusterflux_node_scope_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
record JSONB NOT NULL
);
DO $clusterflux_node_credential_migration$
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_credentials
WHERE record->>'subject' IS DISTINCT FROM subject
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node credential migration refused an internally inconsistent legacy row';
END IF;
UPDATE clusterflux_credentials AS credential
SET subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
),
record = jsonb_set(
credential.record,
'{subject}',
to_jsonb(format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)),
false
)
FROM clusterflux_node_identities AS identity
WHERE credential.subject = 'node:' || identity.node_id
AND credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id;
IF EXISTS (
SELECT 1
FROM clusterflux_credentials AS credential
WHERE credential.subject LIKE 'node:%'
AND NOT EXISTS (
SELECT 1
FROM clusterflux_node_identities AS identity
WHERE credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id
AND credential.subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)
)
) THEN
RAISE EXCEPTION
'clusterflux node credential migration found an unscoped or orphaned node subject';
END IF;
END
$clusterflux_node_credential_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -529,7 +651,13 @@ mod tests {
let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
assert_eq!(restarted.active_process_count(), 0);
}

19
crates/clusterflux-coordinator/src/service.rs
View file

@ -13,7 +13,7 @@ use clusterflux_core::{
};
use thiserror::Error;
use crate::{Coordinator, CoordinatorError};
use crate::{Coordinator, CoordinatorError, NodeScopeKey};
mod admin;
mod artifacts;
@ -154,8 +154,8 @@ pub enum CoordinatorServiceError {
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
@ -180,7 +180,7 @@ pub struct CoordinatorService {
process_cancellations: BTreeSet<ProcessControlKey>,
process_aborts: BTreeSet<ProcessControlKey>,
agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
node_replay_nonces: BTreeMap<(NodeId, String), u64>,
node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
stopped_panels: BTreeSet<PanelStopKey>,
panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@ -284,15 +284,10 @@ impl CoordinatorService {
{
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node process-control request is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
return Ok(());
}
self.coordinator

51
crates/clusterflux-coordinator/src/service/artifacts.rs
View file

@ -8,7 +8,7 @@ use clusterflux_core::{
};
use sha2::{Digest as _, Sha256};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::relay::RelayFinishReason;
use super::{
@ -52,7 +52,11 @@ impl CoordinatorService {
let action = self
.artifact_registry
.download_action(&context, &artifact, &policy)?;
self.ensure_download_source_connectivity(&action.source)?;
self.ensure_download_source_connectivity(
&context.tenant,
&context.project,
&action.source,
)?;
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
@ -163,7 +167,11 @@ impl CoordinatorService {
},
&mut validation_meter,
)?;
self.ensure_download_source_connectivity(&stream.link.source)?;
self.ensure_download_source_connectivity(
&stream.link.tenant,
&stream.link.project,
&stream.link.source,
)?;
self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@ -302,7 +310,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?;
@ -410,7 +418,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination =
@ -682,15 +690,10 @@ impl CoordinatorService {
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer node is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
Ok(())
}
@ -725,15 +728,12 @@ impl CoordinatorService {
) -> Result<NodeEndpoint, CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact export node is outside the tenant/project scope".to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity"
))
@ -744,7 +744,7 @@ impl CoordinatorService {
)
.into());
}
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export"
@ -769,17 +769,20 @@ impl CoordinatorService {
fn ensure_download_source_connectivity(
&self,
tenant: &TenantId,
project: &ProjectId,
source: &StorageLocation,
) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else {
return Ok(());
};
let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download"
))
})?;
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} is offline for artifact download"

38
crates/clusterflux-coordinator/src/service/debug_requests.rs
View file

@ -226,24 +226,25 @@ impl CoordinatorService {
} else if !completed_event_observed {
"selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
} else if let Some(checkpoint) = checkpoint {
let vfs_available =
checkpoint
.checkpoint
.vfs_manifest
.objects
.iter()
.all(|(path, object)| {
let artifact = super::keys::artifact_id_from_path(path);
self.artifact_registry
.metadata(&artifact)
.is_some_and(|metadata| {
metadata.tenant == tenant
&& metadata.project == project
&& metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
})
});
let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
true,
|available, (path, object)| {
let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?;
Ok::<_, CoordinatorServiceError>(
available
&& self
.artifact_registry
.metadata(&tenant, &project, &artifact)
.is_some_and(|metadata| {
metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
}),
)
},
)?;
if !vfs_available {
"selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
} else {
@ -479,6 +480,7 @@ impl CoordinatorService {
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
self.maybe_retire_terminal_process(&tenant, &project, &process)?;
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,

45
crates/clusterflux-coordinator/src/service/keys.rs
View file

@ -1,6 +1,7 @@
use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
use thiserror::Error;
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
(tenant.clone(), project.clone(), grant.to_owned())
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub(super) enum ArtifactPathError {
#[error("path must start with the exact /vfs/artifacts/ prefix")]
WrongPrefix,
#[error("path must name an artifact after /vfs/artifacts/")]
EmptyArtifact,
#[error("mapped artifact identifier is invalid: {0}")]
InvalidArtifactId(#[from] clusterflux_core::IdParseError),
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
let value = path
.as_str()
.strip_prefix("/vfs/artifacts/")
.unwrap_or(path.as_str())
.replace('/', ":");
ArtifactId::new(value)
.ok_or(ArtifactPathError::WrongPrefix)?;
if value.is_empty() {
return Err(ArtifactPathError::EmptyArtifact);
}
ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn artifact_id_conversion_is_fallible_and_structural() {
assert_eq!(
artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
ArtifactId::from("build:output")
);
for path in [
"/vfs/other/output",
"/vfs/artifacts",
"/vfs/artifacts/bad artifact!",
] {
let path = VfsPath::new(path).unwrap();
assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
}
let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
}
}

153
crates/clusterflux-coordinator/src/service/logs.rs
View file

@ -1,5 +1,5 @@
use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
};
@ -85,7 +85,9 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant,
project,
process: process.clone(),
@ -203,7 +205,9 @@ impl CoordinatorService {
event.placement = self.task_placements.remove(&task_key);
if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant: event.tenant.clone(),
project: event.project.clone(),
process: event.process.clone(),
@ -217,34 +221,16 @@ impl CoordinatorService {
self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key);
let no_active_tasks =
!self
.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &event.tenant
&& task_project == &event.project
&& task_process == &event.process
});
let completed_after_main = matches!(event.terminal_state, TaskTerminalState::Completed)
&& self.task_events.iter().rev().any(|retained| {
retained.tenant == event.tenant
&& retained.project == event.project
&& retained.process == event.process
&& matches!(retained.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(retained.terminal_state, TaskTerminalState::Completed)
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != event.tenant
|| assignment.project != event.project
|| assignment.process != event.process
|| assignment.node != event.node
|| assignment.task != event.task
});
if no_active_tasks {
self.process_aborts.remove(&process_key);
if (self.process_cancellations.remove(&process_key) || completed_after_main)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
}
!assignments.is_empty()
});
if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key(
&event.tenant,
@ -261,6 +247,7 @@ impl CoordinatorService {
if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
Ok(CoordinatorResponse::TaskRecorded {
process: event.process,
task: event.task,
@ -553,6 +540,82 @@ impl CoordinatorService {
awaiting_operator
}
pub(super) fn maybe_retire_terminal_process(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<bool, CoordinatorServiceError> {
let process_key = process_control_key(tenant, project, process);
if self.main_runtime.controls.contains_key(&process_key) {
return Ok(false);
}
let has_runnable_remote_work =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == tenant && task_project == project && task_process == process
})
|| self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
})
|| self.task_assignments.values().any(|assignments| {
assignments.iter().any(|assignment| {
&assignment.tenant == tenant
&& &assignment.project == project
&& &assignment.process == process
})
})
|| self.task_attempts.iter().any(
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
attempt_tenant == tenant
&& attempt_project == project
&& attempt_process == process
&& attempts.iter().rev().any(|attempt| {
attempt.current
&& matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
},
);
if has_runnable_remote_work {
return Ok(false);
}
let main_completed = self.task_events.iter().rev().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& matches!(event.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(event.terminal_state, TaskTerminalState::Completed)
});
let cancellation_completed = self.process_cancellations.contains(&process_key);
if !main_completed && !cancellation_completed {
return Ok(false);
}
self.process_aborts.remove(&process_key);
self.process_cancellations.remove(&process_key);
if self
.coordinator
.active_process(tenant, project, process)
.is_none()
{
return Ok(false);
}
self.coordinator.abort_process(tenant, project, process)?;
self.clear_debug_state_for_process(tenant, project, process);
self.clear_operator_panel_state(tenant, project, process);
Ok(true)
}
fn flush_artifact_metadata(
&mut self,
flush: ArtifactFlush,
@ -560,17 +623,25 @@ impl CoordinatorService {
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let pinned = self
.task_restart_checkpoints
.values()
.flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
.chain(
self.pending_task_launches
.iter()
.flat_map(|pending| pending.task_spec.required_artifacts.iter()),
)
.cloned()
.collect::<std::collections::BTreeSet<ArtifactId>>();
let mut pinned = std::collections::BTreeSet::new();
for checkpoint in self.task_restart_checkpoints.values() {
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&checkpoint.assignment.tenant,
&checkpoint.assignment.project,
artifact,
));
}
}
for pending in &self.pending_task_launches {
for artifact in &pending.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&pending.tenant,
&pending.project,
artifact,
));
}
}
self.artifact_registry
.flush_metadata_bounded(flush, &pinned)
.map(|_| ())

14
crates/clusterflux-coordinator/src/service/main_runtime.rs
View file

@ -971,16 +971,10 @@ impl CoordinatorService {
}
}
let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
let has_active_children =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &scope.tenant
&& task_project == &scope.project
&& task_process == &scope.process
});
self.main_runtime.controls.remove(&process_key);
if main_completed && has_active_children {
if main_completed {
let _ =
self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
return;
}
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
@ -1169,7 +1163,7 @@ mod tests {
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
assert!(service.process_aborts.contains(&process_key));
assert!(!service.process_aborts.contains(&process_key));
}
#[test]

76
crates/clusterflux-coordinator/src/service/nodes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
SourceProviderKind, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{
bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@ -27,9 +27,9 @@ impl CoordinatorService {
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.get(scope)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
@ -41,7 +41,11 @@ impl CoordinatorService {
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
&descriptor.tenant,
&descriptor.project,
&descriptor.id,
));
descriptor
})
.collect()
@ -165,7 +169,11 @@ impl CoordinatorService {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
self.coordinator.ensure_tenant_active(&tenant)?;
if self.coordinator.node_identity(&node).is_none() {
if self
.coordinator
.node_identity(&tenant, &project, &node)
.is_none()
{
self.quota.ensure_node_admission(
&tenant,
self.coordinator.node_identity_count_for_tenant(&tenant),
@ -197,12 +205,21 @@ impl CoordinatorService {
pub(super) fn handle_node_heartbeat(
&mut self,
tenant: String,
project: String,
node: String,
node_signature: Option<NodeSignedRequest>,
payload_digest: &Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
self.authenticate_node_request(
&NodeScopeKey::new(tenant, project, node.clone()),
node_signature,
"node_heartbeat",
payload_digest,
)?;
Ok(CoordinatorResponse::NodeHeartbeat {
node,
epoch: self.coordinator.coordinator_epoch(),
@ -225,16 +242,13 @@ impl CoordinatorService {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node capability report is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
capabilities.validate_public_report()?;
for (kind, count) in [
("cached environments", cached_environment_digests.len()),
@ -274,12 +288,16 @@ impl CoordinatorService {
.into_iter()
.map(ArtifactId::new)
.collect::<BTreeSet<_>>();
self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations);
self.artifact_registry.reconcile_node_retention(
&tenant,
&project,
&node,
&artifact_locations,
);
let online = self.node_is_live(&node);
let online = self.node_is_live(&node_scope);
self.node_descriptors.insert(
node.clone(),
node_scope,
NodeDescriptor {
id: node.clone(),
tenant,
@ -333,9 +351,13 @@ impl CoordinatorService {
actor: Actor::User(actor.clone()),
};
self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
self.node_last_seen_epoch_seconds.remove(&node_scope);
self.node_replay_nonces
.retain(|(retained_scope, _), _| retained_scope != &node_scope);
self.artifact_registry
.garbage_collect_node(&tenant, &project, &node);
let queued_assignments_removed = self
.task_assignments
.remove(&(tenant.clone(), project.clone(), node.clone()))
@ -369,14 +391,14 @@ impl CoordinatorService {
pub(super) fn authenticate_node_request(
&mut self,
node: &NodeId,
scope: &NodeScopeKey,
node_signature: Option<NodeSignedRequest>,
request_kind: &str,
payload_digest: &Digest,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(&scope.tenant, &scope.project, &scope.node)
.ok_or(CoordinatorError::UnknownNode)?;
let signature = node_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
@ -401,7 +423,7 @@ impl CoordinatorService {
)
.into());
}
let replay_key = (node.clone(), signature.nonce.clone());
let replay_key = (scope.clone(), signature.nonce.clone());
self.node_replay_nonces.retain(|_, accepted_at| {
now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
});
@ -413,7 +435,7 @@ impl CoordinatorService {
}
verify_node_request_signature(
&identity.public_key,
node,
&scope.node,
request_kind,
payload_digest,
&signature,
@ -422,7 +444,7 @@ impl CoordinatorService {
if self
.node_replay_nonces
.keys()
.filter(|(retained_node, _)| retained_node == node)
.filter(|(retained_scope, _)| retained_scope == scope)
.count()
>= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
{
@ -436,8 +458,8 @@ impl CoordinatorService {
.insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
.insert(scope.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
descriptor.online = true;
}
Ok(())

3
crates/clusterflux-coordinator/src/service/panels.rs
View file

@ -254,7 +254,8 @@ impl CoordinatorService {
.rev()
.find_map(|event| event.artifact_path.as_ref())
{
let artifact = artifact_id_from_path(path);
let artifact = artifact_id_from_path(path)
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
let context = clusterflux_core::AuthContext {
tenant,
project,

29
crates/clusterflux-coordinator/src/service/process_launch.rs
View file

@ -118,14 +118,14 @@ impl CoordinatorService {
let mut objects = BTreeMap::new();
let mut missing_required_artifact = false;
for artifact in &task_spec.required_artifacts {
let Some(metadata) = self.artifact_registry.metadata(artifact) else {
let Some(metadata) =
self.artifact_registry
.metadata(&assignment.tenant, &assignment.project, artifact)
else {
missing_required_artifact = true;
continue;
};
if metadata.tenant != assignment.tenant
|| metadata.project != assignment.project
|| metadata.retaining_nodes.is_empty()
{
if metadata.retaining_nodes.is_empty() {
missing_required_artifact = true;
continue;
}
@ -483,17 +483,14 @@ impl CoordinatorService {
.validate()
.map_err(CoordinatorServiceError::Protocol)?;
for artifact in &task_spec.required_artifacts {
let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired"
))
})?;
if metadata.tenant != tenant || metadata.project != project {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is outside the task tenant/project scope"
))
.into());
}
let metadata = self
.artifact_registry
.metadata(&tenant, &project, artifact)
.ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired in this tenant/project scope"
))
})?;
if metadata.retaining_nodes.is_empty() {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} has no retaining node"

48
crates/clusterflux-coordinator/src/service/processes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::keys::{process_control_key, task_control_key};
use super::{
@ -47,14 +47,10 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"task assignment poll is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let assignment_key = (tenant.clone(), project.clone(), node.clone());
let assignment = self
.task_assignments
@ -77,7 +73,11 @@ impl CoordinatorService {
project: &ProjectId,
node: &NodeId,
) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
let Some(descriptor) = self
.node_descriptors
.get(&NodeScopeKey::from_refs(tenant, project, node))
.cloned()
else {
return Ok(None);
};
let mut remaining = VecDeque::new();
@ -233,20 +233,18 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"source preparation completion is outside the enrolled tenant/project scope"
.to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let descriptor = self
.node_descriptors
.get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
if !descriptor.source_snapshots.contains(&source_snapshot)
&& descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
{
@ -413,14 +411,18 @@ impl CoordinatorService {
pub(super) fn handle_reconnect_node(
&mut self,
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let process = ProcessId::new(process);
self.coordinator
.reconnect_node(&node, Some((&process, epoch)))?;
.reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
Ok(CoordinatorResponse::NodeReconnected { node, process })
}

20
crates/clusterflux-coordinator/src/service/protocol.rs
View file

@ -121,6 +121,8 @@ pub enum CoordinatorRequest {
enrollment_grant: String,
},
NodeHeartbeat {
tenant: String,
project: String,
node: String,
#[serde(default)]
node_signature: Option<NodeSignedRequest>,
@ -269,6 +271,8 @@ pub enum CoordinatorRequest {
restart: bool,
},
ReconnectNode {
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
@ -675,9 +679,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512)
}
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
if let Some(signature) = node_signature {
validate_node_signature(signature, &format!("{path}.node_signature"))?;
@ -846,7 +853,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
&format!("{path}.launch_attempt"),
)
}
CoordinatorRequest::ReconnectNode { node, process, .. } => {
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
..
} => {
validate_tenant_project(tenant, project, path)?;
validate_node(node, &format!("{path}.node"))?;
validate_process(process, &format!("{path}.process"))
}
@ -1567,6 +1581,8 @@ mod external_identifier_tests {
restart: false,
},
CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "bad node!".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: "node-nonce".to_owned(),
@ -1626,6 +1642,8 @@ mod external_identifier_tests {
assert!(error.contains("malformed external token request.agent_signature.nonce"));
let node_request = CoordinatorRequest::NodeHeartbeat {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "node".to_owned(),
node_signature: Some(NodeSignedRequest {
nonce: String::new(),

10
crates/clusterflux-coordinator/src/service/routing.rs
View file

@ -274,9 +274,17 @@ impl CoordinatorService {
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
} => self.handle_node_heartbeat(
tenant,
project,
node,
node_signature,
&request_payload_digest,
),
CoordinatorRequest::SignedNode {
node,
node_signature,

155
crates/clusterflux-coordinator/src/service/signed_nodes.rs
View file

@ -1,6 +1,6 @@
use clusterflux_core::{NodeId, NodeSignedRequest};
use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
@ -12,7 +12,7 @@ impl CoordinatorService {
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request_kind = signed_node_request_kind(&request)?;
let request_node = signed_node_request_node(&request)?;
let request_scope = signed_node_request_scope(&request)?;
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize signed node request: {error}"
@ -22,14 +22,14 @@ impl CoordinatorService {
let signed_node = NodeId::try_new(signed_node).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
})?;
if request_node != signed_node {
if request_scope.node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
)
.into());
}
self.authenticate_node_request(
&signed_node,
&request_scope,
Some(node_signature),
request_kind,
&payload_digest,
@ -160,10 +160,12 @@ impl CoordinatorService {
failure_reason,
),
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
epoch,
} => self.handle_reconnect_node(node, process, epoch),
} => self.handle_reconnect_node(tenant, project, node, process, epoch),
CoordinatorRequest::PollTaskControl {
tenant,
project,
@ -353,36 +355,129 @@ fn signed_node_request_kind(
}
}
fn signed_node_request_node(
fn signed_node_request_scope(
request: &CoordinatorRequest,
) -> Result<NodeId, CoordinatorServiceError> {
) -> Result<NodeScopeKey, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { node, .. }
| CoordinatorRequest::PollTaskAssignment { node, .. }
| CoordinatorRequest::PollArtifactTransfer { node, .. }
| CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
| CoordinatorRequest::FailArtifactTransfer { node, .. }
| CoordinatorRequest::LaunchChildTask { node, .. }
| CoordinatorRequest::JoinChildTask { node, .. }
| CoordinatorRequest::CompleteSourcePreparation { node, .. }
| CoordinatorRequest::ReconnectNode { node, .. }
| CoordinatorRequest::PollTaskControl { node, .. }
| CoordinatorRequest::PollDebugCommand { node, .. }
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => {
NodeId::try_new(node.clone()).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"invalid wrapped node identifier: {error}"
))
})
CoordinatorRequest::ReportNodeCapabilities {
tenant,
project,
node,
..
}
CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()),
| CoordinatorRequest::PollTaskAssignment {
tenant,
project,
node,
}
| CoordinatorRequest::PollArtifactTransfer {
tenant,
project,
node,
}
| CoordinatorRequest::UploadArtifactTransferChunk {
tenant,
project,
node,
..
}
| CoordinatorRequest::FailArtifactTransfer {
tenant,
project,
node,
..
}
| CoordinatorRequest::LaunchChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::JoinChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::CompleteSourcePreparation {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollTaskControl {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollDebugCommand {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugState {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugProbeHit {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportTaskLog {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
node,
..
}
| CoordinatorRequest::TaskCompleted {
tenant,
project,
node,
..
} => node_scope_from_strings(tenant, project, node),
CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
scope.tenant.clone(),
scope.project.clone(),
source.node.clone(),
)),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}
fn node_scope_from_strings(
tenant: &str,
project: &str,
node: &str,
) -> Result<NodeScopeKey, CoordinatorServiceError> {
let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
})?;
let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
})?;
let node = NodeId::try_new(node.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
})?;
Ok(NodeScopeKey::new(tenant, project, node))
}

1076
crates/clusterflux-coordinator/src/service/tests.rs
View file

File diff suppressed because it is too large Load diff