6f52bb46cd
Source commit: a43e907efd9d1561c23fe73499478e881f868355 Public tree identity: sha256:453dea30195485dd8939f575a69b39f4bb7acd84c4df23f8aa29b55d044f2673
1047 lines
34 KiB
Rust
1047 lines
34 KiB
Rust
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use clusterflux_core::{
|
|
Actor, AuthContext, Authorization, CredentialKind, Digest, EnrollmentError, EnrollmentGrant,
|
|
NodeCredential, NodeId, ProcessId, ProjectId, SourceProviderKind, TenantId, UserId,
|
|
};
|
|
use thiserror::Error;
|
|
|
|
mod agents;
|
|
pub mod durable;
|
|
pub mod postgres_store;
|
|
pub mod service;
|
|
mod sessions;
|
|
pub use durable::{
|
|
AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
|
|
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
|
|
ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
|
|
TenantRecord, UserRecord,
|
|
};
|
|
pub use postgres_store::{
|
|
PostgresDurableStore, PostgresStoreError, PostgresTable, POSTGRES_DURABLE_TABLES,
|
|
};
|
|
pub use service::{
|
|
ArtifactRelayConfiguration, ArtifactRelayError, ArtifactRelayUsage, CoordinatorAdmission,
|
|
CoordinatorMainRuntimeConfiguration, CoordinatorRequest, CoordinatorResponse,
|
|
CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugAuditEvent,
|
|
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
|
|
TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
|
|
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskTerminalState,
|
|
};
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct ActiveProcess {
|
|
pub id: ProcessId,
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub connected_nodes: BTreeSet<NodeId>,
|
|
pub coordinator_epoch: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct Coordinator {
|
|
durable: DurableState,
|
|
active_processes: BTreeMap<(TenantId, ProjectId, ProcessId), ActiveProcess>,
|
|
coordinator_epoch: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Error, PartialEq, Eq)]
|
|
pub enum CoordinatorError {
|
|
#[error("node identity is not enrolled")]
|
|
UnknownNode,
|
|
#[error("node enrollment failed: {0:?}")]
|
|
Enrollment(EnrollmentError),
|
|
#[error("stale virtual process state from coordinator epoch {stale_epoch}; current epoch is {current_epoch}")]
|
|
StaleProcessEpoch {
|
|
stale_epoch: u64,
|
|
current_epoch: u64,
|
|
},
|
|
#[error("unauthorized coordinator action: {0}")]
|
|
Unauthorized(String),
|
|
}
|
|
|
|
impl Coordinator {
|
|
pub fn boot(store: &impl DurableStore, coordinator_epoch: u64) -> Self {
|
|
Self {
|
|
durable: store.load(),
|
|
active_processes: BTreeMap::new(),
|
|
coordinator_epoch,
|
|
}
|
|
}
|
|
|
|
pub fn try_boot<S: FallibleDurableStore>(
|
|
store: &mut S,
|
|
coordinator_epoch: u64,
|
|
) -> Result<Self, S::Error> {
|
|
Ok(Self {
|
|
durable: store.load_state()?,
|
|
active_processes: BTreeMap::new(),
|
|
coordinator_epoch,
|
|
})
|
|
}
|
|
|
|
pub fn persist(&self, store: &mut impl DurableStore) {
|
|
store.save(self.durable.clone());
|
|
}
|
|
|
|
pub fn try_persist<S: FallibleDurableStore>(&self, store: &mut S) -> Result<(), S::Error> {
|
|
store.save_state(&self.durable)
|
|
}
|
|
|
|
pub fn coordinator_epoch(&self) -> u64 {
|
|
self.coordinator_epoch
|
|
}
|
|
|
|
pub fn upsert_tenant(&mut self, id: TenantId) {
|
|
self.durable.tenants.insert(id.clone(), TenantRecord { id });
|
|
}
|
|
|
|
pub fn upsert_user(&mut self, tenant: TenantId, id: UserId, credential_kind: CredentialKind) {
|
|
self.durable.users.insert(
|
|
id.clone(),
|
|
UserRecord {
|
|
id,
|
|
tenant,
|
|
credential_kind,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn upsert_project(&mut self, tenant: TenantId, id: ProjectId, name: impl Into<String>) {
|
|
self.durable.projects.insert(
|
|
id.clone(),
|
|
ProjectRecord {
|
|
id,
|
|
tenant,
|
|
name: name.into(),
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn enroll_node(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
node: NodeId,
|
|
public_key: impl Into<String>,
|
|
enrollment_scope: impl Into<String>,
|
|
) {
|
|
self.durable.node_identities.insert(
|
|
node.clone(),
|
|
NodeIdentityRecord {
|
|
id: node,
|
|
tenant,
|
|
project,
|
|
public_key: public_key.into(),
|
|
enrollment_scope: enrollment_scope.into(),
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn create_node_enrollment_grant(
|
|
&self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
grant_id: impl Into<String>,
|
|
scope: impl Into<String>,
|
|
expires_at_epoch_seconds: u64,
|
|
) -> EnrollmentGrant {
|
|
EnrollmentGrant {
|
|
tenant,
|
|
project,
|
|
grant_id: grant_id.into(),
|
|
scope: scope.into(),
|
|
expires_at_epoch_seconds,
|
|
consumed: false,
|
|
}
|
|
}
|
|
|
|
pub fn exchange_node_enrollment_grant(
|
|
&mut self,
|
|
grant: &mut EnrollmentGrant,
|
|
node: NodeId,
|
|
public_key: &str,
|
|
requested_scope: &str,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<NodeCredential, CoordinatorError> {
|
|
let credential = grant
|
|
.exchange_for_node_identity(
|
|
node.clone(),
|
|
public_key,
|
|
requested_scope,
|
|
now_epoch_seconds,
|
|
)
|
|
.map_err(CoordinatorError::Enrollment)?;
|
|
self.enroll_node(
|
|
credential.tenant.clone(),
|
|
credential.project.clone(),
|
|
node.clone(),
|
|
public_key,
|
|
credential.scope.clone(),
|
|
);
|
|
self.durable.credentials.insert(
|
|
format!("node:{node}"),
|
|
CredentialRecord {
|
|
subject: format!("node:{node}"),
|
|
tenant: credential.tenant.clone(),
|
|
project: Some(credential.project.clone()),
|
|
kind: credential.credential_kind.clone(),
|
|
public_key_fingerprint: Some(credential.public_key_fingerprint.clone()),
|
|
},
|
|
);
|
|
Ok(credential)
|
|
}
|
|
|
|
pub fn upsert_source_provider_config(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
provider: SourceProviderKind,
|
|
manifest_digest: Digest,
|
|
) {
|
|
let provider_key = format!("{provider:?}");
|
|
self.durable.source_provider_configs.insert(
|
|
(tenant.clone(), project.clone(), provider_key),
|
|
SourceProviderConfigRecord {
|
|
tenant,
|
|
project,
|
|
provider,
|
|
manifest_digest,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn upsert_service_policy_record(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
name: impl Into<String>,
|
|
digest: Digest,
|
|
) {
|
|
let name = name.into();
|
|
self.durable.service_policy_records.insert(
|
|
(tenant.clone(), name.clone()),
|
|
ServicePolicyRecord {
|
|
tenant,
|
|
name,
|
|
digest,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn suspend_tenant(&mut self, tenant: TenantId, actor: UserId) -> ServicePolicyRecord {
|
|
self.upsert_tenant(tenant.clone());
|
|
let name = "tenant:suspended".to_owned();
|
|
let digest = Digest::from_parts([
|
|
b"tenant-suspension:v1".as_slice(),
|
|
tenant.as_str().as_bytes(),
|
|
actor.as_str().as_bytes(),
|
|
]);
|
|
self.upsert_service_policy_record(tenant.clone(), name.clone(), digest);
|
|
self.service_policy_record(&tenant, &name)
|
|
.expect("tenant suspension record was just inserted")
|
|
.clone()
|
|
}
|
|
|
|
pub fn tenant_suspended(&self, tenant: &TenantId) -> bool {
|
|
self.service_policy_record(tenant, "tenant:suspended")
|
|
.is_some()
|
|
}
|
|
|
|
pub fn tenant_disabled(&self, tenant: &TenantId) -> bool {
|
|
self.service_policy_record(tenant, "tenant:disabled")
|
|
.is_some()
|
|
}
|
|
|
|
pub fn tenant_deleted(&self, tenant: &TenantId) -> bool {
|
|
self.service_policy_record(tenant, "tenant:deleted")
|
|
.is_some()
|
|
}
|
|
|
|
pub fn tenant_manual_review(&self, tenant: &TenantId) -> bool {
|
|
self.service_policy_record(tenant, "tenant:manual_review")
|
|
.is_some()
|
|
}
|
|
|
|
pub fn account_policy_state(&self, tenant: &TenantId) -> AccountPolicyState {
|
|
let suspended = self.tenant_suspended(tenant);
|
|
let disabled = self.tenant_disabled(tenant);
|
|
let deleted = self.tenant_deleted(tenant);
|
|
let manual_review = self.tenant_manual_review(tenant);
|
|
let account_status = if deleted {
|
|
"deleted"
|
|
} else if disabled {
|
|
"disabled"
|
|
} else if suspended {
|
|
"suspended"
|
|
} else if manual_review {
|
|
"manual_review"
|
|
} else {
|
|
"active"
|
|
}
|
|
.to_owned();
|
|
let sanitized_reason = match account_status.as_str() {
|
|
"deleted" => Some("account or tenant is no longer active".to_owned()),
|
|
"disabled" => Some("account or tenant is disabled by hosted policy".to_owned()),
|
|
"suspended" => Some("account or tenant is suspended by hosted policy".to_owned()),
|
|
"manual_review" => Some("account or tenant is pending hosted review".to_owned()),
|
|
_ => None,
|
|
};
|
|
let next_actions = if account_status == "active" {
|
|
Vec::new()
|
|
} else {
|
|
vec![
|
|
"clusterflux auth status --json".to_owned(),
|
|
"contact the hosted operator or use a self-hosted coordinator".to_owned(),
|
|
]
|
|
};
|
|
AccountPolicyState {
|
|
account_status,
|
|
suspended,
|
|
disabled,
|
|
deleted,
|
|
manual_review,
|
|
sanitized_reason,
|
|
next_actions,
|
|
}
|
|
}
|
|
|
|
pub fn ensure_tenant_active(&self, tenant: &TenantId) -> Result<(), CoordinatorError> {
|
|
let account_state = self.account_policy_state(tenant);
|
|
if account_state.account_status != "active" {
|
|
return Err(CoordinatorError::Unauthorized(format!(
|
|
"tenant is {} by admin controls",
|
|
account_state.account_status
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn start_process(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
id: ProcessId,
|
|
) -> ActiveProcess {
|
|
let process = ActiveProcess {
|
|
id: id.clone(),
|
|
tenant,
|
|
project,
|
|
connected_nodes: BTreeSet::new(),
|
|
coordinator_epoch: self.coordinator_epoch,
|
|
};
|
|
self.active_processes.insert(
|
|
(process.tenant.clone(), process.project.clone(), id),
|
|
process.clone(),
|
|
);
|
|
process
|
|
}
|
|
|
|
pub fn authorize_node_for_process(
|
|
&self,
|
|
node: &NodeId,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
process: &ProcessId,
|
|
) -> Result<(), CoordinatorError> {
|
|
let identity = self
|
|
.durable
|
|
.node_identities
|
|
.get(node)
|
|
.ok_or(CoordinatorError::UnknownNode)?;
|
|
if &identity.tenant != tenant || &identity.project != project {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node identity is outside the requested tenant/project scope".to_owned(),
|
|
));
|
|
}
|
|
if !self
|
|
.active_processes
|
|
.contains_key(&(tenant.clone(), project.clone(), process.clone()))
|
|
{
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"virtual process is not active in coordinator memory".to_owned(),
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn reconnect_node(
|
|
&mut self,
|
|
node: &NodeId,
|
|
process: Option<(&ProcessId, u64)>,
|
|
) -> Result<(), CoordinatorError> {
|
|
let identity = self
|
|
.durable
|
|
.node_identities
|
|
.get(node)
|
|
.ok_or(CoordinatorError::UnknownNode)?;
|
|
|
|
if let Some((process_id, stale_epoch)) = process {
|
|
if stale_epoch != self.coordinator_epoch {
|
|
return Err(CoordinatorError::StaleProcessEpoch {
|
|
stale_epoch,
|
|
current_epoch: self.coordinator_epoch,
|
|
});
|
|
}
|
|
let key = (
|
|
identity.tenant.clone(),
|
|
identity.project.clone(),
|
|
process_id.clone(),
|
|
);
|
|
if let Some(active) = self.active_processes.get_mut(&key) {
|
|
active.connected_nodes.insert(node.clone());
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn revoke_node_credential(
|
|
&mut self,
|
|
context: &AuthContext,
|
|
node: &NodeId,
|
|
) -> Result<NodeIdentityRecord, CoordinatorError> {
|
|
let identity = self
|
|
.durable
|
|
.node_identities
|
|
.get(node)
|
|
.ok_or(CoordinatorError::UnknownNode)?
|
|
.clone();
|
|
if identity.tenant != context.tenant || identity.project != context.project {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node credential is outside the signed-in tenant/project scope".to_owned(),
|
|
));
|
|
}
|
|
if !matches!(context.actor, Actor::User(_)) {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node credential revocation requires a user identity".to_owned(),
|
|
));
|
|
}
|
|
self.durable.node_identities.remove(node);
|
|
self.durable.credentials.remove(&format!("node:{node}"));
|
|
for active in self.active_processes.values_mut() {
|
|
active.connected_nodes.remove(node);
|
|
}
|
|
Ok(identity)
|
|
}
|
|
|
|
pub fn list_projects(&self, context: &AuthContext) -> Vec<ProjectRecord> {
|
|
self.durable
|
|
.projects
|
|
.values()
|
|
.filter(|project| project.tenant == context.tenant)
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
pub fn authorize_debug_attach(
|
|
&self,
|
|
context: &AuthContext,
|
|
process: &ProcessId,
|
|
) -> Authorization {
|
|
let Some(active) = self.active_process(&context.tenant, &context.project, process) else {
|
|
return Authorization::deny("virtual process is not active in this tenant or project");
|
|
};
|
|
let Actor::User(user) = &context.actor else {
|
|
return Authorization::deny("debug attach requires a user identity");
|
|
};
|
|
let permission = self.durable.project_permissions.get(&(
|
|
active.tenant.clone(),
|
|
active.project.clone(),
|
|
user.clone(),
|
|
));
|
|
if !permission.is_some_and(|permission| permission.can_debug) {
|
|
return Authorization::deny("debug attach requires explicit project permission");
|
|
}
|
|
Authorization::allow("debug attach authorized for project")
|
|
}
|
|
|
|
pub fn project(&self, id: &ProjectId) -> Option<&ProjectRecord> {
|
|
self.durable.projects.get(id)
|
|
}
|
|
|
|
pub fn active_process(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
id: &ProcessId,
|
|
) -> Option<&ActiveProcess> {
|
|
self.active_processes
|
|
.get(&(tenant.clone(), project.clone(), id.clone()))
|
|
}
|
|
|
|
pub fn active_process_for_project(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
) -> Option<&ActiveProcess> {
|
|
self.active_processes
|
|
.values()
|
|
.find(|active| &active.tenant == tenant && &active.project == project)
|
|
}
|
|
|
|
pub fn active_processes_for_project(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
) -> Vec<ActiveProcess> {
|
|
self.active_processes
|
|
.values()
|
|
.filter(|active| &active.tenant == tenant && &active.project == project)
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
pub fn active_process_exists_outside_scope(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
id: &ProcessId,
|
|
) -> bool {
|
|
self.active_processes.values().any(|active| {
|
|
active.id == *id && (active.tenant != *tenant || active.project != *project)
|
|
})
|
|
}
|
|
|
|
pub fn abort_process(
|
|
&mut self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
process: &ProcessId,
|
|
) -> Result<ActiveProcess, CoordinatorError> {
|
|
let key = (tenant.clone(), project.clone(), process.clone());
|
|
let active = self.active_processes.get(&key).ok_or_else(|| {
|
|
CoordinatorError::Unauthorized(
|
|
"process abort requires an active virtual process".to_owned(),
|
|
)
|
|
})?;
|
|
debug_assert_eq!(&active.tenant, tenant);
|
|
debug_assert_eq!(&active.project, project);
|
|
Ok(self
|
|
.active_processes
|
|
.remove(&key)
|
|
.expect("active process was checked immediately before removal"))
|
|
}
|
|
|
|
pub fn active_process_count(&self) -> usize {
|
|
self.active_processes.len()
|
|
}
|
|
|
|
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
|
|
self.durable.node_identities.get(id)
|
|
}
|
|
|
|
pub fn source_provider_config(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
provider: &str,
|
|
) -> Option<&SourceProviderConfigRecord> {
|
|
self.durable.source_provider_configs.get(&(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
provider.to_owned(),
|
|
))
|
|
}
|
|
|
|
pub fn service_policy_record(
|
|
&self,
|
|
tenant: &TenantId,
|
|
name: &str,
|
|
) -> Option<&ServicePolicyRecord> {
|
|
self.durable
|
|
.service_policy_records
|
|
.get(&(tenant.clone(), name.to_owned()))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use clusterflux_core::AgentId;
|
|
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn coordinator_restart_preserves_project_but_not_live_processes() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut first = Coordinator::boot(&store, 1);
|
|
first.upsert_tenant(TenantId::from("tenant"));
|
|
first.upsert_user(
|
|
TenantId::from("tenant"),
|
|
UserId::from("user"),
|
|
CredentialKind::CliDeviceSession,
|
|
);
|
|
first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
first.upsert_source_provider_config(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
SourceProviderKind::Git,
|
|
Digest::sha256("git-manifest"),
|
|
);
|
|
first.upsert_service_policy_record(
|
|
TenantId::from("tenant"),
|
|
"community tier",
|
|
Digest::sha256("policy"),
|
|
);
|
|
let mut grant = first.create_node_enrollment_grant(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
"grant",
|
|
"node:attach",
|
|
100,
|
|
);
|
|
first
|
|
.exchange_node_enrollment_grant(
|
|
&mut grant,
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
99,
|
|
)
|
|
.unwrap();
|
|
first.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
first.persist(&mut store);
|
|
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
|
|
assert!(restarted
|
|
.durable
|
|
.tenants
|
|
.contains_key(&TenantId::from("tenant")));
|
|
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
|
|
assert!(restarted.project(&ProjectId::from("project")).is_some());
|
|
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
|
|
assert_eq!(
|
|
restarted
|
|
.durable
|
|
.credentials
|
|
.get("node:node")
|
|
.map(|credential| &credential.kind),
|
|
Some(&CredentialKind::NodeCredential)
|
|
);
|
|
assert!(restarted
|
|
.source_provider_config(
|
|
&TenantId::from("tenant"),
|
|
&ProjectId::from("project"),
|
|
"Git"
|
|
)
|
|
.is_some());
|
|
assert!(restarted
|
|
.service_policy_record(&TenantId::from("tenant"), "community tier")
|
|
.is_some());
|
|
assert_eq!(restarted.active_process_count(), 0);
|
|
|
|
let process = ProcessId::from("process-rerun");
|
|
let rerun = restarted.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
process.clone(),
|
|
);
|
|
assert_eq!(rerun.coordinator_epoch, 2);
|
|
restarted
|
|
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
|
|
.unwrap();
|
|
assert!(restarted
|
|
.active_process(
|
|
&TenantId::from("tenant"),
|
|
&ProjectId::from("project"),
|
|
&process,
|
|
)
|
|
.unwrap()
|
|
.connected_nodes
|
|
.contains(&NodeId::from("node")));
|
|
}
|
|
|
|
#[test]
|
|
fn identical_process_ids_are_isolated_by_tenant_and_project() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
let process = ProcessId::from("vp-current");
|
|
let tenant_a = TenantId::from("tenant-a");
|
|
let project_a = ProjectId::from("project-a");
|
|
let tenant_b = TenantId::from("tenant-b");
|
|
let project_b = ProjectId::from("project-b");
|
|
|
|
coordinator.start_process(tenant_a.clone(), project_a.clone(), process.clone());
|
|
coordinator.start_process(tenant_b.clone(), project_b.clone(), process.clone());
|
|
|
|
assert_eq!(coordinator.active_process_count(), 2);
|
|
assert!(coordinator
|
|
.active_process(&tenant_a, &project_a, &process)
|
|
.is_some());
|
|
assert!(coordinator
|
|
.active_process(&tenant_b, &project_b, &process)
|
|
.is_some());
|
|
coordinator
|
|
.abort_process(&tenant_a, &project_a, &process)
|
|
.unwrap();
|
|
assert!(coordinator
|
|
.active_process(&tenant_a, &project_a, &process)
|
|
.is_none());
|
|
assert!(coordinator
|
|
.active_process(&tenant_b, &project_b, &process)
|
|
.is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn node_reconnect_rejects_stale_process_epoch_after_restart() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut first = Coordinator::boot(&store, 1);
|
|
first.upsert_tenant(TenantId::from("tenant"));
|
|
first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
first.enroll_node(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node",
|
|
);
|
|
first.persist(&mut store);
|
|
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
restarted
|
|
.reconnect_node(&NodeId::from("node"), None)
|
|
.unwrap();
|
|
|
|
let error = restarted
|
|
.reconnect_node(
|
|
&NodeId::from("node"),
|
|
Some((&ProcessId::from("process"), 1)),
|
|
)
|
|
.unwrap_err();
|
|
|
|
assert!(matches!(error, CoordinatorError::StaleProcessEpoch { .. }));
|
|
}
|
|
|
|
#[test]
|
|
fn node_enrollment_grant_becomes_persistent_node_identity() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
let mut grant = coordinator.create_node_enrollment_grant(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
"grant",
|
|
"node:attach",
|
|
100,
|
|
);
|
|
|
|
let credential = coordinator
|
|
.exchange_node_enrollment_grant(
|
|
&mut grant,
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
99,
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
|
|
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn node_credential_revocation_is_project_scoped_and_removes_identity() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.enroll_node(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
);
|
|
coordinator.durable.credentials.insert(
|
|
"node:node".to_owned(),
|
|
CredentialRecord {
|
|
subject: "node:node".to_owned(),
|
|
tenant: TenantId::from("tenant"),
|
|
project: Some(ProjectId::from("project")),
|
|
kind: CredentialKind::NodeCredential,
|
|
public_key_fingerprint: Some(Digest::sha256("public-key")),
|
|
},
|
|
);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
coordinator
|
|
.reconnect_node(
|
|
&NodeId::from("node"),
|
|
Some((&ProcessId::from("process"), 1)),
|
|
)
|
|
.unwrap();
|
|
|
|
let foreign = coordinator
|
|
.revoke_node_credential(
|
|
&AuthContext {
|
|
tenant: TenantId::from("other"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
},
|
|
&NodeId::from("node"),
|
|
)
|
|
.unwrap_err();
|
|
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
|
|
|
|
let revoked = coordinator
|
|
.revoke_node_credential(
|
|
&AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
},
|
|
&NodeId::from("node"),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(revoked.id, NodeId::from("node"));
|
|
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
|
|
assert!(!coordinator.durable.credentials.contains_key("node:node"));
|
|
assert!(!coordinator
|
|
.active_process(
|
|
&TenantId::from("tenant"),
|
|
&ProjectId::from("project"),
|
|
&ProcessId::from("process"),
|
|
)
|
|
.unwrap()
|
|
.connected_nodes
|
|
.contains(&NodeId::from("node")));
|
|
}
|
|
|
|
#[test]
|
|
fn project_listing_is_filtered_by_tenant() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.upsert_project(
|
|
TenantId::from("tenant-a"),
|
|
ProjectId::from("project-a"),
|
|
"a",
|
|
);
|
|
coordinator.upsert_project(
|
|
TenantId::from("tenant-b"),
|
|
ProjectId::from("project-b"),
|
|
"b",
|
|
);
|
|
|
|
let projects = coordinator.list_projects(&AuthContext {
|
|
tenant: TenantId::from("tenant-a"),
|
|
project: ProjectId::from("project-a"),
|
|
actor: Actor::User(UserId::from("user-a")),
|
|
});
|
|
|
|
assert_eq!(projects.len(), 1);
|
|
assert_eq!(projects[0].id, ProjectId::from("project-a"));
|
|
}
|
|
|
|
#[test]
|
|
fn tenant_suspension_is_durable_admin_policy_state() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
let record =
|
|
coordinator.suspend_tenant(TenantId::from("tenant"), UserId::from("admin-user"));
|
|
|
|
assert_eq!(record.tenant, TenantId::from("tenant"));
|
|
assert_eq!(record.name, "tenant:suspended");
|
|
assert!(coordinator.tenant_suspended(&TenantId::from("tenant")));
|
|
assert!(coordinator
|
|
.ensure_tenant_active(&TenantId::from("tenant"))
|
|
.unwrap_err()
|
|
.to_string()
|
|
.contains("suspended"));
|
|
|
|
coordinator.persist(&mut store);
|
|
let restarted = Coordinator::boot(&store, 2);
|
|
assert!(restarted.tenant_suspended(&TenantId::from("tenant")));
|
|
}
|
|
|
|
#[test]
|
|
fn account_policy_state_summarizes_private_admin_records_safely() {
|
|
let tenant = TenantId::from("tenant");
|
|
let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
|
|
|
|
let active = coordinator.account_policy_state(&tenant);
|
|
assert_eq!(active.account_status, "active");
|
|
assert!(active.sanitized_reason.is_none());
|
|
assert!(active.next_actions.is_empty());
|
|
|
|
coordinator.upsert_service_policy_record(
|
|
tenant.clone(),
|
|
"tenant:manual_review",
|
|
Digest::from_parts([b"manual-review".as_slice()]),
|
|
);
|
|
let manual_review = coordinator.account_policy_state(&tenant);
|
|
assert_eq!(manual_review.account_status, "manual_review");
|
|
assert!(manual_review.manual_review);
|
|
assert_eq!(
|
|
manual_review.sanitized_reason.as_deref(),
|
|
Some("account or tenant is pending hosted review")
|
|
);
|
|
|
|
coordinator.upsert_service_policy_record(
|
|
tenant.clone(),
|
|
"tenant:disabled",
|
|
Digest::from_parts([b"disabled".as_slice()]),
|
|
);
|
|
let disabled = coordinator.account_policy_state(&tenant);
|
|
assert_eq!(disabled.account_status, "disabled");
|
|
assert!(disabled.disabled);
|
|
assert!(disabled.manual_review);
|
|
assert_eq!(
|
|
disabled.sanitized_reason.as_deref(),
|
|
Some("account or tenant is disabled by hosted policy")
|
|
);
|
|
|
|
coordinator.upsert_service_policy_record(
|
|
tenant.clone(),
|
|
"tenant:deleted",
|
|
Digest::from_parts([b"deleted".as_slice()]),
|
|
);
|
|
let deleted = coordinator.account_policy_state(&tenant);
|
|
assert_eq!(deleted.account_status, "deleted");
|
|
assert!(deleted.deleted);
|
|
assert!(deleted.disabled);
|
|
assert!(deleted.manual_review);
|
|
assert_eq!(
|
|
deleted.sanitized_reason.as_deref(),
|
|
Some("account or tenant is no longer active")
|
|
);
|
|
assert!(deleted
|
|
.next_actions
|
|
.iter()
|
|
.any(|action| action.contains("hosted operator")));
|
|
}
|
|
|
|
#[test]
|
|
fn agent_public_keys_are_project_user_scoped_and_restart_durable() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.upsert_tenant(TenantId::from("tenant"));
|
|
coordinator.upsert_user(
|
|
TenantId::from("tenant"),
|
|
UserId::from("user"),
|
|
CredentialKind::CliDeviceSession,
|
|
);
|
|
coordinator.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
|
|
let registered = coordinator.register_agent_public_key(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
AgentId::from("agent-ci"),
|
|
"agent-key-v1",
|
|
);
|
|
assert_eq!(registered.version, 1);
|
|
assert!(!registered.revoked);
|
|
assert!(!registered.human_account_creation_privilege);
|
|
assert_eq!(registered.scopes, vec!["project:read", "project:run"]);
|
|
|
|
let rotated = coordinator.register_agent_public_key(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
AgentId::from("agent-ci"),
|
|
"agent-key-v2",
|
|
);
|
|
assert_eq!(rotated.version, 2);
|
|
assert_eq!(rotated.public_key, "agent-key-v2");
|
|
|
|
coordinator.persist(&mut store);
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
let context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
let listed = restarted.list_agent_public_keys(&context);
|
|
assert_eq!(listed.len(), 1);
|
|
assert_eq!(listed[0].agent, AgentId::from("agent-ci"));
|
|
assert_eq!(listed[0].version, 2);
|
|
|
|
let foreign_user_context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("other-user")),
|
|
};
|
|
assert!(restarted
|
|
.list_agent_public_keys(&foreign_user_context)
|
|
.is_empty());
|
|
|
|
let foreign_project_context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("other-project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
assert!(restarted
|
|
.list_agent_public_keys(&foreign_project_context)
|
|
.is_empty());
|
|
|
|
let revoked = restarted
|
|
.revoke_agent_public_key(&context, &AgentId::from("agent-ci"))
|
|
.unwrap();
|
|
assert!(revoked.revoked);
|
|
assert!(!restarted
|
|
.durable
|
|
.credentials
|
|
.contains_key("agent:tenant:project:agent-ci"));
|
|
}
|
|
|
|
#[test]
|
|
fn node_cannot_claim_process_outside_authorized_scope() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.enroll_node(
|
|
TenantId::from("tenant-a"),
|
|
ProjectId::from("project-a"),
|
|
NodeId::from("node-a"),
|
|
"public-key",
|
|
"node",
|
|
);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant-b"),
|
|
ProjectId::from("project-b"),
|
|
ProcessId::from("process-b"),
|
|
);
|
|
|
|
let error = coordinator
|
|
.authorize_node_for_process(
|
|
&NodeId::from("node-a"),
|
|
&TenantId::from("tenant-b"),
|
|
&ProjectId::from("project-b"),
|
|
&ProcessId::from("process-b"),
|
|
)
|
|
.unwrap_err();
|
|
|
|
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
|
|
}
|
|
|
|
#[test]
|
|
fn debug_attach_requires_explicit_project_permission() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
let context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
|
|
let denied = coordinator.authorize_debug_attach(&context, &ProcessId::from("process"));
|
|
coordinator.grant_project_debug(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
);
|
|
let allowed = coordinator.authorize_debug_attach(&context, &ProcessId::from("process"));
|
|
|
|
assert!(!denied.allowed);
|
|
assert!(denied.reason.contains("explicit project permission"));
|
|
assert!(allowed.allowed);
|
|
}
|
|
}
|