1086 lines
35 KiB
Rust
1086 lines
35 KiB
Rust
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use disasmer_core::{
|
|
Actor, AgentId, AuthContext, Authorization, CredentialKind, Digest, EnrollmentError,
|
|
EnrollmentGrant, NodeCredential, NodeId, ProcessId, ProjectId, SourceProviderKind, TenantId,
|
|
UserId,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use thiserror::Error;
|
|
|
|
pub mod postgres_store;
|
|
pub mod service;
|
|
pub use postgres_store::{
|
|
PostgresDurableStore, PostgresStoreError, PostgresTable, POSTGRES_DURABLE_TABLES,
|
|
};
|
|
pub use service::{
|
|
CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
|
|
DebugAuditEvent, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
|
|
TaskCancellationTarget, TaskCompletionEvent, TaskTerminalState,
|
|
};
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct TenantRecord {
|
|
pub id: TenantId,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct UserRecord {
|
|
pub id: UserId,
|
|
pub tenant: TenantId,
|
|
pub credential_kind: CredentialKind,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct ProjectRecord {
|
|
pub id: ProjectId,
|
|
pub tenant: TenantId,
|
|
pub name: String,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct NodeIdentityRecord {
|
|
pub id: NodeId,
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub public_key: String,
|
|
pub enrollment_scope: String,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct CredentialRecord {
|
|
pub subject: String,
|
|
pub tenant: TenantId,
|
|
pub project: Option<ProjectId>,
|
|
pub kind: CredentialKind,
|
|
pub public_key_fingerprint: Option<Digest>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct SourceProviderConfigRecord {
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub provider: SourceProviderKind,
|
|
pub manifest_digest: Digest,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct ServicePolicyRecord {
|
|
pub tenant: TenantId,
|
|
pub name: String,
|
|
pub digest: Digest,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct ProjectPermissionRecord {
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub user: UserId,
|
|
pub can_debug: bool,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct AgentPublicKeyRecord {
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub user: UserId,
|
|
pub agent: AgentId,
|
|
pub public_key: String,
|
|
pub public_key_fingerprint: Digest,
|
|
pub version: u64,
|
|
pub revoked: bool,
|
|
pub scopes: Vec<String>,
|
|
pub human_account_creation_privilege: bool,
|
|
pub browser_interaction_required_each_run: bool,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
|
pub struct DurableState {
|
|
pub tenants: BTreeMap<TenantId, TenantRecord>,
|
|
pub users: BTreeMap<UserId, UserRecord>,
|
|
pub projects: BTreeMap<ProjectId, ProjectRecord>,
|
|
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
|
|
pub credentials: BTreeMap<String, CredentialRecord>,
|
|
pub source_provider_configs:
|
|
BTreeMap<(TenantId, ProjectId, String), SourceProviderConfigRecord>,
|
|
pub service_policy_records: BTreeMap<(TenantId, String), ServicePolicyRecord>,
|
|
pub project_permissions: BTreeMap<(TenantId, ProjectId, UserId), ProjectPermissionRecord>,
|
|
pub agent_public_keys: BTreeMap<(TenantId, ProjectId, AgentId), AgentPublicKeyRecord>,
|
|
}
|
|
|
|
pub trait DurableStore {
|
|
fn load(&self) -> DurableState;
|
|
fn save(&mut self, state: DurableState);
|
|
}
|
|
|
|
pub trait FallibleDurableStore {
|
|
type Error;
|
|
|
|
fn load_state(&mut self) -> Result<DurableState, Self::Error>;
|
|
fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error>;
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
pub struct InMemoryDurableStore {
|
|
state: DurableState,
|
|
}
|
|
|
|
impl DurableStore for InMemoryDurableStore {
|
|
fn load(&self) -> DurableState {
|
|
self.state.clone()
|
|
}
|
|
|
|
fn save(&mut self, state: DurableState) {
|
|
self.state = state;
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct ActiveProcess {
|
|
pub id: ProcessId,
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub connected_nodes: BTreeSet<NodeId>,
|
|
pub coordinator_epoch: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct Coordinator {
|
|
durable: DurableState,
|
|
active_processes: BTreeMap<ProcessId, ActiveProcess>,
|
|
coordinator_epoch: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Error, PartialEq, Eq)]
|
|
pub enum CoordinatorError {
|
|
#[error("node identity is not enrolled")]
|
|
UnknownNode,
|
|
#[error("node enrollment failed: {0:?}")]
|
|
Enrollment(EnrollmentError),
|
|
#[error("stale virtual process state from coordinator epoch {stale_epoch}; current epoch is {current_epoch}")]
|
|
StaleProcessEpoch {
|
|
stale_epoch: u64,
|
|
current_epoch: u64,
|
|
},
|
|
#[error("unauthorized coordinator action: {0}")]
|
|
Unauthorized(String),
|
|
}
|
|
|
|
impl Coordinator {
|
|
pub fn boot(store: &impl DurableStore, coordinator_epoch: u64) -> Self {
|
|
Self {
|
|
durable: store.load(),
|
|
active_processes: BTreeMap::new(),
|
|
coordinator_epoch,
|
|
}
|
|
}
|
|
|
|
pub fn try_boot<S: FallibleDurableStore>(
|
|
store: &mut S,
|
|
coordinator_epoch: u64,
|
|
) -> Result<Self, S::Error> {
|
|
Ok(Self {
|
|
durable: store.load_state()?,
|
|
active_processes: BTreeMap::new(),
|
|
coordinator_epoch,
|
|
})
|
|
}
|
|
|
|
pub fn persist(&self, store: &mut impl DurableStore) {
|
|
store.save(self.durable.clone());
|
|
}
|
|
|
|
pub fn try_persist<S: FallibleDurableStore>(&self, store: &mut S) -> Result<(), S::Error> {
|
|
store.save_state(&self.durable)
|
|
}
|
|
|
|
pub fn coordinator_epoch(&self) -> u64 {
|
|
self.coordinator_epoch
|
|
}
|
|
|
|
pub fn upsert_tenant(&mut self, id: TenantId) {
|
|
self.durable.tenants.insert(id.clone(), TenantRecord { id });
|
|
}
|
|
|
|
pub fn upsert_user(&mut self, tenant: TenantId, id: UserId, credential_kind: CredentialKind) {
|
|
self.durable.users.insert(
|
|
id.clone(),
|
|
UserRecord {
|
|
id,
|
|
tenant,
|
|
credential_kind,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn upsert_project(&mut self, tenant: TenantId, id: ProjectId, name: impl Into<String>) {
|
|
self.durable.projects.insert(
|
|
id.clone(),
|
|
ProjectRecord {
|
|
id,
|
|
tenant,
|
|
name: name.into(),
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn enroll_node(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
node: NodeId,
|
|
public_key: impl Into<String>,
|
|
enrollment_scope: impl Into<String>,
|
|
) {
|
|
self.durable.node_identities.insert(
|
|
node.clone(),
|
|
NodeIdentityRecord {
|
|
id: node,
|
|
tenant,
|
|
project,
|
|
public_key: public_key.into(),
|
|
enrollment_scope: enrollment_scope.into(),
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn create_node_enrollment_grant(
|
|
&self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
grant_id: impl Into<String>,
|
|
scope: impl Into<String>,
|
|
expires_at_epoch_seconds: u64,
|
|
) -> EnrollmentGrant {
|
|
EnrollmentGrant {
|
|
tenant,
|
|
project,
|
|
grant_id: grant_id.into(),
|
|
scope: scope.into(),
|
|
expires_at_epoch_seconds,
|
|
consumed: false,
|
|
}
|
|
}
|
|
|
|
pub fn exchange_node_enrollment_grant(
|
|
&mut self,
|
|
grant: &mut EnrollmentGrant,
|
|
node: NodeId,
|
|
public_key: &str,
|
|
requested_scope: &str,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<NodeCredential, CoordinatorError> {
|
|
let credential = grant
|
|
.exchange_for_node_identity(
|
|
node.clone(),
|
|
public_key,
|
|
requested_scope,
|
|
now_epoch_seconds,
|
|
)
|
|
.map_err(CoordinatorError::Enrollment)?;
|
|
self.enroll_node(
|
|
credential.tenant.clone(),
|
|
credential.project.clone(),
|
|
node.clone(),
|
|
public_key,
|
|
credential.scope.clone(),
|
|
);
|
|
self.durable.credentials.insert(
|
|
format!("node:{node}"),
|
|
CredentialRecord {
|
|
subject: format!("node:{node}"),
|
|
tenant: credential.tenant.clone(),
|
|
project: Some(credential.project.clone()),
|
|
kind: credential.credential_kind.clone(),
|
|
public_key_fingerprint: Some(credential.public_key_fingerprint.clone()),
|
|
},
|
|
);
|
|
Ok(credential)
|
|
}
|
|
|
|
pub fn upsert_source_provider_config(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
provider: SourceProviderKind,
|
|
manifest_digest: Digest,
|
|
) {
|
|
let provider_key = format!("{provider:?}");
|
|
self.durable.source_provider_configs.insert(
|
|
(tenant.clone(), project.clone(), provider_key),
|
|
SourceProviderConfigRecord {
|
|
tenant,
|
|
project,
|
|
provider,
|
|
manifest_digest,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn upsert_service_policy_record(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
name: impl Into<String>,
|
|
digest: Digest,
|
|
) {
|
|
let name = name.into();
|
|
self.durable.service_policy_records.insert(
|
|
(tenant.clone(), name.clone()),
|
|
ServicePolicyRecord {
|
|
tenant,
|
|
name,
|
|
digest,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn suspend_tenant(&mut self, tenant: TenantId, actor: UserId) -> ServicePolicyRecord {
|
|
self.upsert_tenant(tenant.clone());
|
|
let name = "tenant:suspended".to_owned();
|
|
let digest = Digest::from_parts([
|
|
b"tenant-suspension:v1".as_slice(),
|
|
tenant.as_str().as_bytes(),
|
|
actor.as_str().as_bytes(),
|
|
]);
|
|
self.upsert_service_policy_record(tenant.clone(), name.clone(), digest);
|
|
self.service_policy_record(&tenant, &name)
|
|
.expect("tenant suspension record was just inserted")
|
|
.clone()
|
|
}
|
|
|
|
pub fn tenant_suspended(&self, tenant: &TenantId) -> bool {
|
|
self.service_policy_record(tenant, "tenant:suspended")
|
|
.is_some()
|
|
}
|
|
|
|
pub fn ensure_tenant_active(&self, tenant: &TenantId) -> Result<(), CoordinatorError> {
|
|
if self.tenant_suspended(tenant) {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"tenant is suspended by admin controls".to_owned(),
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn grant_project_debug(&mut self, tenant: TenantId, project: ProjectId, user: UserId) {
|
|
self.durable.project_permissions.insert(
|
|
(tenant.clone(), project.clone(), user.clone()),
|
|
ProjectPermissionRecord {
|
|
tenant,
|
|
project,
|
|
user,
|
|
can_debug: true,
|
|
},
|
|
);
|
|
}
|
|
|
|
pub fn register_agent_public_key(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
user: UserId,
|
|
agent: AgentId,
|
|
public_key: impl Into<String>,
|
|
) -> AgentPublicKeyRecord {
|
|
let key = (tenant.clone(), project.clone(), agent.clone());
|
|
let version = self
|
|
.durable
|
|
.agent_public_keys
|
|
.get(&key)
|
|
.map_or(1, |record| record.version.saturating_add(1));
|
|
let public_key = public_key.into();
|
|
let public_key_fingerprint = Digest::sha256(&public_key);
|
|
let record = AgentPublicKeyRecord {
|
|
tenant: tenant.clone(),
|
|
project: project.clone(),
|
|
user: user.clone(),
|
|
agent: agent.clone(),
|
|
public_key,
|
|
public_key_fingerprint: public_key_fingerprint.clone(),
|
|
version,
|
|
revoked: false,
|
|
scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
|
|
human_account_creation_privilege: false,
|
|
browser_interaction_required_each_run: false,
|
|
};
|
|
self.durable.agent_public_keys.insert(key, record.clone());
|
|
let subject = format!("agent:{tenant}:{project}:{agent}");
|
|
self.durable.credentials.insert(
|
|
subject.clone(),
|
|
CredentialRecord {
|
|
subject,
|
|
tenant,
|
|
project: Some(project),
|
|
kind: CredentialKind::PublicKey,
|
|
public_key_fingerprint: Some(public_key_fingerprint),
|
|
},
|
|
);
|
|
record
|
|
}
|
|
|
|
pub fn list_agent_public_keys(&self, context: &AuthContext) -> Vec<AgentPublicKeyRecord> {
|
|
self.durable
|
|
.agent_public_keys
|
|
.values()
|
|
.filter(|record| record.tenant == context.tenant && record.project == context.project)
|
|
.filter(|record| match &context.actor {
|
|
Actor::User(user) => &record.user == user,
|
|
Actor::Agent(agent) => &record.agent == agent,
|
|
Actor::Node(_) | Actor::Task(_) => false,
|
|
})
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
pub fn revoke_agent_public_key(
|
|
&mut self,
|
|
context: &AuthContext,
|
|
agent: &AgentId,
|
|
) -> Result<AgentPublicKeyRecord, CoordinatorError> {
|
|
let key = (
|
|
context.tenant.clone(),
|
|
context.project.clone(),
|
|
agent.clone(),
|
|
);
|
|
let record = self
|
|
.durable
|
|
.agent_public_keys
|
|
.get_mut(&key)
|
|
.ok_or_else(|| {
|
|
CoordinatorError::Unauthorized(
|
|
"agent public key is not registered for this project".to_owned(),
|
|
)
|
|
})?;
|
|
match &context.actor {
|
|
Actor::User(user) if &record.user == user => {}
|
|
Actor::User(_) => {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"agent public key is outside the signed-in user scope".to_owned(),
|
|
));
|
|
}
|
|
_ => {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"agent public-key revocation requires a user identity".to_owned(),
|
|
));
|
|
}
|
|
}
|
|
record.revoked = true;
|
|
let subject = format!(
|
|
"agent:{}:{}:{}",
|
|
context.tenant, context.project, record.agent
|
|
);
|
|
self.durable.credentials.remove(&subject);
|
|
Ok(record.clone())
|
|
}
|
|
|
|
pub fn authorize_agent_project_run(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
agent: &AgentId,
|
|
public_key_fingerprint: &Digest,
|
|
) -> Result<AgentPublicKeyRecord, CoordinatorError> {
|
|
let record = self
|
|
.durable
|
|
.agent_public_keys
|
|
.get(&(tenant.clone(), project.clone(), agent.clone()))
|
|
.ok_or_else(|| {
|
|
CoordinatorError::Unauthorized(
|
|
"agent public key is not registered for this tenant/project".to_owned(),
|
|
)
|
|
})?;
|
|
if record.revoked {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"agent public key has been revoked".to_owned(),
|
|
));
|
|
}
|
|
if &record.public_key_fingerprint != public_key_fingerprint {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"agent public key fingerprint does not match the registered key".to_owned(),
|
|
));
|
|
}
|
|
if !record.scopes.iter().any(|scope| scope == "project:run") {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"agent public key is not scoped for project runs".to_owned(),
|
|
));
|
|
}
|
|
Ok(record.clone())
|
|
}
|
|
|
|
pub fn start_process(
|
|
&mut self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
id: ProcessId,
|
|
) -> ActiveProcess {
|
|
let process = ActiveProcess {
|
|
id: id.clone(),
|
|
tenant,
|
|
project,
|
|
connected_nodes: BTreeSet::new(),
|
|
coordinator_epoch: self.coordinator_epoch,
|
|
};
|
|
self.active_processes.insert(id, process.clone());
|
|
process
|
|
}
|
|
|
|
pub fn authorize_node_for_process(
|
|
&self,
|
|
node: &NodeId,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
process: &ProcessId,
|
|
) -> Result<(), CoordinatorError> {
|
|
let identity = self
|
|
.durable
|
|
.node_identities
|
|
.get(node)
|
|
.ok_or(CoordinatorError::UnknownNode)?;
|
|
if &identity.tenant != tenant || &identity.project != project {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node identity is outside the requested tenant/project scope".to_owned(),
|
|
));
|
|
}
|
|
let Some(active) = self.active_processes.get(process) else {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"virtual process is not active in coordinator memory".to_owned(),
|
|
));
|
|
};
|
|
if &active.tenant != tenant || &active.project != project {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node cannot claim tasks or publish artifacts outside its process scope".to_owned(),
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn reconnect_node(
|
|
&mut self,
|
|
node: &NodeId,
|
|
process: Option<(&ProcessId, u64)>,
|
|
) -> Result<(), CoordinatorError> {
|
|
if !self.durable.node_identities.contains_key(node) {
|
|
return Err(CoordinatorError::UnknownNode);
|
|
}
|
|
|
|
if let Some((process_id, stale_epoch)) = process {
|
|
if stale_epoch != self.coordinator_epoch {
|
|
return Err(CoordinatorError::StaleProcessEpoch {
|
|
stale_epoch,
|
|
current_epoch: self.coordinator_epoch,
|
|
});
|
|
}
|
|
if let Some(active) = self.active_processes.get_mut(process_id) {
|
|
active.connected_nodes.insert(node.clone());
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn revoke_node_credential(
|
|
&mut self,
|
|
context: &AuthContext,
|
|
node: &NodeId,
|
|
) -> Result<NodeIdentityRecord, CoordinatorError> {
|
|
let identity = self
|
|
.durable
|
|
.node_identities
|
|
.get(node)
|
|
.ok_or(CoordinatorError::UnknownNode)?
|
|
.clone();
|
|
if identity.tenant != context.tenant || identity.project != context.project {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node credential is outside the signed-in tenant/project scope".to_owned(),
|
|
));
|
|
}
|
|
if !matches!(context.actor, Actor::User(_)) {
|
|
return Err(CoordinatorError::Unauthorized(
|
|
"node credential revocation requires a user identity".to_owned(),
|
|
));
|
|
}
|
|
self.durable.node_identities.remove(node);
|
|
self.durable.credentials.remove(&format!("node:{node}"));
|
|
for active in self.active_processes.values_mut() {
|
|
active.connected_nodes.remove(node);
|
|
}
|
|
Ok(identity)
|
|
}
|
|
|
|
pub fn list_projects(&self, context: &AuthContext) -> Vec<ProjectRecord> {
|
|
self.durable
|
|
.projects
|
|
.values()
|
|
.filter(|project| project.tenant == context.tenant)
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
pub fn authorize_debug_attach(
|
|
&self,
|
|
context: &AuthContext,
|
|
process: &ProcessId,
|
|
) -> Authorization {
|
|
let Some(active) = self.active_processes.get(process) else {
|
|
return Authorization::deny("virtual process is not active");
|
|
};
|
|
if active.tenant != context.tenant || active.project != context.project {
|
|
return Authorization::deny("tenant or project mismatch");
|
|
}
|
|
let Actor::User(user) = &context.actor else {
|
|
return Authorization::deny("debug attach requires a user identity");
|
|
};
|
|
let permission = self.durable.project_permissions.get(&(
|
|
active.tenant.clone(),
|
|
active.project.clone(),
|
|
user.clone(),
|
|
));
|
|
if !permission.is_some_and(|permission| permission.can_debug) {
|
|
return Authorization::deny("debug attach requires explicit project permission");
|
|
}
|
|
Authorization::allow("debug attach authorized for project")
|
|
}
|
|
|
|
pub fn project(&self, id: &ProjectId) -> Option<&ProjectRecord> {
|
|
self.durable.projects.get(id)
|
|
}
|
|
|
|
pub fn active_process(&self, id: &ProcessId) -> Option<&ActiveProcess> {
|
|
self.active_processes.get(id)
|
|
}
|
|
|
|
pub fn active_process_for_project(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
) -> Option<&ActiveProcess> {
|
|
self.active_processes
|
|
.values()
|
|
.find(|active| &active.tenant == tenant && &active.project == project)
|
|
}
|
|
|
|
pub fn active_process_count(&self) -> usize {
|
|
self.active_processes.len()
|
|
}
|
|
|
|
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
|
|
self.durable.node_identities.get(id)
|
|
}
|
|
|
|
pub fn source_provider_config(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
provider: &str,
|
|
) -> Option<&SourceProviderConfigRecord> {
|
|
self.durable.source_provider_configs.get(&(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
provider.to_owned(),
|
|
))
|
|
}
|
|
|
|
pub fn service_policy_record(
|
|
&self,
|
|
tenant: &TenantId,
|
|
name: &str,
|
|
) -> Option<&ServicePolicyRecord> {
|
|
self.durable
|
|
.service_policy_records
|
|
.get(&(tenant.clone(), name.to_owned()))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn coordinator_restart_preserves_project_but_not_live_processes() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut first = Coordinator::boot(&store, 1);
|
|
first.upsert_tenant(TenantId::from("tenant"));
|
|
first.upsert_user(
|
|
TenantId::from("tenant"),
|
|
UserId::from("user"),
|
|
CredentialKind::CliDeviceSession,
|
|
);
|
|
first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
first.upsert_source_provider_config(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
SourceProviderKind::Git,
|
|
Digest::sha256("git-manifest"),
|
|
);
|
|
first.upsert_service_policy_record(
|
|
TenantId::from("tenant"),
|
|
"community tier",
|
|
Digest::sha256("policy"),
|
|
);
|
|
let mut grant = first.create_node_enrollment_grant(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
"grant",
|
|
"node:attach",
|
|
100,
|
|
);
|
|
first
|
|
.exchange_node_enrollment_grant(
|
|
&mut grant,
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
99,
|
|
)
|
|
.unwrap();
|
|
first.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
first.persist(&mut store);
|
|
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
|
|
assert!(restarted
|
|
.durable
|
|
.tenants
|
|
.contains_key(&TenantId::from("tenant")));
|
|
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
|
|
assert!(restarted.project(&ProjectId::from("project")).is_some());
|
|
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
|
|
assert_eq!(
|
|
restarted
|
|
.durable
|
|
.credentials
|
|
.get("node:node")
|
|
.map(|credential| &credential.kind),
|
|
Some(&CredentialKind::NodeCredential)
|
|
);
|
|
assert!(restarted
|
|
.source_provider_config(
|
|
&TenantId::from("tenant"),
|
|
&ProjectId::from("project"),
|
|
"Git"
|
|
)
|
|
.is_some());
|
|
assert!(restarted
|
|
.service_policy_record(&TenantId::from("tenant"), "community tier")
|
|
.is_some());
|
|
assert_eq!(restarted.active_process_count(), 0);
|
|
|
|
let process = ProcessId::from("process-rerun");
|
|
let rerun = restarted.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
process.clone(),
|
|
);
|
|
assert_eq!(rerun.coordinator_epoch, 2);
|
|
restarted
|
|
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
|
|
.unwrap();
|
|
assert!(restarted
|
|
.active_process(&process)
|
|
.unwrap()
|
|
.connected_nodes
|
|
.contains(&NodeId::from("node")));
|
|
}
|
|
|
|
#[test]
|
|
fn node_reconnect_rejects_stale_process_epoch_after_restart() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut first = Coordinator::boot(&store, 1);
|
|
first.upsert_tenant(TenantId::from("tenant"));
|
|
first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
first.enroll_node(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node",
|
|
);
|
|
first.persist(&mut store);
|
|
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
restarted
|
|
.reconnect_node(&NodeId::from("node"), None)
|
|
.unwrap();
|
|
|
|
let error = restarted
|
|
.reconnect_node(
|
|
&NodeId::from("node"),
|
|
Some((&ProcessId::from("process"), 1)),
|
|
)
|
|
.unwrap_err();
|
|
|
|
assert!(matches!(error, CoordinatorError::StaleProcessEpoch { .. }));
|
|
}
|
|
|
|
#[test]
|
|
fn node_enrollment_grant_becomes_persistent_node_identity() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
let mut grant = coordinator.create_node_enrollment_grant(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
"grant",
|
|
"node:attach",
|
|
100,
|
|
);
|
|
|
|
let credential = coordinator
|
|
.exchange_node_enrollment_grant(
|
|
&mut grant,
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
99,
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
|
|
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn node_credential_revocation_is_project_scoped_and_removes_identity() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.enroll_node(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
NodeId::from("node"),
|
|
"public-key",
|
|
"node:attach",
|
|
);
|
|
coordinator.durable.credentials.insert(
|
|
"node:node".to_owned(),
|
|
CredentialRecord {
|
|
subject: "node:node".to_owned(),
|
|
tenant: TenantId::from("tenant"),
|
|
project: Some(ProjectId::from("project")),
|
|
kind: CredentialKind::NodeCredential,
|
|
public_key_fingerprint: Some(Digest::sha256("public-key")),
|
|
},
|
|
);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
coordinator
|
|
.reconnect_node(
|
|
&NodeId::from("node"),
|
|
Some((&ProcessId::from("process"), 1)),
|
|
)
|
|
.unwrap();
|
|
|
|
let foreign = coordinator
|
|
.revoke_node_credential(
|
|
&AuthContext {
|
|
tenant: TenantId::from("other"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
},
|
|
&NodeId::from("node"),
|
|
)
|
|
.unwrap_err();
|
|
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
|
|
|
|
let revoked = coordinator
|
|
.revoke_node_credential(
|
|
&AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
},
|
|
&NodeId::from("node"),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(revoked.id, NodeId::from("node"));
|
|
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
|
|
assert!(!coordinator.durable.credentials.contains_key("node:node"));
|
|
assert!(!coordinator
|
|
.active_process(&ProcessId::from("process"))
|
|
.unwrap()
|
|
.connected_nodes
|
|
.contains(&NodeId::from("node")));
|
|
}
|
|
|
|
#[test]
|
|
fn project_listing_is_filtered_by_tenant() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.upsert_project(
|
|
TenantId::from("tenant-a"),
|
|
ProjectId::from("project-a"),
|
|
"a",
|
|
);
|
|
coordinator.upsert_project(
|
|
TenantId::from("tenant-b"),
|
|
ProjectId::from("project-b"),
|
|
"b",
|
|
);
|
|
|
|
let projects = coordinator.list_projects(&AuthContext {
|
|
tenant: TenantId::from("tenant-a"),
|
|
project: ProjectId::from("project-a"),
|
|
actor: Actor::User(UserId::from("user-a")),
|
|
});
|
|
|
|
assert_eq!(projects.len(), 1);
|
|
assert_eq!(projects[0].id, ProjectId::from("project-a"));
|
|
}
|
|
|
|
#[test]
|
|
fn tenant_suspension_is_durable_admin_policy_state() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
let record =
|
|
coordinator.suspend_tenant(TenantId::from("tenant"), UserId::from("admin-user"));
|
|
|
|
assert_eq!(record.tenant, TenantId::from("tenant"));
|
|
assert_eq!(record.name, "tenant:suspended");
|
|
assert!(coordinator.tenant_suspended(&TenantId::from("tenant")));
|
|
assert!(coordinator
|
|
.ensure_tenant_active(&TenantId::from("tenant"))
|
|
.unwrap_err()
|
|
.to_string()
|
|
.contains("suspended"));
|
|
|
|
coordinator.persist(&mut store);
|
|
let restarted = Coordinator::boot(&store, 2);
|
|
assert!(restarted.tenant_suspended(&TenantId::from("tenant")));
|
|
}
|
|
|
|
#[test]
|
|
fn agent_public_keys_are_project_user_scoped_and_restart_durable() {
|
|
let mut store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.upsert_tenant(TenantId::from("tenant"));
|
|
coordinator.upsert_user(
|
|
TenantId::from("tenant"),
|
|
UserId::from("user"),
|
|
CredentialKind::CliDeviceSession,
|
|
);
|
|
coordinator.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
|
|
|
|
let registered = coordinator.register_agent_public_key(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
AgentId::from("agent-ci"),
|
|
"agent-key-v1",
|
|
);
|
|
assert_eq!(registered.version, 1);
|
|
assert!(!registered.revoked);
|
|
assert!(!registered.human_account_creation_privilege);
|
|
assert_eq!(registered.scopes, vec!["project:read", "project:run"]);
|
|
|
|
let rotated = coordinator.register_agent_public_key(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
AgentId::from("agent-ci"),
|
|
"agent-key-v2",
|
|
);
|
|
assert_eq!(rotated.version, 2);
|
|
assert_eq!(rotated.public_key, "agent-key-v2");
|
|
|
|
coordinator.persist(&mut store);
|
|
let mut restarted = Coordinator::boot(&store, 2);
|
|
let context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
let listed = restarted.list_agent_public_keys(&context);
|
|
assert_eq!(listed.len(), 1);
|
|
assert_eq!(listed[0].agent, AgentId::from("agent-ci"));
|
|
assert_eq!(listed[0].version, 2);
|
|
|
|
let foreign_user_context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("other-user")),
|
|
};
|
|
assert!(restarted
|
|
.list_agent_public_keys(&foreign_user_context)
|
|
.is_empty());
|
|
|
|
let foreign_project_context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("other-project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
assert!(restarted
|
|
.list_agent_public_keys(&foreign_project_context)
|
|
.is_empty());
|
|
|
|
let revoked = restarted
|
|
.revoke_agent_public_key(&context, &AgentId::from("agent-ci"))
|
|
.unwrap();
|
|
assert!(revoked.revoked);
|
|
assert!(!restarted
|
|
.durable
|
|
.credentials
|
|
.contains_key("agent:tenant:project:agent-ci"));
|
|
}
|
|
|
|
#[test]
|
|
fn node_cannot_claim_process_outside_authorized_scope() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.enroll_node(
|
|
TenantId::from("tenant-a"),
|
|
ProjectId::from("project-a"),
|
|
NodeId::from("node-a"),
|
|
"public-key",
|
|
"node",
|
|
);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant-b"),
|
|
ProjectId::from("project-b"),
|
|
ProcessId::from("process-b"),
|
|
);
|
|
|
|
let error = coordinator
|
|
.authorize_node_for_process(
|
|
&NodeId::from("node-a"),
|
|
&TenantId::from("tenant-b"),
|
|
&ProjectId::from("project-b"),
|
|
&ProcessId::from("process-b"),
|
|
)
|
|
.unwrap_err();
|
|
|
|
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
|
|
}
|
|
|
|
#[test]
|
|
fn debug_attach_requires_explicit_project_permission() {
|
|
let store = InMemoryDurableStore::default();
|
|
let mut coordinator = Coordinator::boot(&store, 1);
|
|
coordinator.start_process(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
ProcessId::from("process"),
|
|
);
|
|
let context = AuthContext {
|
|
tenant: TenantId::from("tenant"),
|
|
project: ProjectId::from("project"),
|
|
actor: Actor::User(UserId::from("user")),
|
|
};
|
|
|
|
let denied = coordinator.authorize_debug_attach(&context, &ProcessId::from("process"));
|
|
coordinator.grant_project_debug(
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
);
|
|
let allowed = coordinator.authorize_debug_attach(&context, &ProcessId::from("process"));
|
|
|
|
assert!(!denied.allowed);
|
|
assert!(denied.reason.contains("explicit project permission"));
|
|
assert!(allowed.allowed);
|
|
}
|
|
}
|