clusterflux-public/crates/clusterflux-coordinator/src/service/routing.rs
Clusterflux release a6ca9e26dd Public release release-be1f654ae8de
Source commit: be1f654ae8de7d5eeb2005b4e1a05bf1ebf3124f

Public tree identity: sha256:ab53679496be4be7d1e02bc00723072774d1a3f87e10cc56558a752f28f6abb1
2026-07-17 05:17:01 +02:00

831 lines
33 KiB
Rust

use super::*;
impl CoordinatorService {
pub fn handle_request(
&mut self,
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
self.pump_main_runtime_commands();
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize coordinator request for authentication: {error}"
))
})?;
let request_payload_digest =
clusterflux_core::signed_request_payload_digest(&request_payload);
match request {
CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong {
epoch: self.coordinator.coordinator_epoch(),
}),
CoordinatorRequest::Authenticated {
session_secret,
request,
} => self.handle_authenticated_request(session_secret, request),
CoordinatorRequest::AuthStatus {
tenant,
project,
actor_user,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let account_state = self.coordinator.account_policy_state(&tenant);
Ok(CoordinatorResponse::AuthStatus {
tenant,
project,
actor,
authenticated: true,
account_status: account_state.account_status,
suspended: account_state.suspended,
disabled: account_state.disabled,
deleted: account_state.deleted,
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
CoordinatorRequest::AdminStatus {
tenant,
actor_user,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
} => self.handle_admin_status(
tenant,
actor_user,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
),
CoordinatorRequest::SuspendTenant {
tenant,
actor_user,
target_tenant,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
} => self.handle_suspend_tenant(
tenant,
actor_user,
target_tenant,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
),
CoordinatorRequest::CreateProject {
tenant,
actor_user,
project,
name,
} => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let project = ProjectId::new(project);
self.coordinator.ensure_tenant_active(&tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the signed-in tenant scope".to_owned(),
)
.into());
}
}
if self.coordinator.project(&project).is_none() {
self.quota.ensure_project_admission(
&tenant,
self.coordinator.project_count_for_tenant(&tenant),
)?;
}
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator.upsert_user(
tenant.clone(),
actor.clone(),
CredentialKind::BrowserSession,
);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), name);
self.coordinator.grant_project_debug(
tenant.clone(),
project.clone(),
actor.clone(),
);
self.persist_durable_state()?;
let project = self
.coordinator
.project(&project)
.expect("project was just created")
.clone();
Ok(CoordinatorResponse::ProjectCreated { project, actor })
}
CoordinatorRequest::SelectProject {
tenant,
actor_user,
project,
} => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let project_id = ProjectId::new(project);
let project = self
.coordinator
.project(&project_id)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"project is not visible to the signed-in user".to_owned(),
)
})?
.clone();
if project.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project is outside the signed-in tenant scope".to_owned(),
)
.into());
}
self.coordinator
.upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
self.persist_durable_state()?;
Ok(CoordinatorResponse::ProjectSelected { project, actor })
}
CoordinatorRequest::ListProjects { tenant, actor_user } => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: ProjectId::from("__project_listing__"),
actor: Actor::User(actor.clone()),
};
self.coordinator
.upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
self.persist_durable_state()?;
Ok(CoordinatorResponse::Projects {
projects: self.coordinator.list_projects(&context),
actor,
})
}
CoordinatorRequest::RegisterAgentPublicKey {
tenant,
project,
user,
agent,
public_key,
}
| CoordinatorRequest::RotateAgentPublicKey {
tenant,
project,
user,
agent,
public_key,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let agent = AgentId::new(agent);
self.coordinator.ensure_tenant_active(&tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the signed-in tenant scope".to_owned(),
)
.into());
}
}
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator.upsert_user(
tenant.clone(),
actor.clone(),
CredentialKind::CliDeviceSession,
);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), "local");
let record = self.coordinator.register_agent_public_key(
tenant,
project,
actor.clone(),
agent,
public_key,
);
self.persist_durable_state()?;
Ok(CoordinatorResponse::AgentPublicKey { record, actor })
}
CoordinatorRequest::ListAgentPublicKeys {
tenant,
project,
user,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let context = clusterflux_core::AuthContext {
tenant,
project,
actor: Actor::User(actor.clone()),
};
Ok(CoordinatorResponse::AgentPublicKeys {
records: self.coordinator.list_agent_public_keys(&context),
actor,
})
}
CoordinatorRequest::RevokeAgentPublicKey {
tenant,
project,
user,
agent,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let agent = AgentId::new(agent);
let context = clusterflux_core::AuthContext {
tenant,
project,
actor: Actor::User(actor.clone()),
};
let record = self.coordinator.revoke_agent_public_key(&context, &agent)?;
self.persist_durable_state()?;
Ok(CoordinatorResponse::AgentPublicKey { record, actor })
}
CoordinatorRequest::AttachNode {
tenant,
project,
node,
public_key,
} => self.handle_attach_node(tenant, project, node, public_key),
CoordinatorRequest::CreateNodeEnrollmentGrant {
tenant,
project,
actor_user,
ttl_seconds,
} => self.handle_create_node_enrollment_grant(tenant, project, actor_user, ttl_seconds),
CoordinatorRequest::ExchangeNodeEnrollmentGrant {
tenant,
project,
node,
public_key,
enrollment_grant,
} => self.handle_exchange_node_enrollment_grant(
tenant,
project,
node,
public_key,
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
CoordinatorRequest::SignedNode {
node,
node_signature,
request,
} => self.handle_signed_node_request(node, node_signature, *request),
CoordinatorRequest::ReportNodeCapabilities { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::ListNodeDescriptors {
tenant,
project,
actor_user,
} => self.handle_list_node_descriptors(tenant, project, actor_user),
CoordinatorRequest::RevokeNodeCredential {
tenant,
project,
actor_user,
node,
} => self.handle_revoke_node_credential(tenant, project, actor_user, node),
CoordinatorRequest::ScheduleTask {
tenant,
project,
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
} => self.handle_schedule_task(
tenant,
project,
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
),
CoordinatorRequest::LaunchTask {
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
} => self.handle_launch_task(
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
Some(&request_payload_digest),
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
),
CoordinatorRequest::LaunchChildTask { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::JoinChildTask { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::PollTaskAssignment { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::RequestRendezvous {
scope,
source,
destination,
direct_connectivity,
failure_reason,
} => self.handle_request_rendezvous(
scope,
source,
destination,
direct_connectivity,
failure_reason,
),
CoordinatorRequest::RequestSourcePreparation {
tenant,
project,
provider,
} => self.handle_request_source_preparation(tenant, project, provider),
CoordinatorRequest::CompleteSourcePreparation { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::StartProcess {
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
process,
restart,
} => self.handle_start_process(
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
Some(&request_payload_digest),
process,
restart,
),
CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::CancelTask {
tenant,
project,
process,
node,
task,
} => self.handle_cancel_task(tenant, project, process, node, task),
CoordinatorRequest::CancelProcess {
tenant,
project,
actor_user,
process,
} => self.handle_cancel_process(tenant, project, actor_user, process),
CoordinatorRequest::AbortProcess {
tenant,
project,
actor_user,
process,
} => self.handle_abort_process(tenant, project, actor_user, process),
CoordinatorRequest::ListProcesses {
tenant,
project,
actor_user,
} => self.handle_list_processes(tenant, project, actor_user),
CoordinatorRequest::QuotaStatus {
tenant,
project,
actor_user,
} => self.handle_quota_status(tenant, project, actor_user),
CoordinatorRequest::PollTaskControl { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::PollArtifactTransfer { .. }
| CoordinatorRequest::UploadArtifactTransferChunk { .. }
| CoordinatorRequest::FailArtifactTransfer { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::RestartTask {
tenant,
project,
actor_user,
process,
task,
replacement_bundle,
} => self.handle_restart_task(
tenant,
project,
actor_user,
process,
task,
replacement_bundle,
),
CoordinatorRequest::ResolveTaskFailure {
tenant,
project,
actor_user,
process,
task,
resolution,
} => self.handle_resolve_task_failure(
tenant, project, actor_user, process, task, resolution,
),
request @ (CoordinatorRequest::DebugAttach { .. }
| CoordinatorRequest::SetDebugBreakpoints { .. }
| CoordinatorRequest::InspectDebugBreakpoints { .. }
| CoordinatorRequest::CreateDebugEpoch { .. }
| CoordinatorRequest::ResumeDebugEpoch { .. }
| CoordinatorRequest::InspectDebugEpoch { .. }) => self.handle_debug_request(request),
CoordinatorRequest::PollDebugCommand { .. }
| CoordinatorRequest::ReportDebugState { .. }
| CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ListTaskEvents {
tenant,
project,
actor_user,
process,
} => self.handle_list_task_events(tenant, project, actor_user, process),
CoordinatorRequest::ListTaskSnapshots {
tenant,
project,
actor_user,
process,
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
CoordinatorRequest::JoinTask {
tenant,
project,
actor_user,
process,
task,
} => self.handle_join_task(tenant, project, actor_user, process, task),
CoordinatorRequest::RenderOperatorPanel {
tenant,
project,
process,
actor_user,
max_download_bytes,
stopped,
} => self.handle_render_operator_panel(
tenant,
project,
actor_user,
process,
max_download_bytes,
stopped,
),
CoordinatorRequest::SubmitPanelEvent {
tenant,
project,
process,
widget_id,
kind,
max_events,
} => self
.handle_submit_panel_event(tenant, project, process, widget_id, kind, max_events),
CoordinatorRequest::CreateArtifactDownloadLink {
tenant,
project,
actor_user,
artifact,
max_bytes,
ttl_seconds,
} => self.handle_create_artifact_download_link(
tenant,
project,
actor_user,
artifact,
max_bytes,
ttl_seconds,
),
CoordinatorRequest::OpenArtifactDownloadStream {
tenant,
project,
actor_user,
artifact,
max_bytes,
token_digest,
chunk_bytes,
} => self.handle_open_artifact_download_stream(
tenant,
project,
actor_user,
artifact,
max_bytes,
token_digest,
chunk_bytes,
),
CoordinatorRequest::RevokeArtifactDownloadLink {
tenant,
project,
actor_user,
artifact,
token_digest,
} => self.handle_revoke_artifact_download_link(
tenant,
project,
actor_user,
artifact,
token_digest,
),
CoordinatorRequest::ExportArtifactToNode {
tenant,
project,
actor_user,
artifact,
receiver_node,
direct_connectivity,
failure_reason,
} => self.handle_export_artifact_to_node(
tenant,
project,
actor_user,
artifact,
receiver_node,
direct_connectivity,
failure_reason,
),
}
}
fn handle_authenticated_request(
&mut self,
session_secret: String,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = if matches!(&request, AuthenticatedCoordinatorRequest::AuthStatus) {
self.coordinator
.authenticate_cli_session_for_status(&session_secret)?
} else {
self.coordinator.authenticate_cli_session(&session_secret)?
};
let authorized = authorize_authenticated_user_operation(&context, &request)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota
.charge_api_call(&context.tenant, &context.project, now_epoch_seconds)?;
let _authorized_operation = authorized.operation;
let actor = authorized.actor;
match request {
AuthenticatedCoordinatorRequest::AuthStatus => {
let account_state = self.coordinator.account_policy_state(&context.tenant);
Ok(CoordinatorResponse::AuthStatus {
tenant: context.tenant,
project: context.project,
actor,
authenticated: true,
account_status: account_state.account_status,
suspended: account_state.suspended,
disabled: account_state.disabled,
deleted: account_state.deleted,
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
AuthenticatedCoordinatorRequest::RevokeCliSession => {
self.coordinator.revoke_cli_session(&session_secret)?;
self.persist_durable_state()?;
Ok(CoordinatorResponse::CliSessionRevoked {
tenant: context.tenant,
project: context.project,
actor,
})
}
AuthenticatedCoordinatorRequest::CreateProject { project, name } => {
let project = ProjectId::new(project);
self.coordinator.ensure_tenant_active(&context.tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != context.tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the authenticated tenant scope".to_owned(),
)
.into());
}
}
if self.coordinator.project(&project).is_none() {
self.quota.ensure_project_admission(
&context.tenant,
self.coordinator.project_count_for_tenant(&context.tenant),
)?;
}
self.coordinator
.upsert_project(context.tenant.clone(), project.clone(), name);
self.coordinator.grant_project_debug(
context.tenant.clone(),
project.clone(),
actor.clone(),
);
self.persist_durable_state()?;
let project = self
.coordinator
.project(&project)
.expect("project was just created")
.clone();
Ok(CoordinatorResponse::ProjectCreated { project, actor })
}
AuthenticatedCoordinatorRequest::SelectProject { project } => {
let project_id = ProjectId::new(project);
let project = self
.coordinator
.project(&project_id)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"project is not visible to the authenticated user".to_owned(),
)
})?
.clone();
if project.tenant != context.tenant {
return Err(CoordinatorError::Unauthorized(
"project is outside the authenticated tenant scope".to_owned(),
)
.into());
}
Ok(CoordinatorResponse::ProjectSelected { project, actor })
}
AuthenticatedCoordinatorRequest::ListProjects => Ok(CoordinatorResponse::Projects {
projects: self.coordinator.list_projects(&context),
actor,
}),
request @ (AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. }
| AuthenticatedCoordinatorRequest::ListAgentPublicKeys
| AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. }
| AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. }) => {
self.handle_authenticated_agent_key_request(&context, &actor, request)
}
AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { ttl_seconds } => self
.handle_create_node_enrollment_grant(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
ttl_seconds,
),
AuthenticatedCoordinatorRequest::ListNodeDescriptors => self
.handle_list_node_descriptors(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
.handle_revoke_node_credential(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
node,
),
AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
.handle_start_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
Some(actor.as_str().to_owned()),
None,
None,
None,
None,
process,
restart,
),
request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
self.handle_authenticated_schedule_task(&context, request)
}
request @ AuthenticatedCoordinatorRequest::LaunchTask { .. } => {
self.handle_authenticated_launch_task(&context, &actor, request)
}
AuthenticatedCoordinatorRequest::CancelProcess { process } => self
.handle_cancel_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::RestartTask {
process,
task,
replacement_bundle,
} => self.handle_restart_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
replacement_bundle,
),
AuthenticatedCoordinatorRequest::ResolveTaskFailure {
process,
task,
resolution,
} => self.handle_resolve_task_failure(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
resolution,
),
request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. }
| AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. }
| AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. }
| AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. }
| AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. }
| AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. }) => self
.handle_authenticated_debug_request(
&context.tenant,
&context.project,
&actor,
request,
),
AuthenticatedCoordinatorRequest::ListTaskEvents { process } => self
.handle_list_task_events(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => self
.handle_list_task_snapshots(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
),
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
artifact,
max_bytes,
ttl_seconds,
} => self.handle_create_artifact_download_link(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
max_bytes,
ttl_seconds,
),
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
artifact,
max_bytes,
token_digest,
chunk_bytes,
} => self.handle_open_artifact_download_stream(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
max_bytes,
token_digest,
chunk_bytes,
),
AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink {
artifact,
token_digest,
} => self.handle_revoke_artifact_download_link(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
token_digest,
),
request @ AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
self.handle_authenticated_artifact_export(&context, &actor, request)
}
}
}
}