use std::collections::BTreeMap; use clusterflux_core::{ AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind, DataPlaneObject, DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements, LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryHandle, TaskBoundaryValue, TaskDefinitionId, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, UserId, VfsPath, }; use serde::{Deserialize, Serialize}; mod responses; pub use responses::*; use crate::{AgentPublicKeyRecord, ProjectRecord}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct TaskReplacementBundle { pub bundle_digest: Digest, pub wasm_module_base64: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub source_snapshot: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum TaskFailureResolution { AcceptFailure, Cancel, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum CoordinatorRequest { Ping, Authenticated { session_secret: String, request: AuthenticatedCoordinatorRequest, }, AuthStatus { tenant: String, project: String, actor_user: String, }, AdminStatus { tenant: String, actor_user: String, admin_proof: Digest, admin_nonce: String, issued_at_epoch_seconds: u64, }, SuspendTenant { tenant: String, actor_user: String, target_tenant: String, admin_proof: Digest, admin_nonce: String, issued_at_epoch_seconds: u64, }, CreateProject { tenant: String, actor_user: String, project: String, name: String, }, SelectProject { tenant: String, actor_user: String, project: String, }, ListProjects { tenant: String, actor_user: String, }, RegisterAgentPublicKey { tenant: String, project: String, user: String, agent: String, public_key: String, }, ListAgentPublicKeys { tenant: String, project: String, user: String, }, RotateAgentPublicKey { tenant: String, project: String, user: String, agent: String, public_key: String, }, RevokeAgentPublicKey { tenant: String, project: String, user: String, agent: String, }, AttachNode { tenant: String, project: String, node: String, public_key: String, }, CreateNodeEnrollmentGrant { tenant: String, project: String, actor_user: String, #[serde(default = "default_node_enrollment_ttl_seconds")] ttl_seconds: u64, }, ExchangeNodeEnrollmentGrant { tenant: String, project: String, node: String, public_key: String, enrollment_grant: String, }, NodeHeartbeat { tenant: String, project: String, node: String, #[serde(default)] node_signature: Option, }, SignedNode { node: String, node_signature: NodeSignedRequest, request: Box, }, ReportNodeCapabilities { tenant: String, project: String, node: String, capabilities: NodeCapabilities, cached_environment_digests: Vec, #[serde(default)] dependency_cache_digests: Vec, source_snapshots: Vec, artifact_locations: Vec, direct_connectivity: bool, online: bool, }, ListNodeDescriptors { tenant: String, project: String, actor_user: String, }, RevokeNodeCredential { tenant: String, project: String, actor_user: String, node: String, }, ScheduleTask { tenant: String, project: String, environment: Option, environment_digest: Option, required_capabilities: Vec, #[serde(default)] dependency_cache: Option, source_snapshot: Option, required_artifacts: Vec, prefer_node: Option, }, LaunchTask { tenant: String, project: String, #[serde(default)] actor_user: Option, #[serde(default)] actor_agent: Option, #[serde(default)] agent_public_key_fingerprint: Option, #[serde(default)] agent_signature: Option, task_spec: TaskSpec, #[serde(default)] wait_for_node: bool, artifact_path: String, wasm_module_base64: String, }, LaunchChildTask { tenant: String, project: String, process: String, node: String, parent_task: String, task_spec: TaskSpec, #[serde(default)] wait_for_node: bool, artifact_path: String, wasm_module_base64: String, }, JoinChildTask { tenant: String, project: String, process: String, node: String, parent_task: String, task: String, }, PollTaskAssignment { tenant: String, project: String, node: String, }, PollArtifactTransfer { tenant: String, project: String, node: String, }, UploadArtifactTransferChunk { tenant: String, project: String, node: String, transfer_id: String, artifact: String, offset: u64, content_base64: String, chunk_digest: Digest, eof: bool, }, FailArtifactTransfer { tenant: String, project: String, node: String, transfer_id: String, artifact: String, message: String, }, RequestRendezvous { scope: DataPlaneScope, source: NodeEndpoint, destination: NodeEndpoint, direct_connectivity: bool, failure_reason: String, }, RequestSourcePreparation { tenant: String, project: String, provider: SourceProviderKind, }, CompleteSourcePreparation { tenant: String, project: String, node: String, provider: SourceProviderKind, source_snapshot: Digest, }, StartProcess { tenant: String, project: String, #[serde(default)] actor_user: Option, #[serde(default)] actor_agent: Option, #[serde(default)] agent_public_key_fingerprint: Option, #[serde(default)] agent_signature: Option, process: String, #[serde(default, skip_serializing_if = "Option::is_none")] launch_attempt: Option, #[serde(default)] restart: bool, }, ReconnectNode { tenant: String, project: String, node: String, process: String, epoch: u64, }, CancelTask { tenant: String, project: String, process: String, node: String, task: String, }, CancelProcess { tenant: String, project: String, actor_user: String, process: String, }, AbortProcess { tenant: String, project: String, actor_user: String, process: String, #[serde(default, skip_serializing_if = "Option::is_none")] launch_attempt: Option, }, ListProcesses { tenant: String, project: String, actor_user: String, }, QuotaStatus { tenant: String, project: String, actor_user: String, }, PollTaskControl { tenant: String, project: String, process: String, node: String, task: String, }, RestartTask { tenant: String, project: String, actor_user: String, process: String, task: String, #[serde(default)] replacement_bundle: Option, }, ResolveTaskFailure { tenant: String, project: String, actor_user: String, process: String, task: String, resolution: TaskFailureResolution, }, DebugAttach { tenant: String, project: String, actor_user: String, process: String, }, SetDebugBreakpoints { tenant: String, project: String, actor_user: String, process: String, #[serde(default)] revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { tenant: String, project: String, actor_user: String, process: String, }, CreateDebugEpoch { tenant: String, project: String, actor_user: String, process: String, stopped_task: String, reason: String, }, ResumeDebugEpoch { tenant: String, project: String, actor_user: String, process: String, epoch: u64, }, InspectDebugEpoch { tenant: String, project: String, actor_user: String, process: String, epoch: u64, }, PollDebugCommand { tenant: String, project: String, process: String, node: String, task: String, }, ReportDebugState { tenant: String, project: String, process: String, node: String, task: String, epoch: u64, state: DebugAcknowledgementState, #[serde(default)] stack_frames: Vec, #[serde(default)] local_values: Vec<(String, String)>, #[serde(default)] task_args: Vec<(String, String)>, #[serde(default)] handles: Vec<(String, String)>, #[serde(default)] command_status: Option, #[serde(default)] recent_output: Vec, #[serde(default)] message: Option, }, ReportDebugProbeHit { tenant: String, project: String, process: String, node: String, task: String, probe_symbol: String, }, ReportTaskLog { tenant: String, project: String, process: String, node: String, task: String, stdout_bytes: u64, stderr_bytes: u64, #[serde(default)] stdout_tail: String, #[serde(default)] stderr_tail: String, stdout_truncated: bool, stderr_truncated: bool, backpressured: bool, }, ReportVfsMetadata { tenant: String, project: String, process: String, node: String, task: String, artifact_path: Option, artifact_digest: Option, artifact_size_bytes: Option, large_bytes_uploaded: bool, }, TaskCompleted { tenant: String, project: String, process: String, node: String, task: String, #[serde(default)] terminal_state: Option, status_code: Option, stdout_bytes: u64, stderr_bytes: u64, #[serde(default)] stdout_tail: String, #[serde(default)] stderr_tail: String, #[serde(default)] stdout_truncated: bool, #[serde(default)] stderr_truncated: bool, artifact_path: Option, artifact_digest: Option, artifact_size_bytes: Option, #[serde(default)] result: Option, }, ListTaskEvents { tenant: String, project: String, actor_user: String, #[serde(default)] process: Option, }, ListTaskSnapshots { tenant: String, project: String, actor_user: String, process: String, }, JoinTask { tenant: String, project: String, actor_user: String, process: String, task: String, }, RenderOperatorPanel { tenant: String, project: String, process: String, actor_user: String, max_download_bytes: u64, stopped: bool, }, SubmitPanelEvent { tenant: String, project: String, process: String, widget_id: String, kind: PanelEventKind, max_events: u64, }, CreateArtifactDownloadLink { tenant: String, project: String, actor_user: String, artifact: String, max_bytes: u64, #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, OpenArtifactDownloadStream { tenant: String, project: String, actor_user: String, artifact: String, max_bytes: u64, token_digest: Digest, chunk_bytes: u64, }, RevokeArtifactDownloadLink { tenant: String, project: String, actor_user: String, artifact: String, token_digest: Digest, }, ExportArtifactToNode { tenant: String, project: String, actor_user: String, artifact: String, receiver_node: String, direct_connectivity: bool, failure_reason: String, }, } impl CoordinatorRequest { pub fn validate_external_identifiers(&self) -> Result<(), String> { validate_coordinator_request(self, "request") } pub fn operation(&self) -> Result { serde_json::to_value(self) .map_err(|err| format!("failed to encode coordinator request operation: {err}")) .map(|value| clusterflux_core::coordinator_payload_operation(&value)) } } fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Result<(), String> { match request { CoordinatorRequest::Ping => Ok(()), CoordinatorRequest::Authenticated { session_secret, request, } => { validate_external_token(session_secret, &format!("{path}.session_secret"), 512)?; validate_authenticated_request(request, &format!("{path}.request")) } CoordinatorRequest::AuthStatus { tenant, project, actor_user, } | CoordinatorRequest::CreateProject { tenant, project, actor_user, .. } | CoordinatorRequest::SelectProject { tenant, project, actor_user, } => { validate_tenant(tenant, &format!("{path}.tenant"))?; validate_project(project, &format!("{path}.project"))?; validate_user(actor_user, &format!("{path}.actor_user")) } CoordinatorRequest::ListProjects { tenant, actor_user } => { validate_tenant(tenant, &format!("{path}.tenant"))?; validate_user(actor_user, &format!("{path}.actor_user")) } CoordinatorRequest::AdminStatus { tenant, actor_user, admin_nonce, .. } => { validate_tenant(tenant, &format!("{path}.tenant"))?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) } CoordinatorRequest::SuspendTenant { tenant, actor_user, target_tenant, admin_nonce, .. } => { validate_tenant(tenant, &format!("{path}.tenant"))?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_tenant(target_tenant, &format!("{path}.target_tenant"))?; validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) } CoordinatorRequest::RegisterAgentPublicKey { tenant, project, user, agent, public_key, } | CoordinatorRequest::RotateAgentPublicKey { tenant, project, user, agent, public_key, } => { validate_tenant_project(tenant, project, path)?; validate_user(user, &format!("{path}.user"))?; validate_agent(agent, &format!("{path}.agent"))?; validate_external_token(public_key, &format!("{path}.public_key"), 1024) } CoordinatorRequest::ListAgentPublicKeys { tenant, project, user, } => { validate_tenant_project(tenant, project, path)?; validate_user(user, &format!("{path}.user")) } CoordinatorRequest::RevokeAgentPublicKey { tenant, project, user, agent, } => { validate_tenant_project(tenant, project, path)?; validate_user(user, &format!("{path}.user"))?; validate_agent(agent, &format!("{path}.agent")) } CoordinatorRequest::AttachNode { tenant, project, node, public_key, } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_external_token(public_key, &format!("{path}.public_key"), 1024) } CoordinatorRequest::CreateNodeEnrollmentGrant { tenant, project, actor_user, .. } | CoordinatorRequest::ListNodeDescriptors { tenant, project, actor_user, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } CoordinatorRequest::ExchangeNodeEnrollmentGrant { tenant, project, node, public_key, enrollment_grant, } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_external_token(public_key, &format!("{path}.public_key"), 1024)?; validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512) } CoordinatorRequest::NodeHeartbeat { tenant, project, node, node_signature, } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; if let Some(signature) = node_signature { validate_node_signature(signature, &format!("{path}.node_signature"))?; } Ok(()) } CoordinatorRequest::SignedNode { node, node_signature, request, } => { validate_node(node, &format!("{path}.node"))?; validate_node_signature(node_signature, &format!("{path}.node_signature"))?; validate_coordinator_request(request, &format!("{path}.request")) } CoordinatorRequest::ReportNodeCapabilities { tenant, project, node, artifact_locations, .. } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_artifact_array(artifact_locations, &format!("{path}.artifact_locations")) } CoordinatorRequest::RevokeNodeCredential { tenant, project, actor_user, node, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_node(node, &format!("{path}.node")) } CoordinatorRequest::ScheduleTask { tenant, project, required_artifacts, prefer_node, .. } => { validate_tenant_project(tenant, project, path)?; validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) } CoordinatorRequest::LaunchTask { tenant, project, actor_user, actor_agent, agent_signature, task_spec, .. } => { validate_tenant_project(tenant, project, path)?; validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; if let Some(signature) = agent_signature { validate_agent_signature(signature, &format!("{path}.agent_signature"))?; } validate_task_spec(task_spec, &format!("{path}.task_spec")) } CoordinatorRequest::LaunchChildTask { tenant, project, process, node, parent_task, task_spec, .. } => { validate_tenant_project(tenant, project, path)?; validate_process(process, &format!("{path}.process"))?; validate_node(node, &format!("{path}.node"))?; validate_task_instance(parent_task, &format!("{path}.parent_task"))?; validate_task_spec(task_spec, &format!("{path}.task_spec")) } CoordinatorRequest::JoinChildTask { tenant, project, process, node, parent_task, task, } => { validate_tenant_project(tenant, project, path)?; validate_process(process, &format!("{path}.process"))?; validate_node(node, &format!("{path}.node"))?; validate_task_instance(parent_task, &format!("{path}.parent_task"))?; validate_task_instance(task, &format!("{path}.task")) } CoordinatorRequest::PollTaskAssignment { tenant, project, node, } | CoordinatorRequest::PollArtifactTransfer { tenant, project, node, } | CoordinatorRequest::CompleteSourcePreparation { tenant, project, node, .. } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node")) } CoordinatorRequest::UploadArtifactTransferChunk { tenant, project, node, transfer_id, artifact, .. } | CoordinatorRequest::FailArtifactTransfer { tenant, project, node, transfer_id, artifact, .. } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_external_token(transfer_id, &format!("{path}.transfer_id"), 256)?; validate_artifact(artifact, &format!("{path}.artifact")) } CoordinatorRequest::RequestRendezvous { scope, source, destination, .. } => { validate_data_plane_scope(scope, &format!("{path}.scope"))?; validate_node_endpoint(source, &format!("{path}.source"))?; validate_node_endpoint(destination, &format!("{path}.destination")) } CoordinatorRequest::RequestSourcePreparation { tenant, project, .. } => validate_tenant_project(tenant, project, path), CoordinatorRequest::StartProcess { tenant, project, actor_user, actor_agent, agent_signature, process, launch_attempt, .. } => { validate_tenant_project(tenant, project, path)?; validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; if let Some(signature) = agent_signature { validate_agent_signature(signature, &format!("{path}.agent_signature"))?; } validate_process(process, &format!("{path}.process"))?; validate_optional_launch_attempt( launch_attempt.as_deref(), &format!("{path}.launch_attempt"), ) } CoordinatorRequest::ReconnectNode { tenant, project, node, process, .. } => { validate_tenant_project(tenant, project, path)?; validate_node(node, &format!("{path}.node"))?; validate_process(process, &format!("{path}.process")) } CoordinatorRequest::CancelTask { tenant, project, process, node, task, } | CoordinatorRequest::PollTaskControl { tenant, project, process, node, task, } | CoordinatorRequest::PollDebugCommand { tenant, project, process, node, task, } | CoordinatorRequest::ReportDebugProbeHit { tenant, project, process, node, task, .. } | CoordinatorRequest::ReportTaskLog { tenant, project, process, node, task, .. } | CoordinatorRequest::ReportVfsMetadata { tenant, project, process, node, task, .. } | CoordinatorRequest::TaskCompleted { tenant, project, process, node, task, .. } | CoordinatorRequest::ReportDebugState { tenant, project, process, node, task, .. } => { validate_tenant_project(tenant, project, path)?; validate_process(process, &format!("{path}.process"))?; validate_node(node, &format!("{path}.node"))?; validate_task_instance(task, &format!("{path}.task")) } CoordinatorRequest::CancelProcess { tenant, project, actor_user, process, } | CoordinatorRequest::DebugAttach { tenant, project, actor_user, process, } | CoordinatorRequest::SetDebugBreakpoints { tenant, project, actor_user, process, .. } | CoordinatorRequest::InspectDebugBreakpoints { tenant, project, actor_user, process, } | CoordinatorRequest::ResumeDebugEpoch { tenant, project, actor_user, process, .. } | CoordinatorRequest::InspectDebugEpoch { tenant, project, actor_user, process, .. } | CoordinatorRequest::ListTaskSnapshots { tenant, project, actor_user, process, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process")) } CoordinatorRequest::AbortProcess { tenant, project, actor_user, process, launch_attempt, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process"))?; validate_optional_launch_attempt( launch_attempt.as_deref(), &format!("{path}.launch_attempt"), ) } CoordinatorRequest::ListProcesses { tenant, project, actor_user, } | CoordinatorRequest::QuotaStatus { tenant, project, actor_user, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } CoordinatorRequest::RestartTask { tenant, project, actor_user, process, task, .. } | CoordinatorRequest::ResolveTaskFailure { tenant, project, actor_user, process, task, .. } | CoordinatorRequest::JoinTask { tenant, project, actor_user, process, task, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process"))?; validate_task_instance(task, &format!("{path}.task")) } CoordinatorRequest::CreateDebugEpoch { tenant, project, actor_user, process, stopped_task, .. } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process"))?; validate_task_instance(stopped_task, &format!("{path}.stopped_task")) } CoordinatorRequest::ListTaskEvents { tenant, project, actor_user, process, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_optional_process(process.as_deref(), &format!("{path}.process")) } CoordinatorRequest::RenderOperatorPanel { tenant, project, process, actor_user, .. } => { validate_tenant_project(tenant, project, path)?; validate_process(process, &format!("{path}.process"))?; validate_user(actor_user, &format!("{path}.actor_user")) } CoordinatorRequest::SubmitPanelEvent { tenant, project, process, widget_id, .. } => { validate_tenant_project(tenant, project, path)?; validate_process(process, &format!("{path}.process"))?; validate_external_token(widget_id, &format!("{path}.widget_id"), 256) } CoordinatorRequest::CreateArtifactDownloadLink { tenant, project, actor_user, artifact, .. } | CoordinatorRequest::OpenArtifactDownloadStream { tenant, project, actor_user, artifact, .. } | CoordinatorRequest::RevokeArtifactDownloadLink { tenant, project, actor_user, artifact, .. } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_artifact(artifact, &format!("{path}.artifact")) } CoordinatorRequest::ExportArtifactToNode { tenant, project, actor_user, artifact, receiver_node, .. } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; validate_artifact(artifact, &format!("{path}.artifact"))?; validate_node(receiver_node, &format!("{path}.receiver_node")) } } } fn validate_authenticated_request( request: &AuthenticatedCoordinatorRequest, path: &str, ) -> Result<(), String> { match request { AuthenticatedCoordinatorRequest::AuthStatus | AuthenticatedCoordinatorRequest::RevokeCliSession | AuthenticatedCoordinatorRequest::ListProjects | AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } | AuthenticatedCoordinatorRequest::ListNodeDescriptors | AuthenticatedCoordinatorRequest::ListProcesses | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), AuthenticatedCoordinatorRequest::CreateProject { project, .. } | AuthenticatedCoordinatorRequest::SelectProject { project } => { validate_project(project, &format!("{path}.project")) } AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { agent, public_key, .. } | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { agent, public_key, .. } => { validate_agent(agent, &format!("{path}.agent"))?; validate_external_token(public_key, &format!("{path}.public_key"), 1024) } AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Ok(()), AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => { validate_agent(agent, &format!("{path}.agent")) } AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => { validate_node(node, &format!("{path}.node")) } AuthenticatedCoordinatorRequest::StartProcess { process, launch_attempt, .. } | AuthenticatedCoordinatorRequest::AbortProcess { process, launch_attempt, } => { validate_process(process, &format!("{path}.process"))?; validate_optional_launch_attempt( launch_attempt.as_deref(), &format!("{path}.launch_attempt"), ) } AuthenticatedCoordinatorRequest::ScheduleTask { required_artifacts, prefer_node, .. } => { validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) } AuthenticatedCoordinatorRequest::LaunchTask { task_spec, .. } => { validate_task_spec(task_spec, &format!("{path}.task_spec")) } AuthenticatedCoordinatorRequest::CancelProcess { process } | AuthenticatedCoordinatorRequest::DebugAttach { process } | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, .. } | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, .. } | AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, .. } | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { validate_process(process, &format!("{path}.process")) } AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { validate_process(process, &format!("{path}.process"))?; validate_task_instance(task, &format!("{path}.task")) } AuthenticatedCoordinatorRequest::CreateDebugEpoch { process, stopped_task, .. } => { validate_process(process, &format!("{path}.process"))?; validate_task_instance(stopped_task, &format!("{path}.stopped_task")) } AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { validate_optional_process(process.as_deref(), &format!("{path}.process")) } AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } => { validate_artifact(artifact, &format!("{path}.artifact")) } AuthenticatedCoordinatorRequest::ExportArtifactToNode { artifact, receiver_node, .. } => { validate_artifact(artifact, &format!("{path}.artifact"))?; validate_node(receiver_node, &format!("{path}.receiver_node")) } } } fn validate_task_spec(task_spec: &TaskSpec, path: &str) -> Result<(), String> { validate_existing_id( &task_spec.tenant, &format!("{path}.tenant"), TenantId::try_new, )?; validate_existing_id( &task_spec.project, &format!("{path}.project"), ProjectId::try_new, )?; validate_existing_id( &task_spec.process, &format!("{path}.process"), ProcessId::try_new, )?; validate_existing_id( &task_spec.task_definition, &format!("{path}.task_definition"), TaskDefinitionId::try_new, )?; validate_existing_id( &task_spec.task_instance, &format!("{path}.task_instance"), TaskInstanceId::try_new, )?; if let Some(environment_id) = &task_spec.environment_id { validate_external_token(environment_id, &format!("{path}.environment_id"), 128)?; } for (index, artifact) in task_spec.required_artifacts.iter().enumerate() { validate_existing_id( artifact, &format!("{path}.required_artifacts[{index}]"), ArtifactId::try_new, )?; } for (argument_index, argument) in task_spec.args.iter().enumerate() { validate_task_boundary_value(argument, &format!("{path}.args[{argument_index}]"))?; } Ok(()) } fn validate_task_boundary_value(value: &TaskBoundaryValue, path: &str) -> Result<(), String> { match value { TaskBoundaryValue::Artifact(artifact) => validate_existing_id( &artifact.id, &format!("{path}.artifact.id"), ArtifactId::try_new, ), TaskBoundaryValue::Structured(structured) => { for (index, handle) in structured.handles.iter().enumerate() { if let TaskBoundaryHandle::Artifact(artifact) = handle { validate_existing_id( &artifact.id, &format!("{path}.handles[{index}].artifact.id"), ArtifactId::try_new, )?; } } Ok(()) } TaskBoundaryValue::SmallJson(_) | TaskBoundaryValue::SourceSnapshot(_) | TaskBoundaryValue::Blob(_) | TaskBoundaryValue::VfsManifest(_) => Ok(()), } } fn validate_data_plane_scope(scope: &DataPlaneScope, path: &str) -> Result<(), String> { validate_existing_id(&scope.tenant, &format!("{path}.tenant"), TenantId::try_new)?; validate_existing_id( &scope.project, &format!("{path}.project"), ProjectId::try_new, )?; validate_existing_id( &scope.process, &format!("{path}.process"), ProcessId::try_new, )?; if let DataPlaneObject::Artifact(artifact) = &scope.object { validate_existing_id( artifact, &format!("{path}.object.artifact"), ArtifactId::try_new, )?; } validate_external_token( &scope.authorization_subject, &format!("{path}.authorization_subject"), 512, ) } fn validate_node_endpoint(endpoint: &NodeEndpoint, path: &str) -> Result<(), String> { validate_existing_id(&endpoint.node, &format!("{path}.node"), NodeId::try_new)?; validate_external_token( &endpoint.advertised_addr, &format!("{path}.advertised_addr"), 512, ) } fn validate_node_signature(signature: &NodeSignedRequest, path: &str) -> Result<(), String> { validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; validate_external_token(&signature.signature, &format!("{path}.signature"), 512) } fn validate_agent_signature(signature: &AgentSignedRequest, path: &str) -> Result<(), String> { validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; validate_external_token(&signature.signature, &format!("{path}.signature"), 512) } fn validate_tenant_project(tenant: &str, project: &str, path: &str) -> Result<(), String> { validate_tenant(tenant, &format!("{path}.tenant"))?; validate_project(project, &format!("{path}.project")) } fn validate_artifact_array(values: &[String], path: &str) -> Result<(), String> { for (index, value) in values.iter().enumerate() { validate_artifact(value, &format!("{path}[{index}]"))?; } Ok(()) } fn validate_optional_user(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_user(value, path)) } fn validate_optional_agent(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_agent(value, path)) } fn validate_optional_node(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_node(value, path)) } fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_process(value, path)) } fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) } fn validate_tenant(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, TenantId::try_new) } fn validate_project(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, ProjectId::try_new) } fn validate_user(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, UserId::try_new) } fn validate_agent(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, AgentId::try_new) } fn validate_node(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, NodeId::try_new) } fn validate_process(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, ProcessId::try_new) } fn validate_task_instance(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, TaskInstanceId::try_new) } fn validate_artifact(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, ArtifactId::try_new) } fn validate_launch_attempt(value: &str, path: &str) -> Result<(), String> { validate_external_id(value, path, LaunchAttemptId::try_new) } fn validate_external_id( value: &str, path: &str, parser: impl FnOnce(String) -> Result, ) -> Result<(), String> where E: std::fmt::Display, { parser(value.to_owned()) .map(|_| ()) .map_err(|error| format!("malformed external identifier {path}: {error}")) } fn validate_existing_id( value: &T, path: &str, parser: impl FnOnce(String) -> Result, ) -> Result<(), String> where T: std::fmt::Display, E: std::fmt::Display, { validate_external_id(&value.to_string(), path, parser) } fn validate_external_token(value: &str, path: &str, max_bytes: usize) -> Result<(), String> { clusterflux_core::validate_opaque_token(value, max_bytes) .map_err(|error| format!("malformed external token {path}: {error}")) } #[cfg(test)] mod external_identifier_tests { use super::*; fn valid_task_spec() -> TaskSpec { TaskSpec { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("process"), task_definition: TaskDefinitionId::from("compile"), task_instance: TaskInstanceId::from("compile-1"), dispatch: clusterflux_core::TaskDispatch::CoordinatorNodeWasm { export: Some("compile".to_owned()), abi: clusterflux_core::WasmExportAbi::TaskV1, }, environment_id: Some("linux-rootless".to_owned()), environment: None, environment_digest: None, required_capabilities: Default::default(), dependency_cache: None, source_snapshot: None, required_artifacts: vec![ArtifactId::from("input-artifact")], args: Vec::new(), vfs_epoch: 1, failure_policy: Default::default(), bundle_digest: None, } } fn validation_error(value: serde_json::Value) -> String { match serde_json::from_value::(value) { Ok(request) => request .validate_external_identifiers() .expect_err("request should contain one malformed external identifier"), Err(error) => error.to_string(), } } #[test] fn nested_authenticated_identifiers_are_validated() { let request = CoordinatorRequest::Authenticated { session_secret: "cli_session_valid".to_owned(), request: AuthenticatedCoordinatorRequest::AbortProcess { process: "bad process".to_owned(), launch_attempt: Some("attempt".to_owned()), }, }; let error = request.validate_external_identifiers().unwrap_err(); assert!(error.contains("request.request.process")); } #[test] fn opaque_secrets_are_bounded_as_tokens_instead_of_object_ids() { let request = CoordinatorRequest::Authenticated { session_secret: "opaque secret/+==".to_owned(), request: AuthenticatedCoordinatorRequest::AuthStatus, }; request.validate_external_identifiers().unwrap(); let mut request = request; let CoordinatorRequest::Authenticated { session_secret, .. } = &mut request else { unreachable!() }; *session_secret = "bad\0secret".to_owned(); let error = request.validate_external_identifiers().unwrap_err(); assert!(error.contains("malformed external token request.session_secret")); } #[test] fn real_protocol_variants_reject_exactly_one_malformed_nested_identifier() { let launch = CoordinatorRequest::LaunchTask { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), actor_agent: None, agent_public_key_fingerprint: None, agent_signature: None, task_spec: valid_task_spec(), wait_for_node: false, artifact_path: "/vfs/artifacts/output".to_owned(), wasm_module_base64: "AGFzbQEAAAA=".to_owned(), }; let mut malformed_definition = serde_json::to_value(&launch).unwrap(); malformed_definition["task_spec"]["task_definition"] = serde_json::Value::String("bad task definition!".to_owned()); let error = validation_error(malformed_definition); assert!(error.contains("TaskDefinitionId is invalid")); let mut malformed_artifact = serde_json::to_value(&launch).unwrap(); malformed_artifact["task_spec"]["required_artifacts"][0] = serde_json::Value::String("bad artifact!".to_owned()); let error = validation_error(malformed_artifact); assert!(error.contains("ArtifactId is invalid")); let rendezvous = CoordinatorRequest::RequestRendezvous { scope: DataPlaneScope { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), process: ProcessId::from("process"), object: DataPlaneObject::Artifact(ArtifactId::from("artifact")), authorization_subject: "node-a-to-node-b".to_owned(), }, source: NodeEndpoint { node: NodeId::from("node-a"), advertised_addr: "node-a.invalid:4433".to_owned(), public_key_fingerprint: Digest::sha256("node-a"), }, destination: NodeEndpoint { node: NodeId::from("node-b"), advertised_addr: "node-b.invalid:4433".to_owned(), public_key_fingerprint: Digest::sha256("node-b"), }, direct_connectivity: true, failure_reason: String::new(), }; let mut malformed_endpoint = serde_json::to_value(rendezvous).unwrap(); malformed_endpoint["destination"]["node"] = serde_json::Value::String("bad node!".to_owned()); let error = validation_error(malformed_endpoint); assert!(error.contains("NodeId is invalid")); } #[test] fn signed_and_authenticated_real_variants_validate_scoped_ids_and_tokens() { let cases = [ CoordinatorRequest::Authenticated { session_secret: "session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::AbortProcess { process: "bad process!".to_owned(), launch_attempt: Some("attempt".to_owned()), }, }, CoordinatorRequest::StartProcess { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("bad agent!".to_owned()), agent_public_key_fingerprint: Some(Digest::sha256("agent")), agent_signature: Some(AgentSignedRequest { nonce: "agent-nonce".to_owned(), issued_at_epoch_seconds: 1, signature: "ed25519:syntactically-bounded".to_owned(), }), process: "process".to_owned(), launch_attempt: Some("attempt".to_owned()), restart: false, }, CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "bad node!".to_owned(), node_signature: Some(NodeSignedRequest { nonce: "node-nonce".to_owned(), issued_at_epoch_seconds: 1, signature: "ed25519:syntactically-bounded".to_owned(), }), }, CoordinatorRequest::SignedNode { node: "node".to_owned(), node_signature: NodeSignedRequest { nonce: "node-nonce".to_owned(), issued_at_epoch_seconds: 1, signature: "ed25519:syntactically-bounded".to_owned(), }, request: Box::new(CoordinatorRequest::PollTaskAssignment { tenant: "tenant".to_owned(), project: "bad project!".to_owned(), node: "node".to_owned(), }), }, CoordinatorRequest::AbortProcess { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), launch_attempt: Some("bad attempt!".to_owned()), }, ]; for request in cases { let error = request.validate_external_identifiers().unwrap_err(); assert!( error.contains("malformed external identifier"), "unexpected validation error for {request:?}: {error}" ); } } #[test] fn signed_request_nonces_are_validated_as_opaque_tokens() { let agent_request = CoordinatorRequest::StartProcess { tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, actor_agent: Some("agent".to_owned()), agent_public_key_fingerprint: Some(Digest::sha256("agent")), agent_signature: Some(AgentSignedRequest { nonce: String::new(), issued_at_epoch_seconds: 1, signature: "ed25519:syntactically-bounded".to_owned(), }), process: "process".to_owned(), launch_attempt: Some("attempt".to_owned()), restart: false, }; let error = agent_request.validate_external_identifiers().unwrap_err(); assert!(error.contains("malformed external token request.agent_signature.nonce")); let node_request = CoordinatorRequest::NodeHeartbeat { tenant: "tenant".to_owned(), project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(NodeSignedRequest { nonce: String::new(), issued_at_epoch_seconds: 1, signature: "ed25519:syntactically-bounded".to_owned(), }), }; let error = node_request.validate_external_identifiers().unwrap_err(); assert!(error.contains("malformed external token request.node_signature.nonce")); } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum AuthenticatedCoordinatorRequest { AuthStatus, RevokeCliSession, CreateProject { project: String, name: String, }, SelectProject { project: String, }, ListProjects, RegisterAgentPublicKey { agent: String, public_key: String, }, ListAgentPublicKeys, RotateAgentPublicKey { agent: String, public_key: String, }, RevokeAgentPublicKey { agent: String, }, CreateNodeEnrollmentGrant { #[serde(default = "default_node_enrollment_ttl_seconds")] ttl_seconds: u64, }, ListNodeDescriptors, RevokeNodeCredential { node: String, }, StartProcess { process: String, #[serde(default, skip_serializing_if = "Option::is_none")] launch_attempt: Option, #[serde(default)] restart: bool, }, ScheduleTask { environment: Option, environment_digest: Option, required_capabilities: Vec, #[serde(default)] dependency_cache: Option, source_snapshot: Option, required_artifacts: Vec, prefer_node: Option, }, LaunchTask { task_spec: Box, #[serde(default)] wait_for_node: bool, artifact_path: String, wasm_module_base64: String, }, CancelProcess { process: String, }, AbortProcess { process: String, #[serde(default, skip_serializing_if = "Option::is_none")] launch_attempt: Option, }, ListProcesses, QuotaStatus, RestartTask { process: String, task: String, #[serde(default)] replacement_bundle: Option, }, ResolveTaskFailure { process: String, task: String, resolution: TaskFailureResolution, }, DebugAttach { process: String, }, SetDebugBreakpoints { process: String, #[serde(default)] revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { process: String, }, CreateDebugEpoch { process: String, stopped_task: String, reason: String, }, ResumeDebugEpoch { process: String, epoch: u64, }, InspectDebugEpoch { process: String, epoch: u64, }, ListTaskEvents { #[serde(default)] process: Option, }, ListTaskSnapshots { process: String, }, JoinTask { process: String, task: String, }, CreateArtifactDownloadLink { artifact: String, max_bytes: u64, #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, OpenArtifactDownloadStream { artifact: String, max_bytes: u64, token_digest: Digest, chunk_bytes: u64, }, RevokeArtifactDownloadLink { artifact: String, token_digest: Digest, }, ExportArtifactToNode { artifact: String, receiver_node: String, direct_connectivity: bool, failure_reason: String, }, } fn default_download_ttl_seconds() -> u64 { 900 } fn default_node_enrollment_ttl_seconds() -> u64 { 900 }