From 26fdcb9d8476a00e64343d233ed2670e564673f1 Mon Sep 17 00:00:00 2001 From: Clusterflux Release Date: Sun, 26 Jul 2026 17:22:33 +0200 Subject: [PATCH] Update public backend API surface Private source commit: ba3f7ce2b6d9 --- Cargo.lock | 24 +- Cargo.toml | 1 + crates/clusterflux-client/Cargo.toml | 18 + crates/clusterflux-client/README.md | 68 ++ crates/clusterflux-client/src/lib.rs | 909 ++++++++++++++++++ crates/clusterflux-client/src/protocol.rs | 398 ++++++++ crates/clusterflux-client/src/transport.rs | 187 ++++ crates/clusterflux-client/src/types.rs | 468 +++++++++ .../tests/client_contract.rs | 183 ++++ .../tests/fixtures/web_operations.json | 194 ++++ crates/clusterflux-control/src/lib.rs | 16 +- crates/clusterflux-coordinator/src/lib.rs | 16 + crates/clusterflux-coordinator/src/service.rs | 186 +++- .../src/service/authorization.rs | 17 + .../src/service/logs.rs | 301 +++++- .../src/service/main_runtime.rs | 22 + .../src/service/processes.rs | 19 +- .../src/service/protocol.rs | 214 ++++- .../src/service/protocol/responses.rs | 163 +++- .../src/service/relay.rs | 154 ++- .../src/service/routing.rs | 96 +- .../src/service/signed_nodes.rs | 30 + .../src/service/summaries.rs | 500 ++++++++++ .../src/service/tcp.rs | 75 +- .../src/service/tests.rs | 714 +++++++++++++- .../src/service/wire_protocol.rs | 12 +- crates/clusterflux-core/src/api_error.rs | 296 ++++++ crates/clusterflux-core/src/artifact.rs | 10 + crates/clusterflux-core/src/lib.rs | 2 + .../clusterflux-node/src/assignment_runner.rs | 1 + .../src/assignment_runner/process_runner.rs | 234 ++++- .../src/assignment_runner/tests.rs | 2 + .../src/coordinator_session.rs | 11 + docs/artifacts.md | 4 +- 34 files changed, 5473 insertions(+), 72 deletions(-) create mode 100644 crates/clusterflux-client/Cargo.toml create mode 100644 crates/clusterflux-client/README.md create mode 100644 crates/clusterflux-client/src/lib.rs create mode 100644 crates/clusterflux-client/src/protocol.rs create mode 100644 crates/clusterflux-client/src/transport.rs create mode 100644 crates/clusterflux-client/src/types.rs create mode 100644 crates/clusterflux-client/tests/client_contract.rs create mode 100644 crates/clusterflux-client/tests/fixtures/web_operations.json create mode 100644 crates/clusterflux-coordinator/src/service/summaries.rs create mode 100644 crates/clusterflux-core/src/api_error.rs diff --git a/Cargo.lock b/Cargo.lock index c4ac325..26517cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,20 @@ dependencies = [ "wasmparser 0.245.1", ] +[[package]] +name = "clusterflux-client" +version = "0.1.0" +dependencies = [ + "base64", + "clusterflux-control", + "clusterflux-coordinator", + "clusterflux-core", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "clusterflux-control" version = "0.1.0" @@ -1712,16 +1726,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "runtime-conformance" -version = "0.1.0" -dependencies = [ - "clusterflux-sdk", - "futures-executor", - "serde", - "serde_json", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index 0b813bb..22867aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "crates/clusterflux-cli", + "crates/clusterflux-client", "crates/clusterflux-control", "crates/clusterflux-coordinator", "crates/clusterflux-core", diff --git a/crates/clusterflux-client/Cargo.toml b/crates/clusterflux-client/Cargo.toml new file mode 100644 index 0000000..fa2b5cc --- /dev/null +++ b/crates/clusterflux-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clusterflux-client" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +base64.workspace = true +clusterflux-control = { path = "../clusterflux-control" } +clusterflux-core = { path = "../clusterflux-core" } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["sync", "time"] } + +[dev-dependencies] +clusterflux-coordinator = { path = "../clusterflux-coordinator" } diff --git a/crates/clusterflux-client/README.md b/crates/clusterflux-client/README.md new file mode 100644 index 0000000..89951c1 --- /dev/null +++ b/crates/clusterflux-client/README.md @@ -0,0 +1,68 @@ +# clusterflux-client + +`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web +backend. It talks to the same versioned control and login endpoints as the CLI. +Callers do not construct JSON envelopes or place session secrets into request +objects. + +The crate intentionally contains no coordinator implementation, persistence +types, scheduler policy, or website-specific business rules. A web backend only +needs this crate to authenticate and work with account status, projects, Agent +keys, node enrollment and liveness, current/recent processes, task attempts, +recent logs, artifacts, quota state, process control, task recovery, and Debug +Epoch state. It does not expose workflow launch or whole-process replay. + +```rust,no_run +use clusterflux_client::{ClusterfluxClient, SessionCredential}; + +# async fn example() -> Result<(), clusterflux_client::ClientError> { +let credential = SessionCredential::from_secret( + std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"), +); +let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")? + .with_session_credential(&credential); + +let account = client.account_status().await?; +let nodes = client.list_nodes().await?; +let processes = client.list_processes(None, 20).await?; +# let _ = (account, nodes, processes); +# Ok(()) +# } +``` + +## Login boundary + +`begin_browser_login` starts the hosted Authentik flow. The identity callback +returns only to the fixed configured website URL with a short-lived handoff. +The website backend calls `exchange_browser_login_handoff` once and stores the +returned `SessionCredential` only in encrypted server-side session storage. +The credential has redacted `Debug` output and deliberately does not implement +Serde serialization. Logout calls the existing session-revocation operation. + +The browser must not receive or persist the Clusterflux session credential, +provider tokens, authorization code, PKCE verifier, or CLI polling secret. + +## Errors, bounds, and transport + +API failures are returned as `ClientError::Api(ApiError)`. Decisions should use +the stable `code`, `category`, `retryable`, and `request_id` fields, while +`message` is for people. The client rejects an error whose request ID does not +match the originating request. + +Paginated list methods require an explicit bounded page size. Process pages are +limited to 100 entries; node, artifact, and recent-log pages to 200. +`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes +its cursor. Recent logs are ephemeral and can report truncation or sequence +loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates +offsets and decodes bounded chunks without materializing the complete artifact. + +`ControlTransport` has bounded connect and I/O timeouts and performs blocking +network work outside the async executor. Dropping a request future cancels the +caller’s wait; any already-started blocking I/O remains bounded by its timeout. +`MockTransport` supplies deterministic responses and records exact envelopes +for application tests. + +`CLIENT_API_VERSION` is the one supported control protocol version. The +checked-in `web_operations.json` contract fixture records every website +operation and its stable error shape, and the crate’s contract suite checks the +fixture. diff --git a/crates/clusterflux-client/src/lib.rs b/crates/clusterflux-client/src/lib.rs new file mode 100644 index 0000000..12c94bd --- /dev/null +++ b/crates/clusterflux-client/src/lib.rs @@ -0,0 +1,909 @@ +mod protocol; +mod transport; +mod types; + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; +use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; +use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION}; +use protocol::{AuthenticatedRequest, LoginRequest, WireResponse}; +use serde_json::json; +use thiserror::Error; +use transport::{ClientTransport, TransportRequest}; + +pub use clusterflux_core::{ + AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind, + Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId, + ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId, + UserId, VfsPath, +}; +pub use transport::{ + ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse, +}; +pub use types::*; + +pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION; + +#[derive(Debug, Error)] +pub enum ClientError { + #[error("Clusterflux API error: {0}")] + Api(ApiError), + #[error(transparent)] + Transport(#[from] ClientTransportError), + #[error("Clusterflux client protocol error: {0}")] + Protocol(String), +} + +#[derive(Clone)] +pub struct ClusterfluxClient { + transport: Arc, + session_secret: Arc>>, + next_request: Arc, +} + +impl ClusterfluxClient { + pub fn connect(endpoint: impl Into) -> Result { + Ok(Self::with_transport(ControlTransport::new(endpoint)?)) + } + + pub fn with_transport(transport: impl ClientTransport) -> Self { + Self { + transport: Arc::new(transport), + session_secret: Arc::new(Mutex::new(None)), + next_request: Arc::new(AtomicU64::new(1)), + } + } + + pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self { + self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone()))); + self + } + + pub fn is_session_configured(&self) -> bool { + self.session_secret + .lock() + .map(|secret| secret.is_some()) + .unwrap_or(false) + } + + pub async fn begin_browser_login(&self) -> Result { + match self + .send_login(LoginRequest::BeginWebBrowserLogin {}) + .await? + { + WireResponse::WebBrowserLoginStarted { + transaction_id, + authorization_url, + expires_at_epoch_seconds, + } => Ok(BrowserLoginStart { + transaction_id, + authorization_url, + expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("web_browser_login_started")), + } + } + + pub async fn exchange_browser_login_handoff( + &self, + transaction_id: impl Into, + handoff_code: impl Into, + ) -> Result { + match self + .send_login(LoginRequest::ExchangeWebLoginHandoff { + transaction_id: transaction_id.into(), + handoff_code: handoff_code.into(), + }) + .await? + { + WireResponse::WebBrowserSession { session } => Ok(BrowserSession { + tenant: session.tenant, + project: session.project, + user: session.user, + credential: SessionCredential(session.session_secret), + expires_at_epoch_seconds: session.expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("web_browser_session")), + } + } + + pub async fn account_status(&self) -> Result { + match self + .send_authenticated(AuthenticatedRequest::AuthStatus) + .await? + { + WireResponse::AuthStatus { + tenant, + project, + actor, + authenticated, + account_status, + suspended, + disabled, + deleted, + manual_review, + sanitized_reason, + next_actions, + } => Ok(AccountStatus { + tenant, + project, + actor, + authenticated, + account_status, + suspended, + disabled, + deleted, + manual_review, + sanitized_reason, + next_actions, + }), + _ => Err(unexpected_response("auth_status")), + } + } + + pub async fn logout(&self) -> Result<(), ClientError> { + match self + .send_authenticated(AuthenticatedRequest::RevokeCliSession) + .await? + { + WireResponse::CliSessionRevoked {} => { + *self.session_secret.lock().map_err(|_| { + ClientError::Protocol("session credential lock was poisoned".to_owned()) + })? = None; + Ok(()) + } + _ => Err(unexpected_response("cli_session_revoked")), + } + } + + pub async fn list_projects(&self) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListProjects) + .await? + { + WireResponse::Projects { projects } => Ok(projects), + _ => Err(unexpected_response("projects")), + } + } + + pub async fn create_project( + &self, + project: ProjectId, + name: impl Into, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateProject { + project, + name: name.into(), + }) + .await? + { + WireResponse::ProjectCreated { project } => Ok(project), + _ => Err(unexpected_response("project_created")), + } + } + + pub async fn select_project(&self, project: ProjectId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::SelectProject { project }) + .await? + { + WireResponse::ProjectSelected { project } => Ok(project), + _ => Err(unexpected_response("project_selected")), + } + } + + pub async fn register_agent_public_key( + &self, + agent: AgentId, + public_key: impl Into, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey { + agent, + public_key: public_key.into(), + }) + .await + } + + pub async fn list_agent_public_keys(&self) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys) + .await? + { + WireResponse::AgentPublicKeys { records } => Ok(records), + _ => Err(unexpected_response("agent_public_keys")), + } + } + + pub async fn rotate_agent_public_key( + &self, + agent: AgentId, + public_key: impl Into, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey { + agent, + public_key: public_key.into(), + }) + .await + } + + pub async fn revoke_agent_public_key( + &self, + agent: AgentId, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent }) + .await + } + + async fn agent_key_mutation( + &self, + request: AuthenticatedRequest, + ) -> Result { + match self.send_authenticated(request).await? { + WireResponse::AgentPublicKey { record } => Ok(record), + _ => Err(unexpected_response("agent_public_key")), + } + } + + pub async fn create_node_enrollment_grant( + &self, + ttl_seconds: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds }) + .await? + { + WireResponse::NodeEnrollmentGrantCreated { + tenant, + project, + grant, + scope, + expires_at_epoch_seconds, + } => Ok(NodeEnrollmentGrant { + tenant, + project, + grant, + scope, + expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("node_enrollment_grant_created")), + } + } + + pub async fn list_nodes(&self) -> Result, ClientError> { + Ok(self.list_nodes_page(None, 200).await?.nodes) + } + + pub async fn list_nodes_page( + &self, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit }) + .await? + { + WireResponse::NodeSummaries { nodes, next_cursor } => { + Ok(NodePage { nodes, next_cursor }) + } + _ => Err(unexpected_response("node_summaries")), + } + } + + pub async fn revoke_node(&self, node: NodeId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node }) + .await? + { + WireResponse::NodeCredentialRevoked { + node, + tenant, + project, + actor, + descriptor_removed, + queued_assignments_removed, + } => Ok(NodeRevocation { + node, + tenant, + project, + actor, + descriptor_removed, + queued_assignments_removed, + }), + _ => Err(unexpected_response("node_credential_revoked")), + } + } + + pub async fn list_processes( + &self, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit }) + .await? + { + WireResponse::ProcessSummaries { + processes, + next_cursor, + } => Ok(ProcessPage { + processes, + next_cursor, + }), + _ => Err(unexpected_response("process_summaries")), + } + } + + pub async fn cancel_process( + &self, + process: ProcessId, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CancelProcess { process }) + .await? + { + WireResponse::ProcessCancellationRequested { + process, + cancelled_tasks, + affected_nodes, + } => Ok(ProcessCancellation { + process, + affected_tasks: cancelled_tasks, + affected_nodes, + aborted: false, + }), + _ => Err(unexpected_response("process_cancellation_requested")), + } + } + + pub async fn abort_process( + &self, + process: ProcessId, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::AbortProcess { + process, + launch_attempt: None, + }) + .await? + { + WireResponse::ProcessAborted { + process, + aborted_tasks, + affected_nodes, + } => Ok(ProcessCancellation { + process, + affected_tasks: aborted_tasks, + affected_nodes, + aborted: true, + }), + _ => Err(unexpected_response("process_aborted")), + } + } + + pub async fn quota_status(&self) -> Result { + match self + .send_authenticated(AuthenticatedRequest::QuotaStatus) + .await? + { + WireResponse::QuotaStatus { + tenant, + project, + actor, + policy_label, + limits, + window_seconds, + usage, + window_started_epoch_seconds, + } => Ok(QuotaStatus { + tenant, + project, + actor, + policy_label, + limits, + window_seconds, + usage, + window_started_epoch_seconds, + }), + _ => Err(unexpected_response("quota_status")), + } + } + + pub async fn list_task_events( + &self, + process: Option, + ) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListTaskEvents { process }) + .await? + { + WireResponse::TaskEvents { events } => Ok(events), + _ => Err(unexpected_response("task_events")), + } + } + + pub async fn list_task_snapshots( + &self, + process: ProcessId, + ) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process }) + .await? + { + WireResponse::TaskSnapshots { snapshots } => Ok(snapshots), + _ => Err(unexpected_response("task_snapshots")), + } + } + + pub async fn list_recent_logs( + &self, + process: ProcessId, + task: Option, + after_sequence: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListRecentLogs { + process, + task, + after_sequence, + limit, + }) + .await? + { + WireResponse::RecentLogs { + entries, + next_sequence, + history_truncated, + } => Ok(RecentLogPage { + entries, + next_sequence, + history_truncated, + }), + _ => Err(unexpected_response("recent_logs")), + } + } + + pub async fn restart_task( + &self, + process: ProcessId, + task: TaskInstanceId, + replacement_bundle: Option, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::RestartTask { + process, + task, + replacement_bundle, + }) + .await? + { + WireResponse::TaskRestart { + process, + task, + restarted_task_instance, + restarted_attempt_id, + actor, + accepted, + clean_boundary_available, + active_task, + completed_event_observed, + requires_whole_process_restart, + message, + audit_event, + } => Ok(TaskRestart { + process, + task, + restarted_task_instance, + restarted_attempt_id, + actor, + accepted, + clean_boundary_available, + active_task, + completed_event_observed, + requires_whole_process_restart, + message, + audit_event, + }), + _ => Err(unexpected_response("task_restart")), + } + } + + pub async fn resolve_task_failure( + &self, + process: ProcessId, + task: TaskInstanceId, + resolution: TaskFailureResolution, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ResolveTaskFailure { + process, + task, + resolution, + }) + .await? + { + WireResponse::TaskFailureResolved { + process, + task, + attempt_id, + resolution, + } => Ok(TaskFailureResolutionResult { + process, + task, + attempt_id, + resolution, + }), + _ => Err(unexpected_response("task_failure_resolved")), + } + } + + pub async fn debug_attach(&self, process: ProcessId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::DebugAttach { process }) + .await? + { + WireResponse::DebugAttach { + process, + actor, + authorization, + audit_event, + } => Ok(DebugAttach { + process, + actor, + authorization, + audit_event, + }), + _ => Err(unexpected_response("debug_attach")), + } + } + + pub async fn create_debug_epoch( + &self, + process: ProcessId, + stopped_task: TaskInstanceId, + reason: impl Into, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateDebugEpoch { + process, + stopped_task, + reason: reason.into(), + }) + .await? + { + WireResponse::DebugEpoch { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + } => Ok(DebugEpochControl { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch")), + } + } + + pub async fn resume_debug_epoch( + &self, + process: ProcessId, + epoch: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch }) + .await? + { + WireResponse::DebugEpoch { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + } => Ok(DebugEpochControl { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch")), + } + } + + pub async fn inspect_debug_epoch( + &self, + process: ProcessId, + epoch: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch }) + .await? + { + WireResponse::DebugEpochStatus { + process, + actor, + epoch, + command, + expected_tasks, + acknowledgements, + fully_frozen, + partially_frozen, + fully_resumed, + failed, + failure_messages, + audit_event, + } => Ok(DebugEpochStatus { + process, + actor, + epoch, + command, + expected_tasks, + acknowledgements, + fully_frozen, + partially_frozen, + fully_resumed, + failed, + failure_messages, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch_status")), + } + } + + pub async fn list_artifacts( + &self, + process: Option, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListArtifacts { + process, + cursor, + limit, + }) + .await? + { + WireResponse::Artifacts { + artifacts, + next_cursor, + } => Ok(ArtifactPage { + artifacts, + next_cursor, + }), + _ => Err(unexpected_response("artifacts")), + } + } + + pub async fn get_artifact(&self, artifact: ArtifactId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::GetArtifact { artifact }) + .await? + { + WireResponse::Artifact { artifact } => Ok(artifact), + _ => Err(unexpected_response("artifact")), + } + } + + pub async fn begin_artifact_download( + &self, + artifact: ArtifactId, + max_bytes: u64, + ttl_seconds: u64, + chunk_bytes: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink { + artifact: artifact.clone(), + max_bytes, + ttl_seconds, + }) + .await? + { + WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload { + client: self.clone(), + artifact, + max_bytes, + chunk_bytes, + token_digest: link.scoped_token_digest.clone(), + link, + expected_offset: 0, + finished: false, + }), + _ => Err(unexpected_response("artifact_download_link")), + } + } + + async fn send_authenticated( + &self, + request: AuthenticatedRequest, + ) -> Result { + let session_secret = self + .session_secret + .lock() + .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))? + .clone() + .ok_or_else(|| { + ClientError::Api(ApiError::from_message( + "client", + "no authenticated Clusterflux session is configured", + )) + })?; + self.send( + CONTROL_API_PATH, + json!({ + "type": "authenticated", + "session_secret": session_secret, + "request": request, + }), + ) + .await + } + + async fn send_login(&self, request: LoginRequest) -> Result { + let payload = serde_json::to_value(request) + .map_err(|error| ClientError::Protocol(error.to_string()))?; + self.send(LOGIN_API_PATH, payload).await + } + + async fn send( + &self, + api_path: &str, + payload: serde_json::Value, + ) -> Result { + let request_number = self.next_request.fetch_add(1, Ordering::Relaxed); + let request_id = format!("client-{request_number}"); + let envelope = coordinator_wire_request(&request_id, payload); + let body = serde_json::to_vec(&envelope) + .map_err(|error| ClientError::Protocol(error.to_string()))?; + let response = self + .transport + .send(TransportRequest { + api_path: api_path.to_owned(), + body, + }) + .await?; + let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| { + ClientError::Protocol(format!("decode typed API response: {error}")) + })?; + match response { + WireResponse::Error { + code, + category, + message, + retryable, + request_id: response_request_id, + } => { + if response_request_id != request_id { + return Err(ClientError::Protocol(format!( + "error response request_id {response_request_id} does not match {request_id}" + ))); + } + Err(ClientError::Api(ApiError::new( + code, + category, + message, + retryable, + response_request_id, + ))) + } + response => Ok(response), + } + } +} + +pub struct ArtifactDownload { + client: ClusterfluxClient, + artifact: ArtifactId, + max_bytes: u64, + chunk_bytes: u64, + token_digest: Digest, + link: clusterflux_core::DownloadLink, + expected_offset: u64, + finished: bool, +} + +impl ArtifactDownload { + pub fn link(&self) -> &clusterflux_core::DownloadLink { + &self.link + } + + pub async fn next_chunk(&mut self) -> Result { + if self.finished { + return Ok(ArtifactDownloadPoll::Chunk { + offset: self.expected_offset, + bytes: Vec::new(), + eof: true, + }); + } + match self + .client + .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream { + artifact: self.artifact.clone(), + max_bytes: self.max_bytes, + token_digest: self.token_digest.clone(), + chunk_bytes: self.chunk_bytes, + }) + .await? + { + WireResponse::ArtifactDownloadStream { + content_bytes_available, + content_offset, + content_eof, + content_base64, + .. + } => { + if !content_bytes_available { + return Ok(ArtifactDownloadPoll::Pending); + } + let offset = content_offset.ok_or_else(|| { + ClientError::Protocol( + "artifact response contained bytes without an offset".to_owned(), + ) + })?; + if offset != self.expected_offset { + return Err(ClientError::Protocol(format!( + "artifact chunk offset {offset} does not match expected {}", + self.expected_offset + ))); + } + let bytes = BASE64_STANDARD + .decode(content_base64.unwrap_or_default()) + .map_err(|error| { + ClientError::Protocol(format!( + "artifact response contains invalid base64: {error}" + )) + })?; + self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64); + self.finished = content_eof; + Ok(ArtifactDownloadPoll::Chunk { + offset, + bytes, + eof: content_eof, + }) + } + _ => Err(unexpected_response("artifact_download_stream")), + } + } + + pub async fn cancel(mut self) -> Result<(), ClientError> { + if self.finished { + return Ok(()); + } + match self + .client + .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink { + artifact: self.artifact.clone(), + token_digest: self.token_digest.clone(), + }) + .await? + { + WireResponse::ArtifactDownloadLinkRevoked {} => { + self.finished = true; + Ok(()) + } + _ => Err(unexpected_response("artifact_download_link_revoked")), + } + } +} + +fn unexpected_response(expected: &str) -> ClientError { + ClientError::Protocol(format!( + "expected typed response {expected}, received another response variant" + )) +} diff --git a/crates/clusterflux-client/src/protocol.rs b/crates/clusterflux-client/src/protocol.rs new file mode 100644 index 0000000..084f3ba --- /dev/null +++ b/crates/clusterflux-client/src/protocol.rs @@ -0,0 +1,398 @@ +use std::collections::BTreeMap; + +use clusterflux_core::{ + AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink, + LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId, +}; +use serde::{Deserialize, Serialize}; + +use crate::types::*; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum AuthenticatedRequest { + AuthStatus, + RevokeCliSession, + CreateProject { + project: ProjectId, + name: String, + }, + SelectProject { + project: ProjectId, + }, + ListProjects, + RegisterAgentPublicKey { + agent: AgentId, + public_key: String, + }, + ListAgentPublicKeys, + RotateAgentPublicKey { + agent: AgentId, + public_key: String, + }, + RevokeAgentPublicKey { + agent: AgentId, + }, + CreateNodeEnrollmentGrant { + ttl_seconds: u64, + }, + ListNodeSummaries { + cursor: Option, + limit: u32, + }, + RevokeNodeCredential { + node: NodeId, + }, + ListProcessSummaries { + cursor: Option, + limit: u32, + }, + CancelProcess { + process: ProcessId, + }, + AbortProcess { + process: ProcessId, + launch_attempt: Option, + }, + QuotaStatus, + ListTaskEvents { + process: Option, + }, + ListTaskSnapshots { + process: ProcessId, + }, + ListRecentLogs { + process: ProcessId, + task: Option, + after_sequence: Option, + limit: u32, + }, + RestartTask { + process: ProcessId, + task: TaskInstanceId, + replacement_bundle: Option, + }, + ResolveTaskFailure { + process: ProcessId, + task: TaskInstanceId, + resolution: TaskFailureResolution, + }, + DebugAttach { + process: ProcessId, + }, + CreateDebugEpoch { + process: ProcessId, + stopped_task: TaskInstanceId, + reason: String, + }, + ResumeDebugEpoch { + process: ProcessId, + epoch: u64, + }, + InspectDebugEpoch { + process: ProcessId, + epoch: u64, + }, + ListArtifacts { + process: Option, + cursor: Option, + limit: u32, + }, + GetArtifact { + artifact: ArtifactId, + }, + CreateArtifactDownloadLink { + artifact: ArtifactId, + max_bytes: u64, + ttl_seconds: u64, + }, + OpenArtifactDownloadStream { + artifact: ArtifactId, + max_bytes: u64, + token_digest: Digest, + chunk_bytes: u64, + }, + RevokeArtifactDownloadLink { + artifact: ArtifactId, + token_digest: Digest, + }, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum LoginRequest { + BeginWebBrowserLogin {}, + ExchangeWebLoginHandoff { + transaction_id: String, + handoff_code: String, + }, +} + +#[derive(Clone, Deserialize)] +pub(crate) struct WireBrowserSession { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub session_secret: String, + pub expires_at_epoch_seconds: u64, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum WireResponse { + AuthStatus { + tenant: TenantId, + project: ProjectId, + actor: UserId, + authenticated: bool, + account_status: String, + suspended: bool, + disabled: bool, + deleted: bool, + manual_review: bool, + sanitized_reason: Option, + next_actions: Vec, + }, + CliSessionRevoked {}, + ProjectCreated { + project: Project, + }, + ProjectSelected { + project: Project, + }, + Projects { + projects: Vec, + }, + AgentPublicKey { + record: AgentPublicKey, + }, + AgentPublicKeys { + records: Vec, + }, + NodeEnrollmentGrantCreated { + tenant: TenantId, + project: ProjectId, + grant: String, + scope: String, + expires_at_epoch_seconds: u64, + }, + NodeSummaries { + nodes: Vec, + next_cursor: Option, + }, + NodeCredentialRevoked { + node: NodeId, + tenant: TenantId, + project: ProjectId, + actor: UserId, + descriptor_removed: bool, + queued_assignments_removed: usize, + }, + ProcessSummaries { + processes: Vec, + next_cursor: Option, + }, + ProcessCancellationRequested { + process: ProcessId, + cancelled_tasks: Vec, + affected_nodes: Vec, + }, + ProcessAborted { + process: ProcessId, + aborted_tasks: Vec, + affected_nodes: Vec, + }, + QuotaStatus { + tenant: TenantId, + project: ProjectId, + actor: UserId, + policy_label: Option, + limits: ResourceLimits, + window_seconds: BTreeMap, + usage: BTreeMap, + window_started_epoch_seconds: BTreeMap, + }, + TaskEvents { + events: Vec, + }, + TaskSnapshots { + snapshots: Vec, + }, + RecentLogs { + entries: Vec, + next_sequence: Option, + history_truncated: bool, + }, + TaskRestart { + process: ProcessId, + task: TaskInstanceId, + restarted_task_instance: Option, + restarted_attempt_id: Option, + actor: UserId, + accepted: bool, + clean_boundary_available: bool, + active_task: bool, + completed_event_observed: bool, + requires_whole_process_restart: bool, + message: String, + audit_event: DebugAuditEvent, + }, + TaskFailureResolved { + process: ProcessId, + task: TaskInstanceId, + attempt_id: String, + resolution: TaskFailureResolution, + }, + DebugAttach { + process: ProcessId, + actor: UserId, + authorization: Authorization, + audit_event: DebugAuditEvent, + }, + DebugEpoch { + process: ProcessId, + actor: UserId, + epoch: u64, + command: String, + affected_tasks: Vec, + all_stop_requested: bool, + audit_event: DebugAuditEvent, + }, + DebugEpochStatus { + process: ProcessId, + actor: UserId, + epoch: u64, + command: String, + expected_tasks: Vec, + acknowledgements: Vec, + fully_frozen: bool, + partially_frozen: bool, + fully_resumed: bool, + failed: bool, + failure_messages: Vec, + audit_event: DebugAuditEvent, + }, + Artifacts { + artifacts: Vec, + next_cursor: Option, + }, + Artifact { + artifact: ArtifactSummary, + }, + ArtifactDownloadLink { + link: DownloadLink, + }, + ArtifactDownloadLinkRevoked {}, + ArtifactDownloadStream { + #[serde(rename = "link")] + _link: DownloadLink, + content_bytes_available: bool, + content_offset: Option, + content_eof: bool, + content_base64: Option, + }, + WebBrowserLoginStarted { + transaction_id: String, + authorization_url: String, + expires_at_epoch_seconds: u64, + }, + WebBrowserSession { + session: WireBrowserSession, + }, + Error { + code: ApiErrorCode, + category: ApiErrorCategory, + message: String, + retryable: bool, + request_id: String, + }, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use serde::Deserialize; + use serde_json::Value; + + use super::*; + + #[derive(Deserialize)] + struct OperationFixture { + operation: String, + boundary: String, + request: Value, + response: Value, + } + + #[test] + fn website_operation_contract_fixtures_cover_every_typed_request() { + let fixtures: Vec = + serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap(); + let expected = BTreeSet::from([ + "abort_process", + "auth_status", + "begin_web_browser_login", + "cancel_process", + "create_artifact_download_link", + "create_debug_epoch", + "create_node_enrollment_grant", + "create_project", + "debug_attach", + "exchange_web_login_handoff", + "get_artifact", + "inspect_debug_epoch", + "list_agent_public_keys", + "list_artifacts", + "list_node_summaries", + "list_process_summaries", + "list_projects", + "list_recent_logs", + "list_task_events", + "list_task_snapshots", + "open_artifact_download_stream", + "quota_status", + "register_agent_public_key", + "resolve_task_failure", + "restart_task", + "resume_debug_epoch", + "revoke_agent_public_key", + "revoke_artifact_download_link", + "revoke_cli_session", + "revoke_node_credential", + "rotate_agent_public_key", + "select_project", + ]) + .into_iter() + .map(str::to_owned) + .collect(); + let mut observed = BTreeSet::new(); + + for fixture in fixtures { + assert_eq!(fixture.request["type"], fixture.operation); + let round_trip = match fixture.boundary.as_str() { + "control" => { + let typed: AuthenticatedRequest = + serde_json::from_value(fixture.request.clone()).unwrap(); + serde_json::to_value(typed).unwrap() + } + "login" => { + let typed: LoginRequest = + serde_json::from_value(fixture.request.clone()).unwrap(); + serde_json::to_value(typed).unwrap() + } + boundary => panic!("unknown fixture boundary {boundary}"), + }; + assert_eq!(round_trip, fixture.request); + let response: WireResponse = serde_json::from_value(fixture.response).unwrap(); + assert!( + !matches!(response, WireResponse::Error { .. }), + "{} must carry its operation-specific success response fixture", + fixture.operation + ); + assert!(observed.insert(fixture.operation)); + } + assert_eq!(observed, expected); + } +} diff --git a/crates/clusterflux-client/src/transport.rs b/crates/clusterflux-client/src/transport.rs new file mode 100644 index 0000000..5c440ff --- /dev/null +++ b/crates/clusterflux-client/src/transport.rs @@ -0,0 +1,187 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use clusterflux_control::{endpoint_identity, ControlSession}; +use thiserror::Error; + +pub type TransportFuture = + Pin> + Send + 'static>>; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportRequest { + pub api_path: String, + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportResponse { + pub body: Vec, +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum ClientTransportError { + #[error("client transport failed: {0}")] + Failed(String), + #[error("client transport task failed: {0}")] + Task(String), +} + +pub trait ClientTransport: Send + Sync + 'static { + fn send(&self, request: TransportRequest) -> TransportFuture; +} + +pub struct ControlTransport { + endpoint: String, + connect_timeout: Duration, + io_timeout: Duration, + sessions: Arc>>, +} + +impl ControlTransport { + pub fn new(endpoint: impl Into) -> Result { + Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30)) + } + + pub fn with_timeouts( + endpoint: impl Into, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result { + let endpoint = endpoint.into(); + endpoint_identity(&endpoint) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?; + Ok(Self { + endpoint, + connect_timeout, + io_timeout, + sessions: Arc::new(Mutex::new(BTreeMap::new())), + }) + } +} + +impl ClientTransport for ControlTransport { + fn send(&self, request: TransportRequest) -> TransportFuture { + let endpoint = self.endpoint.clone(); + let connect_timeout = self.connect_timeout; + let io_timeout = self.io_timeout; + let sessions = Arc::clone(&self.sessions); + Box::pin(async move { + tokio::task::spawn_blocking(move || { + let value = serde_json::from_slice(&request.body) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?; + let mut sessions = sessions.lock().map_err(|_| { + ClientTransportError::Failed( + "client transport session lock was poisoned".to_owned(), + ) + })?; + if !sessions.contains_key(&request.api_path) { + let session = ControlSession::connect_to_api_path_with_timeouts( + &endpoint, + &request.api_path, + connect_timeout, + io_timeout, + ) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?; + sessions.insert(request.api_path.clone(), session); + } + let response = sessions + .get_mut(&request.api_path) + .expect("session was inserted for the requested API path") + .request(&value); + match response { + Ok(response) => serde_json::to_vec(&response) + .map(|body| TransportResponse { body }) + .map_err(|error| ClientTransportError::Failed(error.to_string())), + Err(error) => { + sessions.remove(&request.api_path); + Err(ClientTransportError::Failed(error.to_string())) + } + } + }) + .await + .map_err(|error| ClientTransportError::Task(error.to_string()))? + }) + } +} + +#[derive(Clone, Default)] +pub struct MockTransport { + state: Arc>, +} + +#[derive(Default)] +struct MockTransportState { + responses: VecDeque, ClientTransportError>>, + requests: Vec, +} + +impl MockTransport { + pub fn from_json_responses(responses: impl IntoIterator>) -> Self { + let responses = responses + .into_iter() + .map(|response| Ok(response.into().into_bytes())) + .collect(); + Self { + state: Arc::new(Mutex::new(MockTransportState { + responses, + requests: Vec::new(), + })), + } + } + + pub fn push_json_response(&self, response: impl Into) { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .responses + .push_back(Ok(response.into().into_bytes())); + } + + pub fn push_error(&self, message: impl Into) { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .responses + .push_back(Err(ClientTransportError::Failed(message.into()))); + } + + pub fn request_bodies(&self) -> Vec { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .requests + .iter() + .map(|request| String::from_utf8_lossy(&request.body).into_owned()) + .collect() + } + + pub fn requests(&self) -> Vec { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .requests + .clone() + } +} + +impl ClientTransport for MockTransport { + fn send(&self, request: TransportRequest) -> TransportFuture { + let state = Arc::clone(&self.state); + Box::pin(async move { + let mut state = state.lock().map_err(|_| { + ClientTransportError::Failed("mock transport lock was poisoned".to_owned()) + })?; + state.requests.push(request); + state + .responses + .pop_front() + .ok_or_else(|| { + ClientTransportError::Failed("mock transport has no queued response".to_owned()) + })? + .map(|body| TransportResponse { body }) + }) + } +} diff --git a/crates/clusterflux-client/src/types.rs b/crates/clusterflux-client/src/types.rs new file mode 100644 index 0000000..eb17e6e --- /dev/null +++ b/crates/clusterflux-client/src/types.rs @@ -0,0 +1,468 @@ +use std::collections::BTreeMap; + +use clusterflux_core::{ + AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement, + ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy, + TaskInstanceId, TenantId, UserId, VfsPath, +}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountStatus { + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub authenticated: bool, + pub account_status: String, + pub suspended: bool, + pub disabled: bool, + pub deleted: bool, + pub manual_review: bool, + pub sanitized_reason: Option, + pub next_actions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Project { + pub id: ProjectId, + pub tenant: TenantId, + pub name: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentPublicKey { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub agent: AgentId, + pub public_key: String, + pub public_key_fingerprint: Digest, + pub version: u64, + pub revoked: bool, + pub scopes: Vec, + pub human_account_creation_privilege: bool, + pub browser_interaction_required_each_run: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeEnrollmentGrant { + pub tenant: TenantId, + pub project: ProjectId, + pub grant: String, + pub scope: String, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeSummary { + pub id: NodeId, + pub display_name: String, + pub online: bool, + pub stale: bool, + pub last_seen_epoch_seconds: Option, + pub capabilities: NodeCapabilities, + pub direct_connectivity: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodePage { + pub nodes: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeRevocation { + pub node: NodeId, + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub descriptor_removed: bool, + pub queued_assignments_removed: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessLifecycleState { + Active, + RecentTerminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessActivityState { + Running, + WaitingForNode, + WaitingForTask, + AwaitingAction, + DebugEpochPartial, + Cancelling, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessFinalResult { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochSummary { + pub epoch: u64, + pub command: String, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSummary { + pub process: ProcessId, + pub lifecycle: ProcessLifecycleState, + pub activity: ProcessActivityState, + pub main_wait_state: Option, + pub started_at_epoch_seconds: u64, + pub ended_at_epoch_seconds: Option, + pub final_result: Option, + pub connected_nodes: Vec, + pub current_debug_epoch: Option, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessPage { + pub processes: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskTerminalState { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskExecutor { + CoordinatorMain, + Node, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCompletionEvent { + pub tenant: TenantId, + pub project: ProjectId, + pub process: ProcessId, + pub node: NodeId, + pub executor: TaskExecutor, + pub task_definition: TaskDefinitionId, + pub task: TaskInstanceId, + pub attempt_id: Option, + pub placement: Option, + pub terminal_state: TaskTerminalState, + pub status_code: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, + pub stdout_tail: String, + pub stderr_tail: String, + pub stdout_truncated: bool, + pub stderr_truncated: bool, + pub artifact_path: Option, + pub artifact_digest: Option, + pub artifact_size_bytes: Option, + pub result: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskAttemptState { + Queued, + Running, + FailedAwaitingAction, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskAttemptSnapshot { + pub process: ProcessId, + pub task: TaskInstanceId, + pub attempt_id: String, + pub attempt_number: u32, + pub task_definition: TaskDefinitionId, + pub display_name: String, + pub state: TaskAttemptState, + pub current: bool, + pub node: Option, + pub environment_id: Option, + pub environment_digest: Option, + pub argument_summary: Vec, + pub handle_summary: Vec, + pub command_state: Option, + pub vfs_checkpoint: String, + pub probe_symbol: Option, + pub source_path: Option, + pub source_line: Option, + pub restart_compatible: bool, + pub failure_policy: TaskFailurePolicy, + pub artifact_path: Option, + pub artifact_digest: Option, + pub artifact_size_bytes: Option, + pub status_code: Option, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskLogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogEntry { + pub sequence: u64, + pub process: ProcessId, + pub task: TaskInstanceId, + pub stream: TaskLogStream, + pub text: String, + pub server_timestamp_epoch_seconds: u64, + pub truncated: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogPage { + pub entries: Vec, + pub next_sequence: Option, + pub history_truncated: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactAvailability { + Available, + NodeOffline, + Unavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactRetentionState { + NodeRetained, + ExplicitStorage, + Lost, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactSummary { + pub id: ArtifactId, + pub display_path: String, + pub display_name: String, + pub process: ProcessId, + pub producer_task: TaskInstanceId, + pub safe_node: Option, + pub digest: Digest, + pub size_bytes: u64, + pub availability: ArtifactAvailability, + pub downloadable_now: bool, + pub retention_state: ArtifactRetentionState, + pub explicit_storage: bool, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactPage { + pub artifacts: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct QuotaStatus { + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub policy_label: Option, + pub limits: ResourceLimits, + pub window_seconds: BTreeMap, + pub usage: BTreeMap, + pub window_started_epoch_seconds: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCancellationTarget { + pub process: ProcessId, + pub task: TaskInstanceId, + pub node: NodeId, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessCancellation { + pub process: ProcessId, + pub affected_tasks: Vec, + pub affected_nodes: Vec, + pub aborted: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugAuditEvent { + pub tenant: TenantId, + pub project: ProjectId, + pub process: ProcessId, + pub task: Option, + pub actor: UserId, + pub operation: String, + pub allowed: bool, + pub reason: String, + pub charged_debug_read_bytes: u64, + pub used_debug_read_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugAttach { + pub process: ProcessId, + pub actor: UserId, + pub authorization: Authorization, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochControl { + pub process: ProcessId, + pub actor: UserId, + pub epoch: u64, + pub command: String, + pub affected_tasks: Vec, + pub all_stop_requested: bool, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DebugAcknowledgementState { + Frozen, + Running, + Failed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugParticipantAcknowledgement { + pub node: NodeId, + pub task_definition: TaskDefinitionId, + pub task: TaskInstanceId, + pub epoch: u64, + pub state: DebugAcknowledgementState, + pub stack_frames: Vec, + pub local_values: Vec<(String, String)>, + pub task_args: Vec<(String, String)>, + pub handles: Vec<(String, String)>, + pub command_status: Option, + pub recent_output: Vec, + pub message: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochStatus { + pub process: ProcessId, + pub actor: UserId, + pub epoch: u64, + pub command: String, + pub expected_tasks: Vec, + pub acknowledgements: Vec, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, + pub failure_messages: Vec, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskRestart { + pub process: ProcessId, + pub task: TaskInstanceId, + pub restarted_task_instance: Option, + pub restarted_attempt_id: Option, + pub actor: UserId, + pub accepted: bool, + pub clean_boundary_available: bool, + pub active_task: bool, + pub completed_event_observed: bool, + pub requires_whole_process_restart: bool, + pub message: String, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskFailureResolutionResult { + pub process: ProcessId, + pub task: TaskInstanceId, + pub attempt_id: String, + pub resolution: TaskFailureResolution, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskFailureResolution { + AcceptFailure, + Cancel, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskReplacementBundle { + pub bundle_digest: Digest, + pub wasm_module_base64: String, + pub source_snapshot: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserLoginStart { + pub transaction_id: String, + pub authorization_url: String, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct SessionCredential(pub(crate) String); + +impl SessionCredential { + pub fn from_secret(secret: impl Into) -> Self { + Self(secret.into()) + } + + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for SessionCredential { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SessionCredential([REDACTED])") + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowserSession { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub credential: SessionCredential, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArtifactDownloadPoll { + Pending, + Chunk { + offset: u64, + bytes: Vec, + eof: bool, + }, +} diff --git a/crates/clusterflux-client/tests/client_contract.rs b/crates/clusterflux-client/tests/client_contract.rs new file mode 100644 index 0000000..47b5bd7 --- /dev/null +++ b/crates/clusterflux-client/tests/client_contract.rs @@ -0,0 +1,183 @@ +use std::net::TcpListener; + +use clusterflux_client::{ + ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError, + ClusterfluxClient, MockTransport, ProjectId, SessionCredential, TenantId, UserId, + CLIENT_API_VERSION, +}; +use clusterflux_control::CONTROL_API_PATH; +use clusterflux_coordinator::service::CoordinatorService; +use serde_json::{json, Value}; + +#[tokio::test] +async fn mock_transport_exercises_typed_envelope_and_session_plumbing() { + let transport = MockTransport::from_json_responses([json!({ + "type": "projects", + "projects": [{ + "id": "project-one", + "tenant": "tenant-one", + "name": "Project one" + }], + "actor": "user-one" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport.clone()) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let projects = client.list_projects().await.unwrap(); + assert_eq!(projects.len(), 1); + assert_eq!(projects[0].id, ProjectId::from("project-one")); + + let requests = transport.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].api_path, CONTROL_API_PATH); + let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); + assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION); + assert_eq!(envelope["request_id"], "client-1"); + assert_eq!(envelope["operation"], "authenticated"); + assert_eq!(envelope["payload"]["type"], "authenticated"); + assert_eq!(envelope["payload"]["request"]["type"], "list_projects"); + assert_eq!(envelope["payload"]["session_secret"], "test-session-secret"); +} + +#[tokio::test] +async fn structured_errors_retain_machine_fields_and_originating_request_id() { + let transport = MockTransport::from_json_responses([json!({ + "type": "error", + "code": "account_suspended", + "category": "authorization", + "message": "account access is suspended", + "retryable": false, + "request_id": "client-1" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let ClientError::Api(error) = client.account_status().await.unwrap_err() else { + panic!("expected typed API error"); + }; + assert_eq!(error.code, ApiErrorCode::AccountSuspended); + assert_eq!(error.category, ApiErrorCategory::Authorization); + assert_eq!(error.request_id, "client-1"); + assert!(!error.retryable); +} + +#[tokio::test] +async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() { + let transport = MockTransport::from_json_responses([json!({ + "type": "error", + "code": "validation_error", + "category": "validation", + "message": "bad request", + "retryable": false, + "request_id": "another-request" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else { + panic!("expected protocol error"); + }; + assert!(message.contains("does not match client-1")); +} + +#[test] +fn session_credentials_are_redacted_from_debug_output() { + let credential = SessionCredential::from_secret("must-not-appear"); + assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])"); +} + +#[tokio::test] +async fn artifact_download_is_a_typed_bounded_chunk_stream() { + let link = json!({ + "artifact": "artifact-one", + "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "artifact_size_bytes": 5, + "source": { "RetainedNode": "node-one" }, + "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", + "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "expires_at_epoch_seconds": 1000, + "tenant": "tenant-one", + "project": "project-one", + "process": "process-one", + "actor": { "User": "user-one" }, + "max_bytes": 1024, + "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }); + let transport = MockTransport::from_json_responses([ + json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(), + json!({ + "type": "artifact_download_stream", + "link": link.clone(), + "streamed_bytes": 0, + "charged_download_bytes": 0, + "content_bytes_available": false, + "content_eof": false + }) + .to_string(), + json!({ + "type": "artifact_download_stream", + "link": link, + "streamed_bytes": 5, + "charged_download_bytes": 5, + "content_bytes_available": true, + "content_offset": 0, + "content_eof": true, + "content_base64": "aGVsbG8=" + }) + .to_string(), + ]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + let mut download = client + .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64) + .await + .unwrap(); + assert_eq!( + download.next_chunk().await.unwrap(), + ArtifactDownloadPoll::Pending + ); + assert_eq!( + download.next_chunk().await.unwrap(), + ArtifactDownloadPoll::Chunk { + offset: 0, + bytes: b"hello".to_vec(), + eof: true, + } + ); +} + +#[tokio::test] +async fn typed_client_runs_against_the_real_strict_control_endpoint() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let mut service = CoordinatorService::new(7); + service + .issue_cli_session( + TenantId::from("tenant-one"), + ProjectId::from("project-one"), + UserId::from("user-one"), + "real-endpoint-session", + None, + ) + .unwrap(); + let (stream, _) = listener.accept().unwrap(); + service.handle_stream(stream).unwrap(); + }); + + let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}")) + .unwrap() + .with_session_credential(&SessionCredential::from_secret("real-endpoint-session")); + let projects = client.list_projects().await.unwrap(); + assert_eq!(projects.len(), 1); + assert_eq!(projects[0].id, ProjectId::from("project-one")); + let status = client.account_status().await.unwrap(); + assert!(status.authenticated); + assert_eq!(status.actor, UserId::from("user-one")); + + drop(client); + server.join().unwrap(); +} diff --git a/crates/clusterflux-client/tests/fixtures/web_operations.json b/crates/clusterflux-client/tests/fixtures/web_operations.json new file mode 100644 index 0000000..16ea235 --- /dev/null +++ b/crates/clusterflux-client/tests/fixtures/web_operations.json @@ -0,0 +1,194 @@ +[ + { + "operation": "begin_web_browser_login", + "boundary": "login", + "request": { "type": "begin_web_browser_login" }, + "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 } + }, + { + "operation": "exchange_web_login_handoff", + "boundary": "login", + "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" }, + "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } } + }, + { + "operation": "auth_status", + "boundary": "control", + "request": { "type": "auth_status" }, + "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] } + }, + { + "operation": "revoke_cli_session", + "boundary": "control", + "request": { "type": "revoke_cli_session" }, + "response": { "type": "cli_session_revoked" } + }, + { + "operation": "create_project", + "boundary": "control", + "request": { "type": "create_project", "project": "project-new", "name": "New project" }, + "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } } + }, + { + "operation": "select_project", + "boundary": "control", + "request": { "type": "select_project", "project": "project-selected" }, + "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } } + }, + { + "operation": "list_projects", + "boundary": "control", + "request": { "type": "list_projects" }, + "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] } + }, + { + "operation": "register_agent_public_key", + "boundary": "control", + "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "list_agent_public_keys", + "boundary": "control", + "request": { "type": "list_agent_public_keys" }, + "response": { "type": "agent_public_keys", "records": [] } + }, + { + "operation": "rotate_agent_public_key", + "boundary": "control", + "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "revoke_agent_public_key", + "boundary": "control", + "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "create_node_enrollment_grant", + "boundary": "control", + "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 }, + "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 } + }, + { + "operation": "list_node_summaries", + "boundary": "control", + "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 }, + "response": { "type": "node_summaries", "nodes": [], "next_cursor": null } + }, + { + "operation": "revoke_node_credential", + "boundary": "control", + "request": { "type": "revoke_node_credential", "node": "node-one" }, + "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 } + }, + { + "operation": "list_process_summaries", + "boundary": "control", + "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 }, + "response": { "type": "process_summaries", "processes": [], "next_cursor": null } + }, + { + "operation": "cancel_process", + "boundary": "control", + "request": { "type": "cancel_process", "process": "process-one" }, + "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] } + }, + { + "operation": "abort_process", + "boundary": "control", + "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null }, + "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] } + }, + { + "operation": "quota_status", + "boundary": "control", + "request": { "type": "quota_status" }, + "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } } + }, + { + "operation": "list_task_events", + "boundary": "control", + "request": { "type": "list_task_events", "process": "process-one" }, + "response": { "type": "task_events", "events": [] } + }, + { + "operation": "list_task_snapshots", + "boundary": "control", + "request": { "type": "list_task_snapshots", "process": "process-one" }, + "response": { "type": "task_snapshots", "snapshots": [] } + }, + { + "operation": "list_recent_logs", + "boundary": "control", + "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 }, + "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false } + }, + { + "operation": "restart_task", + "boundary": "control", + "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null }, + "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "resolve_task_failure", + "boundary": "control", + "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" }, + "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" } + }, + { + "operation": "debug_attach", + "boundary": "control", + "request": { "type": "debug_attach", "process": "process-one" }, + "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "create_debug_epoch", + "boundary": "control", + "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" }, + "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "resume_debug_epoch", + "boundary": "control", + "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 }, + "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "inspect_debug_epoch", + "boundary": "control", + "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 }, + "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "list_artifacts", + "boundary": "control", + "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 }, + "response": { "type": "artifacts", "artifacts": [], "next_cursor": null } + }, + { + "operation": "get_artifact", + "boundary": "control", + "request": { "type": "get_artifact", "artifact": "artifact-one" }, + "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } } + }, + { + "operation": "create_artifact_download_link", + "boundary": "control", + "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 }, + "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } } + }, + { + "operation": "open_artifact_download_stream", + "boundary": "control", + "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 }, + "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" } + }, + { + "operation": "revoke_artifact_download_link", + "boundary": "control", + "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + "response": { "type": "artifact_download_link_revoked" } + } +] diff --git a/crates/clusterflux-control/src/lib.rs b/crates/clusterflux-control/src/lib.rs index 481e718..39a150d 100644 --- a/crates/clusterflux-control/src/lib.rs +++ b/crates/clusterflux-control/src/lib.rs @@ -61,7 +61,21 @@ impl ControlSession { endpoint: &str, api_path: &str, ) -> Result { - let session = Self::connect(endpoint)?; + Self::connect_to_api_path_with_timeouts( + endpoint, + api_path, + Duration::from_secs(10), + Duration::from_secs(30), + ) + } + + pub fn connect_to_api_path_with_timeouts( + endpoint: &str, + api_path: &str, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result { + let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?; #[cfg(not(target_arch = "wasm32"))] { let mut session = session; diff --git a/crates/clusterflux-coordinator/src/lib.rs b/crates/clusterflux-coordinator/src/lib.rs index 403e8d2..a110834 100644 --- a/crates/clusterflux-coordinator/src/lib.rs +++ b/crates/clusterflux-coordinator/src/lib.rs @@ -585,6 +585,22 @@ impl Coordinator { .count() } + pub fn tenant_count(&self) -> usize { + self.durable.tenants.len() + } + + pub fn user_count(&self) -> usize { + self.durable.users.len() + } + + pub fn project_count(&self) -> usize { + self.durable.projects.len() + } + + pub fn node_identity_count(&self) -> usize { + self.durable.node_identities.len() + } + pub fn node_identity( &self, tenant: &TenantId, diff --git a/crates/clusterflux-coordinator/src/service.rs b/crates/clusterflux-coordinator/src/service.rs index c1ea1f0..d7472bd 100644 --- a/crates/clusterflux-coordinator/src/service.rs +++ b/crates/clusterflux-coordinator/src/service.rs @@ -7,9 +7,10 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::time::{SystemTime, UNIX_EPOCH}; use clusterflux_core::{ - Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError, - NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId, - RateLimit, TenantId, TransportError, UserId, + Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry, + CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport, + NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit, + TenantId, TransportError, UserId, }; use thiserror::Error; @@ -34,6 +35,7 @@ mod quota; mod relay; mod routing; mod signed_nodes; +mod summaries; mod tcp; mod wire_protocol; use authorization::authorize_authenticated_user_operation; @@ -43,12 +45,14 @@ use keys::{ ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey, }; pub use protocol::{ - ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest, - CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent, - DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, - TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, - TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle, - TaskTerminalState, VirtualProcessStatus, WorkflowActor, + ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment, + AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, + DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement, + NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, + RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment, + TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent, + TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState, + VirtualProcessStatus, WorkflowActor, }; pub use quota::CoordinatorQuotaConfiguration; pub use relay::{ @@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128; const MAX_TASK_EVENTS_TOTAL: usize = 8_192; const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192; const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096; -const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096; +const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000; const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256; const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024; +const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256; +const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024; +const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024; +const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024; +const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32; +const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192; const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30; fn bounded_ttl(requested: u64, maximum: u64) -> u64 { requested.clamp(1, maximum) @@ -111,6 +121,24 @@ pub struct CoordinatorAdmission { pub max_artifact_download_ttl_seconds: u64, } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CoordinatorOperationalMetrics { + pub tenants: usize, + pub users: usize, + pub projects: usize, + pub enrolled_nodes: usize, + pub reported_nodes: usize, + pub live_nodes: usize, + pub active_processes: usize, + pub active_coordinator_mains: usize, + pub max_active_coordinator_mains: usize, + pub active_tasks: usize, + pub queued_tasks: usize, + pub artifacts: usize, + pub retained_download_links: usize, + pub relay: ArtifactRelayUsage, +} + impl Default for CoordinatorAdmission { fn default() -> Self { Self { @@ -151,6 +179,98 @@ pub enum CoordinatorServiceError { Durable(String), } +impl CoordinatorServiceError { + pub fn api_error(&self, request_id: impl Into) -> ApiError { + let request_id = request_id.into(); + let message = self.to_string(); + let (code, category, retryable) = match self { + Self::Io(_) => ( + ApiErrorCode::TemporaryCapacity, + ApiErrorCategory::Availability, + true, + ), + Self::Json(_) + | Self::CapabilityReport(_) + | Self::InvalidArtifactPath(_) + | Self::InvalidTaskLogTail(_) => ( + ApiErrorCode::ValidationError, + ApiErrorCategory::Validation, + false, + ), + Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => { + return ApiError::from_message(request_id, message); + } + Self::Coordinator(CoordinatorError::UnknownNode) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Coordinator(CoordinatorError::Enrollment(_)) => ( + ApiErrorCode::Unauthenticated, + ApiErrorCategory::Authentication, + false, + ), + Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => { + (ApiErrorCode::Conflict, ApiErrorCategory::State, true) + } + Self::Download(DownloadError::NotFound) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Download(DownloadError::Unavailable) + | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => ( + ApiErrorCode::ArtifactUnavailable, + ApiErrorCategory::Availability, + true, + ), + Self::Download(DownloadError::LimitExceeded { .. }) => ( + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCategory::Resource, + false, + ), + Self::Download(DownloadError::Unauthorized(_)) + | Self::Download(DownloadError::InvalidToken) + | Self::Download(DownloadError::Expired) + | Self::Download(DownloadError::Revoked) => ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ), + Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ), + Self::Scheduler(_) => ( + ApiErrorCode::NoCapableNode, + ApiErrorCategory::Availability, + true, + ), + Self::Transport(_) => ( + ApiErrorCode::NodeOffline, + ApiErrorCategory::Availability, + true, + ), + Self::Panel(PanelError::RateLimited) => ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ), + Self::Panel(PanelError::UnknownWidget(_)) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Panel(_) => ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ), + Self::Durable(_) => ( + ApiErrorCode::InternalError, + ApiErrorCategory::Internal, + true, + ), + }; + ApiError::new(code, category, message, retryable, request_id) + } +} + pub struct CoordinatorService { coordinator: Coordinator, store: RuntimeDurableStore, @@ -161,6 +281,22 @@ pub struct CoordinatorService { enrollment_grants: BTreeMap, task_events: VecDeque, process_scope_history: VecDeque, + process_summaries: BTreeMap, + process_summary_order: VecDeque, + next_process_summary_order: u64, + recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque>, + recent_log_dropped_through: BTreeMap, + recent_log_offsets: BTreeMap< + ( + TenantId, + ProjectId, + ProcessId, + clusterflux_core::TaskInstanceId, + String, + ), + u64, + >, + next_recent_log_sequence: u64, debug_audit_events: VecDeque, debug_epochs: BTreeMap, debug_epoch_runtime: BTreeMap, @@ -215,6 +351,29 @@ impl CoordinatorService { self.artifact_relay.usage() } + pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics { + CoordinatorOperationalMetrics { + tenants: self.coordinator.tenant_count(), + users: self.coordinator.user_count(), + projects: self.coordinator.project_count(), + enrolled_nodes: self.coordinator.node_identity_count(), + reported_nodes: self.node_descriptors.len(), + live_nodes: self + .node_descriptors + .keys() + .filter(|scope| self.node_is_live(scope)) + .count(), + active_processes: self.coordinator.active_process_count(), + active_coordinator_mains: self.main_runtime.active_main_count(), + max_active_coordinator_mains: self.main_runtime.max_active_mains(), + active_tasks: self.active_tasks.len(), + queued_tasks: self.pending_task_launches.len(), + artifacts: self.artifact_registry.artifact_count(), + retained_download_links: self.artifact_registry.retained_download_link_count(), + relay: self.artifact_relay.usage(), + } + } + fn commit_artifact_relay( &mut self, candidate: relay::ArtifactRelayLedger, @@ -404,6 +563,13 @@ impl CoordinatorService { enrollment_grants: BTreeMap::new(), task_events: VecDeque::new(), process_scope_history: VecDeque::new(), + process_summaries: BTreeMap::new(), + process_summary_order: VecDeque::new(), + next_process_summary_order: 1, + recent_logs: BTreeMap::new(), + recent_log_dropped_through: BTreeMap::new(), + recent_log_offsets: BTreeMap::new(), + next_recent_log_sequence: 1, debug_audit_events: VecDeque::new(), debug_epochs: BTreeMap::new(), debug_epoch_runtime: BTreeMap::new(), diff --git a/crates/clusterflux-coordinator/src/service/authorization.rs b/crates/clusterflux-coordinator/src/service/authorization.rs index 92c1e6d..f4d60fd 100644 --- a/crates/clusterflux-coordinator/src/service/authorization.rs +++ b/crates/clusterflux-coordinator/src/service/authorization.rs @@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation { RevokeAgentPublicKey, CreateNodeEnrollmentGrant, ListNodeDescriptors, + ListNodeSummaries, RevokeNodeCredential, StartProcess, ScheduleTask, @@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation { CancelProcess, AbortProcess, ListProcesses, + ListProcessSummaries, QuotaStatus, RestartTask, ResolveTaskFailure, @@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation { InspectDebugEpoch, ListTaskEvents, ListTaskSnapshots, + ListRecentLogs, JoinTask, + ListArtifacts, + GetArtifact, CreateArtifactDownloadLink, OpenArtifactDownloadStream, RevokeArtifactDownloadLink, @@ -56,6 +61,7 @@ impl PublicUserOperation { Self::RevokeAgentPublicKey => "revoke_agent_public_key", Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant", Self::ListNodeDescriptors => "list_node_descriptors", + Self::ListNodeSummaries => "list_node_summaries", Self::RevokeNodeCredential => "revoke_node_credential", Self::StartProcess => "start_process", Self::ScheduleTask => "schedule_task", @@ -63,6 +69,7 @@ impl PublicUserOperation { Self::CancelProcess => "cancel_process", Self::AbortProcess => "abort_process", Self::ListProcesses => "list_processes", + Self::ListProcessSummaries => "list_process_summaries", Self::QuotaStatus => "quota_status", Self::RestartTask => "restart_task", Self::ResolveTaskFailure => "resolve_task_failure", @@ -74,7 +81,10 @@ impl PublicUserOperation { Self::InspectDebugEpoch => "inspect_debug_epoch", Self::ListTaskEvents => "list_task_events", Self::ListTaskSnapshots => "list_task_snapshots", + Self::ListRecentLogs => "list_recent_logs", Self::JoinTask => "join_task", + Self::ListArtifacts => "list_artifacts", + Self::GetArtifact => "get_artifact", Self::CreateArtifactDownloadLink => "create_artifact_download_link", Self::OpenArtifactDownloadStream => "open_artifact_download_stream", Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link", @@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { Self::CreateNodeEnrollmentGrant } AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors, + AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries, AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => { Self::RevokeNodeCredential } @@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess, AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess, AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses, + AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => { + Self::ListProcessSummaries + } AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus, AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask, AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure, @@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch, AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents, AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots, + AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs, AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask, + AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts, + AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact, AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => { Self::CreateArtifactDownloadLink } diff --git a/crates/clusterflux-coordinator/src/service/logs.rs b/crates/clusterflux-coordinator/src/service/logs.rs index 957e475..1c84a3b 100644 --- a/crates/clusterflux-coordinator/src/service/logs.rs +++ b/crates/clusterflux-coordinator/src/service/logs.rs @@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key}; use super::protocol::TaskAttemptState; use super::{ artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError, - TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES, + RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState, + MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES, + MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT, + MAX_TASK_LOG_TAIL_BYTES, }; impl CoordinatorService { @@ -42,6 +45,34 @@ impl CoordinatorService { .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; self.quota .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; + let stdout_key = + recent_log_offset_key(&tenant, &project, &process, &task, &TaskLogStream::Stdout); + let stderr_key = + recent_log_offset_key(&tenant, &project, &process, &task, &TaskLogStream::Stderr); + if !stdout_tail.is_empty() && !self.recent_log_offsets.contains_key(&stdout_key) { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + TaskLogStream::Stdout, + stdout_tail.clone(), + stdout_truncated, + now_epoch_seconds, + ); + } + if !stderr_tail.is_empty() && !self.recent_log_offsets.contains_key(&stderr_key) { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + TaskLogStream::Stderr, + stderr_tail.clone(), + stderr_truncated, + now_epoch_seconds, + ); + } Ok(CoordinatorResponse::TaskLogRecorded { process, task, @@ -61,6 +92,98 @@ impl CoordinatorService { }) } + #[allow(clippy::too_many_arguments)] + pub(super) fn handle_report_task_log_chunk( + &mut self, + tenant: String, + project: String, + process: String, + node: String, + task: String, + stream: TaskLogStream, + offset: u64, + source_bytes: u64, + text: String, + truncated: bool, + ) -> Result { + if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { + return Err(CoordinatorServiceError::InvalidTaskLogTail(format!( + "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}", + text.len() + ))); + } + if source_bytes == 0 && !text.is_empty() && !truncated { + return Err(CoordinatorServiceError::Protocol( + "live log chunk source_bytes must describe non-empty text".to_owned(), + )); + } + if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) { + return Err(CoordinatorServiceError::Protocol( + "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(), + )); + } + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let process = ProcessId::new(process); + let node = NodeId::new(node); + let task = TaskInstanceId::new(task); + self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; + let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream); + let expected = self.recent_log_offsets.get(&key).copied().unwrap_or(0); + let end = offset.checked_add(source_bytes).ok_or_else(|| { + CoordinatorServiceError::Protocol( + "live log chunk offset exceeds the supported range".to_owned(), + ) + })?; + if end <= expected { + return Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence: None, + next_offset: expected, + }); + } + let now_epoch_seconds = self.current_epoch_seconds()?; + if offset > expected { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!("[log output lost: {} bytes]", offset - expected), + true, + now_epoch_seconds, + ); + } else if offset < expected { + return Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence: None, + next_offset: expected, + }); + } + let sequence = (!text.is_empty()).then(|| { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream, + text, + truncated, + now_epoch_seconds, + ) + }); + self.recent_log_offsets.insert(key, end); + Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence, + next_offset: end, + }) + } + pub(super) fn handle_report_vfs_metadata( &mut self, tenant: String, @@ -221,6 +344,12 @@ impl CoordinatorService { self.task_aborts.remove(&task_key); self.debug_commands.remove(&task_key); self.active_tasks.remove(&task_key); + self.clear_recent_log_offsets_for_task( + &event.tenant, + &event.project, + &event.process, + &event.task, + ); self.task_assignments.retain(|_, assignments| { assignments.retain(|assignment| { assignment.tenant != event.tenant @@ -320,7 +449,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::TaskSnapshots { snapshots }) } - fn authorize_task_event_process_scope( + pub(super) fn authorize_task_event_process_scope( &self, tenant: &TenantId, project: &ProjectId, @@ -360,6 +489,47 @@ impl CoordinatorService { Ok(()) } + pub(super) fn handle_list_recent_logs( + &mut self, + tenant: String, + project: String, + actor_user: String, + process: String, + task: Option, + after_sequence: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let process = ProcessId::new(process); + let task = task.map(TaskInstanceId::new); + self.authorize_task_event_process_scope(&tenant, &project, &process)?; + let after_sequence = after_sequence.unwrap_or(0); + let retained = self.recent_logs.get(&(tenant.clone(), project.clone())); + let history_truncated = self + .recent_log_dropped_through + .get(&process_control_key(&tenant, &project, &process)) + .is_some_and(|dropped_through| *dropped_through > after_sequence); + let entries = retained + .into_iter() + .flatten() + .filter(|entry| { + entry.process == process + && entry.sequence > after_sequence + && task.as_ref().is_none_or(|task| &entry.task == task) + }) + .take(limit as usize) + .cloned() + .collect::>(); + let next_sequence = entries.last().map(|entry| entry.sequence); + Ok(CoordinatorResponse::RecentLogs { + entries, + next_sequence, + history_truncated, + }) + } + pub(super) fn handle_join_task( &mut self, tenant: String, @@ -515,6 +685,94 @@ impl CoordinatorService { self.task_events.push_back(event); } + #[allow(clippy::too_many_arguments)] + fn record_recent_log( + &mut self, + tenant: TenantId, + project: ProjectId, + process: ProcessId, + task: TaskInstanceId, + stream: TaskLogStream, + mut text: String, + mut truncated: bool, + server_timestamp_epoch_seconds: u64, + ) -> u64 { + if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { + let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES; + while !text.is_char_boundary(boundary) { + boundary -= 1; + } + text.truncate(boundary); + truncated = true; + } + let sequence = self.next_recent_log_sequence; + self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1); + let logs = self + .recent_logs + .entry((tenant.clone(), project.clone())) + .or_default(); + let mut dropped = Vec::new(); + while logs.iter().filter(|entry| entry.process == process).count() + >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS + { + let Some(index) = logs.iter().position(|entry| entry.process == process) else { + break; + }; + if let Some(entry) = logs.remove(index) { + dropped.push(entry); + } + } + while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT + || logs + .iter() + .map(|entry| entry.text.len()) + .sum::() + .saturating_add(text.len()) + > MAX_RECENT_LOG_BYTES_PER_PROJECT + { + match logs.pop_front() { + Some(entry) => dropped.push(entry), + None => break, + } + } + logs.push_back(RecentLogEntry { + sequence, + process, + task, + stream, + text, + server_timestamp_epoch_seconds, + truncated, + }); + for entry in dropped { + let key = process_control_key(&tenant, &project, &entry.process); + self.recent_log_dropped_through + .entry(key) + .and_modify(|dropped_through| { + *dropped_through = (*dropped_through).max(entry.sequence); + }) + .or_insert(entry.sequence); + } + sequence + } + + pub(super) fn clear_recent_log_offsets_for_task( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + ) { + self.recent_log_offsets.retain( + |(entry_tenant, entry_project, entry_process, entry_task, _), _| { + entry_tenant != tenant + || entry_project != project + || entry_process != process + || entry_task != task + }, + ); + } + fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool { let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task); let Some(attempt) = self @@ -619,6 +877,25 @@ impl CoordinatorService { { return Ok(false); } + let final_result = if cancellation_completed { + super::ProcessFinalResult::Cancelled + } else if self.task_events.iter().any(|event| { + &event.tenant == tenant + && &event.project == project + && &event.process == process + && event.terminal_state == TaskTerminalState::Failed + }) { + super::ProcessFinalResult::Failed + } else { + super::ProcessFinalResult::Completed + }; + self.record_process_terminal( + tenant, + project, + process, + final_result, + self.current_epoch_seconds()?, + ); self.coordinator.abort_process(tenant, project, process)?; self.clear_debug_state_for_process(tenant, project, process); self.clear_operator_panel_state(tenant, project, process); @@ -736,6 +1013,26 @@ fn checked_reported_log_bytes( }) } +fn recent_log_offset_key( + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + stream: &TaskLogStream, +) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) { + ( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + match stream { + TaskLogStream::Stdout => "stdout", + TaskLogStream::Stderr => "stderr", + } + .to_owned(), + ) +} + fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> { if value.len() > MAX_TASK_LOG_TAIL_BYTES { return Err(CoordinatorServiceError::InvalidTaskLogTail(format!( diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index 735d2f1..30b81a8 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime { } impl CoordinatorMainRuntime { + pub(super) fn active_main_count(&self) -> usize { + self.controls.len() + } + + pub(super) fn max_active_mains(&self) -> usize { + self.max_active_mains + } + pub(super) fn configure( &mut self, configuration: super::CoordinatorMainRuntimeConfiguration, @@ -729,6 +737,13 @@ impl CoordinatorService { &process, "coordinator main launch failed admission or validation", ); + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Failed, + self.liveness_now_epoch_seconds(), + ); let _ = self.coordinator.abort_process(&tenant, &project, &process); } result @@ -992,6 +1007,13 @@ impl CoordinatorService { } } self.process_aborts.insert(process_key.clone()); + self.record_process_terminal( + &scope.tenant, + &scope.project, + &scope.process, + super::ProcessFinalResult::Failed, + self.liveness_now_epoch_seconds(), + ); let _ = self .coordinator .abort_process(&scope.tenant, &scope.project, &scope.process); diff --git a/crates/clusterflux-coordinator/src/service/processes.rs b/crates/clusterflux-coordinator/src/service/processes.rs index 54f2f32..ec6ca29 100644 --- a/crates/clusterflux-coordinator/src/service/processes.rs +++ b/crates/clusterflux-coordinator/src/service/processes.rs @@ -392,11 +392,12 @@ impl CoordinatorService { event.tenant != tenant || event.project != project || event.process != process }); let active = self.coordinator.start_process_for_launch_attempt( - tenant, - project, + tenant.clone(), + project.clone(), process.clone(), launch_attempt.map(clusterflux_core::LaunchAttemptId::new), ); + self.record_process_started(&tenant, &project, &process, now_epoch_seconds); Ok(CoordinatorResponse::ProcessStarted { process, launch_attempt: active @@ -513,6 +514,13 @@ impl CoordinatorService { } let process_key = process_control_key(&tenant, &project, &process); if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) { + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Cancelled, + self.current_epoch_seconds()?, + ); self.coordinator .abort_process(&tenant, &project, &process)?; self.clear_operator_panel_state(&tenant, &project, &process); @@ -598,6 +606,13 @@ impl CoordinatorService { } } + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Cancelled, + self.current_epoch_seconds()?, + ); if let Some(launch_attempt) = launch_attempt.as_ref() { self.coordinator.abort_process_for_launch_attempt( &tenant, diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index edaa988..61faf98 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -150,6 +150,15 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, + ListNodeSummaries { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, RevokeNodeCredential { tenant: String, project: String, @@ -303,6 +312,15 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, + ListProcessSummaries { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, QuotaStatus { tenant: String, project: String, @@ -429,6 +447,19 @@ pub enum CoordinatorRequest { stderr_truncated: bool, backpressured: bool, }, + ReportTaskLogChunk { + tenant: String, + project: String, + process: String, + node: String, + task: String, + stream: TaskLogStream, + offset: u64, + source_bytes: u64, + text: String, + #[serde(default)] + truncated: bool, + }, ReportVfsMetadata { tenant: String, project: String, @@ -478,6 +509,18 @@ pub enum CoordinatorRequest { actor_user: String, process: String, }, + ListRecentLogs { + tenant: String, + project: String, + actor_user: String, + process: String, + #[serde(default)] + task: Option, + #[serde(default)] + after_sequence: Option, + #[serde(default = "default_log_page_limit")] + limit: u32, + }, JoinTask { tenant: String, project: String, @@ -510,6 +553,23 @@ pub enum CoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, + ListArtifacts { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + process: Option, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, + GetArtifact { + tenant: String, + project: String, + actor_user: String, + artifact: String, + }, OpenArtifactDownloadStream { tenant: String, project: String, @@ -666,6 +726,18 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } + CoordinatorRequest::ListNodeSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::ExchangeNodeEnrollmentGrant { tenant, project, @@ -901,6 +973,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res task, .. } + | CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + process, + node, + task, + .. + } | CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -979,6 +1059,23 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process")) } + CoordinatorRequest::ListRecentLogs { + tenant, + project, + actor_user, + process, + task, + limit, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process"))?; + if let Some(task) = task { + validate_task_instance(task, &format!("{path}.task"))?; + } + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::AbortProcess { tenant, project, @@ -1007,6 +1104,18 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } + CoordinatorRequest::ListProcessSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 100) + } CoordinatorRequest::RestartTask { tenant, project, @@ -1080,6 +1189,20 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_process(process, &format!("{path}.process"))?; validate_external_token(widget_id, &format!("{path}.widget_id"), 256) } + CoordinatorRequest::ListArtifacts { + tenant, + project, + actor_user, + process, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_process(process.as_deref(), &format!("{path}.process"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::CreateArtifactDownloadLink { tenant, project, @@ -1100,6 +1223,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res actor_user, artifact, .. + } + | CoordinatorRequest::GetArtifact { + tenant, + project, + actor_user, + artifact, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; @@ -1133,6 +1262,14 @@ fn validate_authenticated_request( | AuthenticatedCoordinatorRequest::ListNodeDescriptors | AuthenticatedCoordinatorRequest::ListProcesses | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), + AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => { + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } + AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => { + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 100) + } AuthenticatedCoordinatorRequest::CreateProject { project, .. } | AuthenticatedCoordinatorRequest::SelectProject { project } => { validate_project(project, &format!("{path}.project")) @@ -1188,6 +1325,18 @@ fn validate_authenticated_request( | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { validate_process(process, &format!("{path}.process")) } + AuthenticatedCoordinatorRequest::ListRecentLogs { + process, + task, + limit, + .. + } => { + validate_process(process, &format!("{path}.process"))?; + if let Some(task) = task { + validate_task_instance(task, &format!("{path}.task"))?; + } + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { @@ -1205,9 +1354,19 @@ fn validate_authenticated_request( AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { validate_optional_process(process.as_deref(), &format!("{path}.process")) } + AuthenticatedCoordinatorRequest::ListArtifacts { + process, + cursor, + limit, + } => { + validate_optional_process(process.as_deref(), &format!("{path}.process"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } - | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } => { + | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } + | AuthenticatedCoordinatorRequest::GetArtifact { artifact } => { validate_artifact(artifact, &format!("{path}.artifact")) } AuthenticatedCoordinatorRequest::ExportArtifactToNode { @@ -1362,6 +1521,19 @@ fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), Stri value.map_or(Ok(()), |value| validate_process(value, path)) } +fn validate_optional_cursor(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_external_token(value, path, 256)) +} + +fn validate_page_limit(value: u32, path: &str, maximum: u32) -> Result<(), String> { + if value == 0 || value > maximum { + return Err(format!( + "malformed pagination limit {path}: expected 1 through {maximum}, received {value}" + )); + } + Ok(()) +} + fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) } @@ -1686,6 +1858,12 @@ pub enum AuthenticatedCoordinatorRequest { ttl_seconds: u64, }, ListNodeDescriptors, + ListNodeSummaries { + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, RevokeNodeCredential { node: String, }, @@ -1722,6 +1900,12 @@ pub enum AuthenticatedCoordinatorRequest { launch_attempt: Option, }, ListProcesses, + ListProcessSummaries { + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, QuotaStatus, RestartTask { process: String, @@ -1766,6 +1950,15 @@ pub enum AuthenticatedCoordinatorRequest { ListTaskSnapshots { process: String, }, + ListRecentLogs { + process: String, + #[serde(default)] + task: Option, + #[serde(default)] + after_sequence: Option, + #[serde(default = "default_log_page_limit")] + limit: u32, + }, JoinTask { process: String, task: String, @@ -1776,6 +1969,17 @@ pub enum AuthenticatedCoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, + ListArtifacts { + #[serde(default)] + process: Option, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, + GetArtifact { + artifact: String, + }, OpenArtifactDownloadStream { artifact: String, max_bytes: u64, @@ -1798,6 +2002,14 @@ fn default_download_ttl_seconds() -> u64 { 900 } +fn default_page_limit() -> u32 { + 50 +} + +fn default_log_page_limit() -> u32 { + 100 +} + fn default_node_enrollment_ttl_seconds() -> u64 { 900 } diff --git a/crates/clusterflux-coordinator/src/service/protocol/responses.rs b/crates/clusterflux-coordinator/src/service/protocol/responses.rs index b99bc21..8245536 100644 --- a/crates/clusterflux-coordinator/src/service/protocol/responses.rs +++ b/crates/clusterflux-coordinator/src/service/protocol/responses.rs @@ -183,6 +183,121 @@ pub struct VirtualProcessStatus { pub coordinator_epoch: u64, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeSummary { + pub id: NodeId, + pub display_name: String, + pub online: bool, + pub stale: bool, + pub last_seen_epoch_seconds: Option, + pub capabilities: NodeCapabilities, + pub direct_connectivity: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessLifecycleState { + Active, + RecentTerminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessActivityState { + Running, + WaitingForNode, + WaitingForTask, + AwaitingAction, + DebugEpochPartial, + Cancelling, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessFinalResult { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochSummary { + pub epoch: u64, + pub command: String, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSummary { + pub process: ProcessId, + pub lifecycle: ProcessLifecycleState, + pub activity: ProcessActivityState, + pub main_wait_state: Option, + pub started_at_epoch_seconds: u64, + pub ended_at_epoch_seconds: Option, + pub final_result: Option, + pub connected_nodes: Vec, + pub current_debug_epoch: Option, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactAvailability { + Available, + NodeOffline, + Unavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactRetentionState { + NodeRetained, + ExplicitStorage, + Lost, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactSummary { + pub id: ArtifactId, + pub display_path: String, + pub display_name: String, + pub process: ProcessId, + pub producer_task: TaskInstanceId, + pub safe_node: Option, + pub digest: Digest, + pub size_bytes: u64, + pub availability: ArtifactAvailability, + pub downloadable_now: bool, + pub retention_state: ArtifactRetentionState, + pub explicit_storage: bool, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskLogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogEntry { + pub sequence: u64, + pub process: ProcessId, + pub task: TaskInstanceId, + pub stream: TaskLogStream, + pub text: String, + pub server_timestamp_epoch_seconds: u64, + pub truncated: bool, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum SourcePreparationDisposition { Pending { reason: String }, @@ -282,6 +397,11 @@ pub enum CoordinatorResponse { descriptors: Vec, actor: UserId, }, + NodeSummaries { + nodes: Vec, + next_cursor: Option, + actor: UserId, + }, NodeCredentialRevoked { node: NodeId, tenant: TenantId, @@ -373,6 +493,11 @@ pub enum CoordinatorResponse { processes: Vec, actor: UserId, }, + ProcessSummaries { + processes: Vec, + next_cursor: Option, + actor: UserId, + }, QuotaStatus { tenant: TenantId, project: ProjectId, @@ -483,6 +608,17 @@ pub enum CoordinatorResponse { stderr_tail: String, backpressured: bool, }, + TaskLogChunkRecorded { + process: ProcessId, + task: TaskInstanceId, + sequence: Option, + next_offset: u64, + }, + RecentLogs { + entries: Vec, + next_sequence: Option, + history_truncated: bool, + }, VfsMetadataRecorded { process: ProcessId, task: TaskInstanceId, @@ -519,6 +655,13 @@ pub enum CoordinatorResponse { ArtifactDownloadLink { link: DownloadLink, }, + Artifacts { + artifacts: Vec, + next_cursor: Option, + }, + Artifact { + artifact: ArtifactSummary, + }, ArtifactDownloadLinkRevoked { link: DownloadLink, }, @@ -543,6 +686,24 @@ pub enum CoordinatorResponse { artifact_size_bytes: u64, }, Error { - message: String, + #[serde(flatten)] + error: clusterflux_core::ApiError, }, } + +impl CoordinatorResponse { + pub fn error(request_id: impl Into, message: impl Into) -> Self { + Self::Error { + error: clusterflux_core::ApiError::from_message(request_id, message), + } + } + + pub fn service_error( + request_id: impl Into, + error: &crate::service::CoordinatorServiceError, + ) -> Self { + Self::Error { + error: error.api_error(request_id), + } + } +} diff --git a/crates/clusterflux-coordinator/src/service/relay.rs b/crates/clusterflux-coordinator/src/service/relay.rs index cbe92dd..b79db3d 100644 --- a/crates/clusterflux-coordinator/src/service/relay.rs +++ b/crates/clusterflux-coordinator/src/service/relay.rs @@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage { pub egress_bytes: u64, pub abandoned_or_failed_bytes: u64, pub reserved_bytes: u64, + pub lifetime_ingress_bytes: u64, + pub lifetime_egress_bytes: u64, + pub lifetime_abandoned_or_failed_bytes: u64, + pub completed_transfers: u64, + pub failed_transfers: u64, + pub cancelled_transfers: u64, + pub expired_transfers: u64, + pub tracked_scopes: usize, + pub max_active_global: usize, + pub max_tracked_scopes: usize, } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState { pub ingress_used: u64, pub egress_used: u64, pub abandoned_or_failed_used: u64, + #[serde(default)] + pub lifetime_ingress_bytes: u64, + #[serde(default)] + pub lifetime_egress_bytes: u64, + #[serde(default)] + pub lifetime_abandoned_or_failed_bytes: u64, + #[serde(default)] + pub completed_transfers: u64, + #[serde(default)] + pub failed_transfers: u64, + #[serde(default)] + pub cancelled_transfers: u64, + #[serde(default)] + pub expired_transfers: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger { ingress_used: u64, egress_used: u64, abandoned_or_failed_used: u64, + lifetime_ingress_bytes: u64, + lifetime_egress_bytes: u64, + lifetime_abandoned_or_failed_bytes: u64, + completed_transfers: u64, + failed_transfers: u64, + cancelled_transfers: u64, + expired_transfers: u64, } impl Default for ArtifactRelayLedger { @@ -165,6 +196,13 @@ impl ArtifactRelayLedger { ingress_used: 0, egress_used: 0, abandoned_or_failed_used: 0, + lifetime_ingress_bytes: 0, + lifetime_egress_bytes: 0, + lifetime_abandoned_or_failed_bytes: 0, + completed_transfers: 0, + failed_transfers: 0, + cancelled_transfers: 0, + expired_transfers: 0, } } @@ -218,6 +256,13 @@ impl ArtifactRelayLedger { ingress_used: state.ingress_used, egress_used: state.egress_used, abandoned_or_failed_used: state.abandoned_or_failed_used, + lifetime_ingress_bytes: state.lifetime_ingress_bytes, + lifetime_egress_bytes: state.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes, + completed_transfers: state.completed_transfers, + failed_transfers: state.failed_transfers, + cancelled_transfers: state.cancelled_transfers, + expired_transfers: state.expired_transfers, } } @@ -260,6 +305,13 @@ impl ArtifactRelayLedger { ingress_used: self.ingress_used, egress_used: self.egress_used, abandoned_or_failed_used: self.abandoned_or_failed_used, + lifetime_ingress_bytes: self.lifetime_ingress_bytes, + lifetime_egress_bytes: self.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, + completed_transfers: self.completed_transfers, + failed_transfers: self.failed_transfers, + cancelled_transfers: self.cancelled_transfers, + expired_transfers: self.expired_transfers, } } @@ -491,9 +543,11 @@ impl ArtifactRelayLedger { if ingress { reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes); self.ingress_used = self.ingress_used.saturating_add(bytes); + self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes); } else { reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes); self.egress_used = self.egress_used.saturating_add(bytes); + self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes); } let project_key = ( reservation.scope.tenant.clone(), @@ -554,10 +608,29 @@ impl ArtifactRelayLedger { pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) { if let Some(reservation) = self.reservations.remove(key) { if reason != RelayFinishReason::Completed { + let abandoned_or_failed_bytes = reservation + .ingress_bytes + .saturating_add(reservation.egress_bytes); self.abandoned_or_failed_used = self .abandoned_or_failed_used - .saturating_add(reservation.ingress_bytes) - .saturating_add(reservation.egress_bytes); + .saturating_add(abandoned_or_failed_bytes); + self.lifetime_abandoned_or_failed_bytes = self + .lifetime_abandoned_or_failed_bytes + .saturating_add(abandoned_or_failed_bytes); + } + match reason { + RelayFinishReason::Completed => { + self.completed_transfers = self.completed_transfers.saturating_add(1); + } + RelayFinishReason::Failed => { + self.failed_transfers = self.failed_transfers.saturating_add(1); + } + RelayFinishReason::Cancelled => { + self.cancelled_transfers = self.cancelled_transfers.saturating_add(1); + } + RelayFinishReason::Expired => { + self.expired_transfers = self.expired_transfers.saturating_add(1); + } } } } @@ -580,6 +653,18 @@ impl ArtifactRelayLedger { ingress_bytes: self.ingress_used, egress_bytes: self.egress_used, abandoned_or_failed_bytes: self.abandoned_or_failed_used, + lifetime_ingress_bytes: self.lifetime_ingress_bytes, + lifetime_egress_bytes: self.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, + completed_transfers: self.completed_transfers, + failed_transfers: self.failed_transfers, + cancelled_transfers: self.cancelled_transfers, + expired_transfers: self.expired_transfers, + tracked_scopes: self.project_used.len() + + self.tenant_used.len() + + self.account_used.len(), + max_active_global: self.configuration.max_active_global, + max_tracked_scopes: self.configuration.max_tracked_scopes, reserved_bytes: self .reservations .values() @@ -588,3 +673,68 @@ impl ArtifactRelayLedger { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() { + let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited()); + ledger + .reserve( + "transfer".to_owned(), + TenantId::from("tenant"), + ProjectId::from("project"), + UserId::from("user"), + 16, + 16, + 100, + 1, + ) + .unwrap(); + ledger.charge_ingress("transfer", 24, 1).unwrap(); + ledger.charge_egress("transfer", 24, 1).unwrap(); + ledger.finish("transfer", RelayFinishReason::Completed); + + let usage = ledger.usage(); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + assert_eq!(usage.completed_transfers, 1); + + ledger.prepare_period(u64::MAX); + let usage = ledger.usage(); + assert_eq!(usage.ingress_bytes, 0); + assert_eq!(usage.egress_bytes, 0); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + + let restored = ArtifactRelayLedger::from_durable( + ArtifactRelayConfiguration::unlimited(), + ledger.durable_state(), + ); + let usage = restored.usage(); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + assert_eq!(usage.completed_transfers, 1); + } + + #[test] + fn older_durable_relay_state_defaults_new_metrics() { + let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({ + "reservations": {}, + "period": 1, + "project_used": [], + "tenant_used": [], + "account_used": [], + "ingress_used": 3, + "egress_used": 4, + "abandoned_or_failed_used": 2 + })) + .unwrap(); + + assert_eq!(state.lifetime_ingress_bytes, 0); + assert_eq!(state.lifetime_egress_bytes, 0); + assert_eq!(state.completed_transfers, 0); + } +} diff --git a/crates/clusterflux-coordinator/src/service/routing.rs b/crates/clusterflux-coordinator/src/service/routing.rs index aecaeef..e05a670 100644 --- a/crates/clusterflux-coordinator/src/service/routing.rs +++ b/crates/clusterflux-coordinator/src/service/routing.rs @@ -298,6 +298,13 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_node_descriptors(tenant, project, actor_user), + CoordinatorRequest::ListNodeSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::RevokeNodeCredential { tenant, project, @@ -421,6 +428,13 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_processes(tenant, project, actor_user), + CoordinatorRequest::ListProcessSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::QuotaStatus { tenant, project, @@ -466,7 +480,8 @@ impl CoordinatorService { CoordinatorRequest::PollDebugCommand { .. } | CoordinatorRequest::ReportDebugState { .. } | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(), - CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(), + CoordinatorRequest::ReportTaskLog { .. } + | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ListTaskEvents { @@ -481,6 +496,23 @@ impl CoordinatorService { actor_user, process, } => self.handle_list_task_snapshots(tenant, project, actor_user, process), + CoordinatorRequest::ListRecentLogs { + tenant, + project, + actor_user, + process, + task, + after_sequence, + limit, + } => self.handle_list_recent_logs( + tenant, + project, + actor_user, + process, + task, + after_sequence, + limit, + ), CoordinatorRequest::JoinTask { tenant, project, @@ -527,6 +559,20 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), + CoordinatorRequest::ListArtifacts { + tenant, + project, + actor_user, + process, + cursor, + limit, + } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit), + CoordinatorRequest::GetArtifact { + tenant, + project, + actor_user, + artifact, + } => self.handle_get_artifact(tenant, project, actor_user, artifact), CoordinatorRequest::OpenArtifactDownloadStream { tenant, project, @@ -696,6 +742,14 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), + AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self + .handle_list_node_summaries( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + cursor, + limit, + ), AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self .handle_revoke_node_credential( context.tenant.as_str().to_owned(), @@ -747,6 +801,14 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), + AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self + .handle_list_process_summaries( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + cursor, + limit, + ), AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -802,6 +864,20 @@ impl CoordinatorService { actor.as_str().to_owned(), process, ), + AuthenticatedCoordinatorRequest::ListRecentLogs { + process, + task, + after_sequence, + limit, + } => self.handle_list_recent_logs( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + process, + task, + after_sequence, + limit, + ), AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -821,6 +897,24 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), + AuthenticatedCoordinatorRequest::ListArtifacts { + process, + cursor, + limit, + } => self.handle_list_artifacts( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + process, + cursor, + limit, + ), + AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + artifact, + ), AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, max_bytes, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index 19384df..40f633a 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -253,6 +253,29 @@ impl CoordinatorService { stderr_truncated, backpressured, ), + CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + process, + node, + task, + stream, + offset, + source_bytes, + text, + truncated, + } => self.handle_report_task_log_chunk( + tenant, + project, + process, + node, + task, + stream, + offset, + source_bytes, + text, + truncated, + ), CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -346,6 +369,7 @@ fn signed_node_request_kind( CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"), CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"), CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"), + CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"), CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"), CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"), _ => Err(CoordinatorError::Unauthorized( @@ -441,6 +465,12 @@ fn signed_node_request_scope( node, .. } + | CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + node, + .. + } | CoordinatorRequest::ReportVfsMetadata { tenant, project, diff --git a/crates/clusterflux-coordinator/src/service/summaries.rs b/crates/clusterflux-coordinator/src/service/summaries.rs new file mode 100644 index 0000000..c8dc59a --- /dev/null +++ b/crates/clusterflux-coordinator/src/service/summaries.rs @@ -0,0 +1,500 @@ +use std::collections::BTreeSet; +use std::time::Instant; + +use clusterflux_core::{ + ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TenantId, UserId, +}; + +use super::keys::{process_control_key, ProcessControlKey}; +use super::{ + ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse, + CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary, + NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, + TaskAttemptState, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct StoredProcessSummary { + pub(super) started_at_epoch_seconds: u64, + pub(super) ended_at_epoch_seconds: Option, + pub(super) final_result: Option, + pub(super) connected_nodes: Vec, + pub(super) order: u64, +} + +impl CoordinatorService { + pub(super) fn handle_list_node_summaries( + &mut self, + tenant: String, + project: String, + actor_user: String, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let actor = UserId::new(actor_user); + let cursor = cursor.as_deref(); + let mut nodes = self + .node_descriptors + .iter() + .filter(|(scope, descriptor)| { + scope.tenant == tenant + && scope.project == project + && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor) + }) + .map(|(scope, descriptor)| { + let online = self.node_is_live(scope); + NodeSummary { + id: descriptor.id.clone(), + display_name: descriptor.id.as_str().to_owned(), + online, + stale: !online, + last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(), + capabilities: descriptor.capabilities.clone(), + direct_connectivity: descriptor.direct_connectivity, + } + }) + .collect::>(); + nodes.sort_by(|left, right| left.id.cmp(&right.id)); + let has_more = nodes.len() > limit as usize; + nodes.truncate(limit as usize); + let next_cursor = has_more + .then(|| nodes.last().map(|node| node.id.as_str().to_owned())) + .flatten(); + Ok(CoordinatorResponse::NodeSummaries { + nodes, + next_cursor, + actor, + }) + } + + pub(super) fn record_process_started( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + now_epoch_seconds: u64, + ) { + let key = process_control_key(tenant, project, process); + if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) { + logs.retain(|entry| &entry.process != process); + if logs.is_empty() { + self.recent_logs.remove(&(tenant.clone(), project.clone())); + } + } + self.recent_log_dropped_through.remove(&key); + self.recent_log_offsets + .retain(|(entry_tenant, entry_project, entry_process, _, _), _| { + entry_tenant != tenant || entry_project != project || entry_process != process + }); + self.process_summary_order + .retain(|retained| retained != &key); + self.evict_process_summaries_for_project(tenant, project); + self.evict_process_summaries_total(); + let order = self.next_process_summary_order; + self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); + self.process_summaries.insert( + key.clone(), + StoredProcessSummary { + started_at_epoch_seconds: now_epoch_seconds, + ended_at_epoch_seconds: None, + final_result: None, + connected_nodes: Vec::new(), + order, + }, + ); + self.process_summary_order.push_back(key); + } + + pub(super) fn record_process_terminal( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + final_result: ProcessFinalResult, + now_epoch_seconds: u64, + ) { + let key = process_control_key(tenant, project, process); + let connected_nodes = self + .coordinator + .active_process(tenant, project, process) + .map(|active| active.connected_nodes.iter().cloned().collect()) + .unwrap_or_default(); + let order = self.next_process_summary_order; + let entry = self + .process_summaries + .entry(key.clone()) + .or_insert_with(|| { + self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); + self.process_summary_order.push_back(key.clone()); + StoredProcessSummary { + started_at_epoch_seconds: now_epoch_seconds, + ended_at_epoch_seconds: None, + final_result: None, + connected_nodes: Vec::new(), + order, + } + }); + entry.ended_at_epoch_seconds = Some(now_epoch_seconds); + entry.final_result = Some(final_result); + entry.connected_nodes = connected_nodes; + } + + pub(super) fn handle_list_process_summaries( + &mut self, + tenant: String, + project: String, + actor_user: String, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let actor = UserId::new(actor_user); + let cursor = parse_order_cursor(cursor.as_deref(), "process")?; + let mut stored = self + .process_summaries + .iter() + .filter(|((entry_tenant, entry_project, _), summary)| { + entry_tenant == &tenant + && entry_project == &project + && cursor.is_none_or(|cursor| summary.order < cursor) + }) + .map(|(key, summary)| (key.clone(), summary.clone())) + .collect::>(); + stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order)); + let has_more = stored.len() > limit as usize; + stored.truncate(limit as usize); + let processes = stored + .into_iter() + .map(|(key, stored)| self.process_summary_from_stored(&key, stored)) + .collect::>(); + let next_cursor = has_more + .then(|| processes.last().map(|process| process.order_cursor.clone())) + .flatten(); + Ok(CoordinatorResponse::ProcessSummaries { + processes, + next_cursor, + actor, + }) + } + + fn process_summary_from_stored( + &self, + key: &ProcessControlKey, + stored: StoredProcessSummary, + ) -> ProcessSummary { + let (tenant, project, process) = key; + let active = self.coordinator.active_process(tenant, project, process); + let process_key = process_control_key(tenant, project, process); + let main_wait_state = active.and_then(|_| { + if self.pending_task_launches.iter().any(|pending| { + &pending.tenant == tenant + && &pending.project == project + && &pending.process == process + }) { + Some("waiting_for_node".to_owned()) + } else if self + .main_runtime + .is_waiting_for_task(tenant, project, process) + { + Some("waiting_for_task".to_owned()) + } else { + None + } + }); + let current_debug_epoch = self.debug_epoch_summary(&process_key); + let awaiting_action = self.task_attempts.iter().any( + |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { + attempt_tenant == tenant + && attempt_project == project + && attempt_process == process + && attempts.iter().any(|attempt| { + attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction + }) + }, + ); + let activity = if let Some(result) = &stored.final_result { + match result { + ProcessFinalResult::Completed => ProcessActivityState::Completed, + ProcessFinalResult::Failed => ProcessActivityState::Failed, + ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled, + } + } else if self.process_cancellations.contains(&process_key) { + ProcessActivityState::Cancelling + } else if current_debug_epoch + .as_ref() + .is_some_and(|epoch| epoch.partially_frozen) + { + ProcessActivityState::DebugEpochPartial + } else if awaiting_action { + ProcessActivityState::AwaitingAction + } else { + match main_wait_state.as_deref() { + Some("waiting_for_node") => ProcessActivityState::WaitingForNode, + Some("waiting_for_task") => ProcessActivityState::WaitingForTask, + _ => ProcessActivityState::Running, + } + }; + let connected_nodes = active + .map(|active| active.connected_nodes.iter().cloned().collect()) + .unwrap_or(stored.connected_nodes); + ProcessSummary { + process: process.clone(), + lifecycle: if active.is_some() { + ProcessLifecycleState::Active + } else { + ProcessLifecycleState::RecentTerminal + }, + activity, + main_wait_state, + started_at_epoch_seconds: stored.started_at_epoch_seconds, + ended_at_epoch_seconds: stored.ended_at_epoch_seconds, + final_result: stored.final_result, + connected_nodes, + current_debug_epoch, + order_cursor: format!("process:{}", stored.order), + } + } + + fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option { + let runtime = self.debug_epoch_runtime.get(key)?; + let acknowledgements = runtime.acknowledgements.values().collect::>(); + let all_acknowledged = !runtime.expected.is_empty() + && runtime + .expected + .iter() + .all(|participant| runtime.acknowledgements.contains_key(participant)); + let fully_frozen = runtime.command == "freeze" + && all_acknowledged + && acknowledgements + .iter() + .all(|ack| ack.state == DebugAcknowledgementState::Frozen); + let freeze_deadline_elapsed = + runtime.command == "freeze" && Instant::now() >= runtime.deadline; + let frozen_count = acknowledgements + .iter() + .filter(|ack| ack.state == DebugAcknowledgementState::Frozen) + .count(); + let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen; + let fully_resumed = runtime.command == "resume" + && all_acknowledged + && acknowledgements + .iter() + .all(|ack| ack.state == DebugAcknowledgementState::Running); + let failed = acknowledgements + .iter() + .any(|ack| ack.state == DebugAcknowledgementState::Failed) + || (freeze_deadline_elapsed + && runtime + .expected + .iter() + .any(|participant| !runtime.acknowledgements.contains_key(participant))); + Some(DebugEpochSummary { + epoch: runtime.epoch, + command: runtime.command.clone(), + fully_frozen, + partially_frozen, + fully_resumed, + failed, + }) + } + + pub(super) fn handle_list_artifacts( + &mut self, + tenant: String, + project: String, + actor_user: String, + process: Option, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let process = process.map(ProcessId::new); + if let Some(process) = &process { + self.authorize_task_event_process_scope(&tenant, &project, process)?; + } + let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?; + let mut metadata = self + .artifact_registry + .metadata_for_project(&tenant, &project) + .filter(|metadata| { + process + .as_ref() + .is_none_or(|process| &metadata.process == process) + && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor) + }) + .cloned() + .collect::>(); + metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch)); + let has_more = metadata.len() > limit as usize; + metadata.truncate(limit as usize); + let artifacts = metadata + .into_iter() + .map(|metadata| self.artifact_summary(metadata)) + .collect::>(); + let next_cursor = has_more + .then(|| { + artifacts + .last() + .map(|artifact| artifact.order_cursor.clone()) + }) + .flatten(); + Ok(CoordinatorResponse::Artifacts { + artifacts, + next_cursor, + }) + } + + pub(super) fn handle_get_artifact( + &mut self, + tenant: String, + project: String, + actor_user: String, + artifact: String, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let artifact = ArtifactId::new(artifact); + let metadata = self + .artifact_registry + .metadata(&tenant, &project, &artifact) + .cloned() + .ok_or(clusterflux_core::DownloadError::NotFound)?; + Ok(CoordinatorResponse::Artifact { + artifact: self.artifact_summary(metadata), + }) + } + + fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary { + let live_retaining_nodes = metadata + .retaining_nodes + .iter() + .filter(|node| { + self.node_is_live(&crate::NodeScopeKey::from_refs( + &metadata.tenant, + &metadata.project, + node, + )) + }) + .cloned() + .collect::>(); + let safe_node = live_retaining_nodes + .iter() + .next() + .cloned() + .or_else(|| metadata.retaining_nodes.iter().next().cloned()); + let explicit_storage = !metadata.explicit_locations.is_empty(); + let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage; + let availability = if downloadable_now { + ArtifactAvailability::Available + } else if !metadata.retaining_nodes.is_empty() { + ArtifactAvailability::NodeOffline + } else { + ArtifactAvailability::Unavailable + }; + let retention_state = if explicit_storage { + ArtifactRetentionState::ExplicitStorage + } else if !metadata.retaining_nodes.is_empty() { + ArtifactRetentionState::NodeRetained + } else { + ArtifactRetentionState::Lost + }; + let display_suffix = metadata.id.as_str().replace(':', "/"); + let display_path = format!("/vfs/artifacts/{display_suffix}"); + let display_name = display_suffix + .rsplit('/') + .next() + .unwrap_or(metadata.id.as_str()) + .to_owned(); + ArtifactSummary { + id: metadata.id, + display_path, + display_name, + process: metadata.process, + producer_task: metadata.producer_task, + safe_node, + digest: metadata.digest, + size_bytes: metadata.size, + availability, + downloadable_now, + retention_state, + explicit_storage, + order_cursor: format!("artifact:{}", metadata.flushed_epoch), + } + } + + fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) { + while self + .process_summaries + .keys() + .filter(|(entry_tenant, entry_project, _)| { + entry_tenant == tenant && entry_project == project + }) + .count() + >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT + { + let candidate = self.process_summary_order.iter().find(|key| { + &key.0 == tenant + && &key.1 == project + && self + .process_summaries + .get(*key) + .is_some_and(|summary| summary.final_result.is_some()) + }); + let Some(candidate) = candidate.cloned() else { + break; + }; + self.process_summaries.remove(&candidate); + self.recent_log_dropped_through.remove(&candidate); + self.process_summary_order + .retain(|retained| retained != &candidate); + } + } + + fn evict_process_summaries_total(&mut self) { + while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL { + let candidate = self.process_summary_order.iter().find(|key| { + self.process_summaries + .get(*key) + .is_some_and(|summary| summary.final_result.is_some()) + }); + let Some(candidate) = candidate.cloned() else { + break; + }; + self.process_summaries.remove(&candidate); + self.recent_log_dropped_through.remove(&candidate); + self.process_summary_order + .retain(|retained| retained != &candidate); + } + } +} + +fn parse_order_cursor( + cursor: Option<&str>, + expected_kind: &str, +) -> Result, CoordinatorServiceError> { + cursor + .map(|cursor| { + let (kind, order) = cursor.split_once(':').ok_or_else(|| { + CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + )) + })?; + if kind != expected_kind { + return Err(CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + ))); + } + order.parse::().map_err(|_| { + CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + )) + }) + }) + .transpose() +} diff --git a/crates/clusterflux-coordinator/src/service/tcp.rs b/crates/clusterflux-coordinator/src/service/tcp.rs index 0b42969..2b6e9c9 100644 --- a/crates/clusterflux-coordinator/src/service/tcp.rs +++ b/crates/clusterflux-coordinator/src/service/tcp.rs @@ -79,17 +79,15 @@ impl CoordinatorService { continue; } let response = match decode_wire_request(&line) { - Ok(request) => match authorize_client_request(&request, authority_mode) - .and_then(|()| self.handle_request(request)) - { - Ok(response) => response, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, + Ok((request_id, request)) => { + match authorize_client_request(&request, authority_mode) + .and_then(|()| self.handle_request(request)) + { + Ok(response) => response, + Err(err) => CoordinatorResponse::service_error(request_id, &err), + } + } + Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -114,25 +112,19 @@ fn handle_shared_stream( continue; } let response = match decode_wire_request(&line) { - Ok(request) => match authorize_client_request(&request, authority_mode) { + Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) { Ok(()) => match service.lock() { Ok(mut service) => match service.handle_request(request) { Ok(response) => response, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(_) => CoordinatorResponse::Error { - message: "coordinator service lock poisoned".to_owned(), + Err(err) => CoordinatorResponse::service_error(request_id, &err), }, + Err(_) => { + CoordinatorResponse::error(request_id, "coordinator service lock poisoned") + } }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), + Err(err) => CoordinatorResponse::service_error(request_id, &err), }, + Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato Ok((listener, addr)) } -fn decode_wire_request(line: &str) -> Result { +fn decode_wire_request( + line: &str, +) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> { serde_json::from_str::(line)? - .into_request() + .into_parts() .map_err(CoordinatorServiceError::Protocol) } +fn wire_request_id_hint(line: &str) -> String { + serde_json::from_str::(line) + .ok() + .and_then(|value| { + value + .get("request_id") + .and_then(|value| value.as_str()) + .map(str::to_owned) + }) + .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok()) + .unwrap_or_else(|| "unavailable".to_owned()) +} + fn authorize_client_request( request: &CoordinatorRequest, authority_mode: ClientAuthorityMode, @@ -180,10 +187,11 @@ fn authorize_client_request( } | CoordinatorRequest::AdminStatus { .. } | CoordinatorRequest::SuspendTenant { .. } => Ok(()), - _ => Err(CoordinatorServiceError::Protocol( + _ => Err(crate::CoordinatorError::Unauthorized( "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority" .to_owned(), - )), + ) + .into()), } } @@ -253,16 +261,19 @@ mod transport_boundary_tests { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { message } = + let CoordinatorResponse::Error { error } = serde_json::from_str::(&line).unwrap() else { panic!("malformed identifier request unexpectedly succeeded"); }; assert!( - message.contains("malformed external identifier") - && message.contains("request.request.process"), - "unexpected malformed identifier response: {message}" + error.message.contains("malformed external identifier") + && error.message.contains("request.request.process"), + "unexpected malformed identifier response: {}", + error.message ); + assert_eq!(error.request_id, format!("malformed-{index}")); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); let valid = coordinator_wire_request( format!("healthy-{index}"), diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index c20f42b..91fc7b3 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -829,6 +829,7 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { | CoordinatorRequest::ReportDebugState { node, .. } | CoordinatorRequest::ReportDebugProbeHit { node, .. } | CoordinatorRequest::ReportTaskLog { node, .. } + | CoordinatorRequest::ReportTaskLogChunk { node, .. } | CoordinatorRequest::ReportVfsMetadata { node, .. } | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), @@ -874,6 +875,9 @@ fn signed_node_request_auto_with_private_key( (node.clone(), "report_debug_probe_hit") } CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"), + CoordinatorRequest::ReportTaskLogChunk { node, .. } => { + (node.clone(), "report_task_log_chunk") + } CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"), CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"), _ => panic!("test helper only signs node-originated requests"), @@ -7850,12 +7854,16 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { message } = + let CoordinatorResponse::Error { error } = serde_json::from_str::(&line).unwrap() else { panic!("expected strict body-authority denial"); }; - assert!(message.contains("request-body identity fields are not authority")); + assert!(error + .message + .contains("request-body identity fields are not authority")); + assert_eq!(error.request_id, "strict-stream-forged"); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden); write_coordinator_wire_request( &mut stream, @@ -7911,10 +7919,14 @@ fn service_stream_rejects_invalid_versioned_envelope_metadata() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); let response = serde_json::from_str::(&line).unwrap(); - let CoordinatorResponse::Error { message } = response else { + let CoordinatorResponse::Error { error } = response else { panic!("expected invalid wire envelope response"); }; - assert!(message.contains("operation attach_node does not match payload operation ping")); + assert!(error + .message + .contains("operation attach_node does not match payload operation ping")); + assert_eq!(error.request_id, "bad-operation"); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); @@ -8011,3 +8023,697 @@ fn coordinator_generates_and_bounds_node_enrollment_grants() { assert_ne!(first_grant, second_grant); assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60); } + +#[test] +fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret) in [ + ("tenant-a", "project-a", "user-a", "session-a"), + ("tenant-b", "project-b", "user-b", "session-b"), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + } + service.set_server_time(100); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: "process-one".to_owned(), + restart: false, + }, + }) + .unwrap(); + + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected process summaries"); + }; + assert_eq!(processes.len(), 1); + assert_eq!(processes[0].process, ProcessId::from("process-one")); + assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active); + assert_eq!(processes[0].activity, ProcessActivityState::Running); + assert_eq!(processes[0].started_at_epoch_seconds, 100); + assert!(next_cursor.is_none()); + + let CoordinatorResponse::ProcessSummaries { processes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 10, + }, + }) + .unwrap() + else { + panic!("expected scoped process summaries"); + }; + assert!(processes.is_empty()); + + service.set_server_time(120); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::AbortProcess { + process: "process-one".to_owned(), + launch_attempt: None, + }, + }) + .unwrap(); + let CoordinatorResponse::ProcessSummaries { processes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 10, + }, + }) + .unwrap() + else { + panic!("expected terminal process summary"); + }; + assert_eq!( + processes[0].lifecycle, + ProcessLifecycleState::RecentTerminal + ); + assert_eq!(processes[0].activity, ProcessActivityState::Cancelled); + assert_eq!( + processes[0].final_result, + Some(ProcessFinalResult::Cancelled) + ); + assert_eq!(processes[0].ended_at_epoch_seconds, Some(120)); + + service.set_server_time(130); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: "process-two".to_owned(), + restart: false, + }, + }) + .unwrap(); + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor: Some(cursor), + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected first process summary page"); + }; + assert_eq!(processes[0].process, ProcessId::from("process-two")); + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor: None, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: Some(cursor), + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected final process summary page"); + }; + assert_eq!(processes[0].process, ProcessId::from("process-one")); + + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 101, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("100")); +} + +#[test] +fn web_node_summaries_are_scoped_paginated_and_hard_bounded() { + let mut service = CoordinatorService::new(7); + service + .issue_cli_session( + TenantId::from("tenant"), + ProjectId::from("project"), + UserId::from("user"), + "session", + None, + ) + .unwrap(); + for node in ["node-a", "node-b", "node-c"] { + enroll_test_node( + &mut service, + "tenant", + "project", + node, + &test_node_public_key(node), + ); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: node.to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: Vec::new(), + direct_connectivity: true, + online: true, + }) + .unwrap(); + } + + let CoordinatorResponse::NodeSummaries { + nodes, + next_cursor: Some(cursor), + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected first node-summary page"); + }; + assert_eq!( + nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + ["node-a", "node-b"] + ); + + let CoordinatorResponse::NodeSummaries { + nodes, + next_cursor: None, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: Some(cursor), + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected final node-summary page"); + }; + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].id, NodeId::from("node-c")); + + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 201, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("200")); +} + +#[test] +fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret, node) in [ + ("tenant-a", "project-a", "user-a", "session-a", "node-a"), + ("tenant-b", "project-b", "user-b", "session-b", "node-b"), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + enroll_test_node( + &mut service, + tenant, + project, + node, + &test_node_public_key(node), + ); + } + service.set_server_time(100); + for (tenant, project, node) in [ + ("tenant-a", "project-a", "node-a"), + ("tenant-b", "project-b", "node-b"), + ] { + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: vec!["shared-artifact".to_owned()], + direct_connectivity: true, + online: false, + }) + .unwrap(); + } + let CoordinatorResponse::NodeSummaries { nodes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected node summaries"); + }; + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].id, NodeId::from("node-a")); + assert!(nodes[0].online); + assert!(!nodes[0].stale); + assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); + assert_eq!(nodes[0].capabilities.os, Os::Linux); + for (tenant, project, node, digest) in [ + ("tenant-a", "project-a", "node-a", "tenant-a-bytes"), + ("tenant-b", "project-b", "node-b", "tenant-b-bytes"), + ] { + service.artifact_registry.flush_metadata(ArtifactFlush { + id: ArtifactId::from("shared-artifact"), + tenant: TenantId::from(tenant), + project: ProjectId::from(project), + process: ProcessId::from("process-one"), + producer_task: TaskInstanceId::from("task-one"), + retaining_node: NodeId::from(node), + digest: Digest::sha256(digest), + size: digest.len() as u64, + }); + } + service.artifact_registry.flush_metadata(ArtifactFlush { + id: ArtifactId::from("second-artifact"), + tenant: TenantId::from("tenant-a"), + project: ProjectId::from("project-a"), + process: ProcessId::from("process-two"), + producer_task: TaskInstanceId::from("task-two"), + retaining_node: NodeId::from("node-a"), + digest: Digest::sha256("second"), + size: 6, + }); + + let CoordinatorResponse::Artifacts { + artifacts, + next_cursor: Some(cursor), + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListArtifacts { + process: None, + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected first artifact page"); + }; + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact")); + assert_eq!(artifacts[0].availability, ArtifactAvailability::Available); + let CoordinatorResponse::Artifacts { + artifacts, + next_cursor: None, + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListArtifacts { + process: None, + cursor: Some(cursor), + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected final artifact page"); + }; + assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact")); + assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes")); + + let cross_tenant = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "tenant-b-only".to_owned(), + }, + }) + .unwrap_err(); + assert!(cross_tenant.to_string().contains("does not exist")); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected tenant-b artifact"); + }; + assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes")); + + service.set_server_time(131); + let CoordinatorResponse::NodeSummaries { nodes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected stale node summary"); + }; + assert!(!nodes[0].online); + assert!(nodes[0].stale); + assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected offline artifact metadata"); + }; + assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline); + assert!(!artifact.downloadable_now); + + service + .artifact_registry + .sync_to_explicit_store( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + &ArtifactId::from("shared-artifact"), + "store://tenant-a/shared-artifact", + ) + .unwrap(); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected explicitly retained artifact metadata"); + }; + assert_eq!(artifact.availability, ArtifactAvailability::Available); + assert_eq!( + artifact.retention_state, + ArtifactRetentionState::ExplicitStorage + ); + assert!(artifact.downloadable_now); +} + +#[test] +fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret, node, process) in [ + ( + "tenant-a", + "project-a", + "user-a", + "session-a", + "node-a", + "process-shared", + ), + ( + "tenant-b", + "project-b", + "user-b", + "session-b", + "node-b", + "process-b", + ), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + enroll_test_node( + &mut service, + tenant, + project, + node, + &test_node_public_key(node), + ); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: secret.to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: process.to_owned(), + restart: false, + }, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + process: process.to_owned(), + epoch: 7, + }) + .unwrap(); + } + service.set_server_time(100); + let first = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 0, + source_bytes: 5, + text: "hello".to_owned(), + truncated: false, + }) + .unwrap(); + let CoordinatorResponse::TaskLogChunkRecorded { + sequence: Some(first_sequence), + next_offset: 5, + .. + } = first + else { + panic!("expected first live log sequence"); + }; + let retry = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 0, + source_bytes: 5, + text: "hello".to_owned(), + truncated: false, + }) + .unwrap(); + assert!(matches!( + retry, + CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. } + )); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 8, + source_bytes: 2, + text: "ok".to_owned(), + truncated: false, + }) + .unwrap(); + + let CoordinatorResponse::RecentLogs { + entries, + next_sequence: Some(cursor), + history_truncated: false, + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected first recent-log page"); + }; + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].sequence, first_sequence); + assert_eq!(entries[0].text, "hello"); + assert!(entries[1].text.contains("3 bytes")); + assert!(entries[1].truncated); + let CoordinatorResponse::RecentLogs { entries, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: Some(cursor), + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected second recent-log page"); + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].text, "ok"); + + let cross_tenant = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 10, + }, + }) + .unwrap_err(); + assert!(cross_tenant.to_string().contains("outside")); + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 201, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("200")); + + service + .handle_report_task_log_chunk( + "tenant-b".to_owned(), + "project-b".to_owned(), + "process-b".to_owned(), + "node-b".to_owned(), + "task-b".to_owned(), + TaskLogStream::Stderr, + 0, + 1, + "b".to_owned(), + false, + ) + .unwrap(); + for offset in 10..310 { + service + .handle_report_task_log_chunk( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + TaskLogStream::Stdout, + offset, + 1, + "x".to_owned(), + false, + ) + .unwrap(); + } + let tenant_a_logs = + &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))]; + assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS); + let tenant_b_logs = + &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))]; + assert_eq!(tenant_b_logs.len(), 1); + assert_eq!(tenant_b_logs[0].text, "b"); + let CoordinatorResponse::RecentLogs { + entries, + history_truncated, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected bounded recent-log response"); + }; + assert_eq!(entries.len(), 200); + assert!(history_truncated); +} diff --git a/crates/clusterflux-coordinator/src/service/wire_protocol.rs b/crates/clusterflux-coordinator/src/service/wire_protocol.rs index 07cd995..07aec2a 100644 --- a/crates/clusterflux-coordinator/src/service/wire_protocol.rs +++ b/crates/clusterflux-coordinator/src/service/wire_protocol.rs @@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest { impl CoordinatorWireRequest { pub fn into_request(self) -> Result { + self.into_parts().map(|(_, request)| request) + } + + pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { match self { - Self::Envelope(envelope) => envelope.into_request(), + Self::Envelope(envelope) => envelope.into_parts(), } } } @@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope { impl CoordinatorRequestEnvelope { pub fn into_request(self) -> Result { + self.into_parts().map(|(_, request)| request) + } + + pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE { return Err(format!( "unsupported coordinator wire request type {}; expected {}", @@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope { self.operation, payload_operation )); } - Ok(self.payload) + Ok((self.request_id, self.payload)) } } diff --git a/crates/clusterflux-core/src/api_error.rs b/crates/clusterflux-core/src/api_error.rs new file mode 100644 index 0000000..b6625cf --- /dev/null +++ b/crates/clusterflux-core/src/api_error.rs @@ -0,0 +1,296 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ApiErrorCode { + Unauthenticated, + SessionExpired, + AccountSuspended, + Forbidden, + ValidationError, + NotFound, + Conflict, + ActiveProcessExists, + NodeOffline, + NoCapableNode, + TaskNotRestartable, + ArtifactUnavailable, + ArtifactLimitExceeded, + QuotaExceeded, + TemporaryCapacity, + DebugEpochPartial, + InternalError, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ApiErrorCategory { + Authentication, + Authorization, + Validation, + State, + Availability, + Resource, + Internal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ApiError { + pub code: ApiErrorCode, + pub category: ApiErrorCategory, + pub message: String, + pub retryable: bool, + pub request_id: String, +} + +impl fmt::Display for ApiError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} (code {:?}, request {})", + self.message, self.code, self.request_id + ) + } +} + +impl std::error::Error for ApiError {} + +impl ApiError { + pub fn new( + code: ApiErrorCode, + category: ApiErrorCategory, + message: impl Into, + retryable: bool, + request_id: impl Into, + ) -> Self { + Self { + code, + category, + message: message.into(), + retryable, + request_id: request_id.into(), + } + } + + pub fn from_message(request_id: impl Into, message: impl Into) -> Self { + let request_id = request_id.into(); + let message = message.into(); + let normalized = message.to_ascii_lowercase(); + let (code, category, retryable) = if normalized.contains("session credential has expired") + || normalized.contains("session expired") + { + ( + ApiErrorCode::SessionExpired, + ApiErrorCategory::Authentication, + false, + ) + } else if normalized.contains("tenant is suspended") + || normalized.contains("account is suspended") + || normalized.contains("suspended by hosted") + { + ( + ApiErrorCode::AccountSuspended, + ApiErrorCategory::Authorization, + false, + ) + } else if normalized.contains("session credential") + || normalized.contains("no authenticated") + || normalized.contains("not authenticated") + || normalized.contains("credential is not") + { + ( + ApiErrorCode::Unauthenticated, + ApiErrorCategory::Authentication, + false, + ) + } else if normalized.contains("project already has active virtual process") { + ( + ApiErrorCode::ActiveProcessExists, + ApiErrorCategory::State, + false, + ) + } else if normalized.contains("no capable node") { + ( + ApiErrorCode::NoCapableNode, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("node offline") + || normalized.contains("node is not live") + || normalized.contains("source node is not connected") + || normalized.contains("direct connectivity unavailable") + { + ( + ApiErrorCode::NodeOffline, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("restart") + && (normalized.contains("not restartable") + || normalized.contains("requires whole") + || normalized.contains("clean boundary")) + { + ( + ApiErrorCode::TaskNotRestartable, + ApiErrorCategory::State, + false, + ) + } else if normalized.contains("artifact") + && (normalized.contains("exceeds download limit") + || normalized.contains("download session limit") + || normalized.contains("artifact limit")) + { + ( + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCategory::Resource, + false, + ) + } else if normalized.contains("artifact") + && (normalized.contains("does not exist") + || normalized.contains("not found") + || normalized.contains("unknown artifact")) + { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } else if normalized.contains("artifact") + && (normalized.contains("unavailable") || normalized.contains("retention")) + { + ( + ApiErrorCode::ArtifactUnavailable, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("resource limit") + || normalized.contains("quota") + || normalized.contains("limit exceeded") + { + ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ) + } else if normalized.contains("capacity") + || normalized.contains("temporarily full") + || normalized.contains("replay window is full") + { + ( + ApiErrorCode::TemporaryCapacity, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("partial debug epoch") + || normalized.contains("debug epoch is partially") + { + ( + ApiErrorCode::DebugEpochPartial, + ApiErrorCategory::State, + true, + ) + } else if normalized.contains("malformed") + || normalized.contains("invalid ") + || normalized.contains("protocol") + || normalized.contains("unknown field") + || normalized.contains("missing field") + || normalized.contains("must ") + { + ( + ApiErrorCode::ValidationError, + ApiErrorCategory::Validation, + false, + ) + } else if normalized.contains("outside") + || normalized.contains("unauthorized") + || normalized.contains("denied") + || normalized.contains("requires an authenticated") + || normalized.contains("may only") + { + ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ) + } else if normalized.contains("not found") + || normalized.contains("does not exist") + || normalized.contains("unknown ") + { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } else if normalized.contains("already") + || normalized.contains("conflict") + || normalized.contains("requires an active") + { + (ApiErrorCode::Conflict, ApiErrorCategory::State, false) + } else { + ( + ApiErrorCode::InternalError, + ApiErrorCategory::Internal, + false, + ) + }; + Self::new(code, category, message, retryable, request_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn required_machine_codes_are_stably_serialized() { + let required = [ + ApiErrorCode::Unauthenticated, + ApiErrorCode::SessionExpired, + ApiErrorCode::AccountSuspended, + ApiErrorCode::Forbidden, + ApiErrorCode::ValidationError, + ApiErrorCode::NotFound, + ApiErrorCode::Conflict, + ApiErrorCode::ActiveProcessExists, + ApiErrorCode::NodeOffline, + ApiErrorCode::NoCapableNode, + ApiErrorCode::TaskNotRestartable, + ApiErrorCode::ArtifactUnavailable, + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCode::QuotaExceeded, + ApiErrorCode::TemporaryCapacity, + ApiErrorCode::DebugEpochPartial, + ]; + let serialized = required + .into_iter() + .map(|code| serde_json::to_value(code).unwrap()) + .collect::>(); + assert_eq!( + serialized, + vec![ + "unauthenticated", + "session_expired", + "account_suspended", + "forbidden", + "validation_error", + "not_found", + "conflict", + "active_process_exists", + "node_offline", + "no_capable_node", + "task_not_restartable", + "artifact_unavailable", + "artifact_limit_exceeded", + "quota_exceeded", + "temporary_capacity", + "debug_epoch_partial", + ] + ); + } + + #[test] + fn message_classification_keeps_request_identity() { + let error = ApiError::from_message( + "request-17", + "CLI session credential has expired; run login again", + ); + assert_eq!(error.code, ApiErrorCode::SessionExpired); + assert_eq!(error.category, ApiErrorCategory::Authentication); + assert_eq!(error.request_id, "request-17"); + assert!(!error.retryable); + } +} diff --git a/crates/clusterflux-core/src/artifact.rs b/crates/clusterflux-core/src/artifact.rs index bc0e177..42970bd 100644 --- a/crates/clusterflux-core/src/artifact.rs +++ b/crates/clusterflux-core/src/artifact.rs @@ -324,6 +324,16 @@ impl ArtifactRegistry { .get(&ArtifactScopeKey::from_refs(tenant, project, artifact)) } + pub fn metadata_for_project<'a>( + &'a self, + tenant: &'a TenantId, + project: &'a ProjectId, + ) -> impl Iterator + 'a { + self.artifacts + .values() + .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project) + } + pub fn download_action( &self, context: &AuthContext, diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index b2f9188..c9d5b77 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -1,3 +1,4 @@ +mod api_error; pub mod artifact; pub mod auth; pub mod bundle; @@ -18,6 +19,7 @@ pub mod transport; pub mod vfs; pub mod wire; +pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode}; pub use artifact::{ ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry, ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, diff --git a/crates/clusterflux-node/src/assignment_runner.rs b/crates/clusterflux-node/src/assignment_runner.rs index 0e6ce6d..dc63a24 100644 --- a/crates/clusterflux-node/src/assignment_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner.rs @@ -676,6 +676,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost { let mut runner = CoordinatorControlledProcessRunner::new( self, Duration::from_millis(request.timeout_ms), + configured_secrets.clone(), ); let output = LinuxRootlessPodmanBackend .execute_local_checkout_task( diff --git a/crates/clusterflux-node/src/assignment_runner/process_runner.rs b/crates/clusterflux-node/src/assignment_runner/process_runner.rs index 2b6dbd3..48cfc97 100644 --- a/crates/clusterflux-node/src/assignment_runner/process_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner/process_runner.rs @@ -1,4 +1,63 @@ use super::*; +use std::sync::mpsc::{self, Receiver, SyncSender}; + +struct LiveLogChunk { + stream: &'static str, + offset: u64, + source_bytes: u64, + bytes: Vec, + truncated: bool, +} + +fn redact_safe_live_log_prefix( + pending: &[u8], + configured_secrets: &[String], + final_chunk: bool, +) -> Option<(usize, String)> { + if pending.is_empty() { + return None; + } + let secret_bytes = configured_secrets + .iter() + .filter(|secret| secret.len() >= 4) + .map(String::as_bytes) + .collect::>(); + let maximum_secret_bytes = secret_bytes + .iter() + .map(|secret| secret.len()) + .max() + .unwrap_or(0); + if !final_chunk && pending.len() <= maximum_secret_bytes { + return None; + } + let mut consumed = if final_chunk { + pending.len() + } else { + pending.len() - maximum_secret_bytes + }; + loop { + let previous = consumed; + for secret in &secret_bytes { + for start in 0..=pending.len().saturating_sub(secret.len()) { + let end = start + secret.len(); + if start < consumed && end > consumed && &pending[start..end] == *secret { + consumed = end; + } + } + } + if consumed == previous { + break; + } + } + if consumed == 0 { + return None; + } + let text = redact_configured_values( + String::from_utf8_lossy(&pending[..consumed]).into_owned(), + configured_secrets, + ); + Some((consumed, text)) +} pub(super) struct CoordinatorControlledProcessRunner { pub(super) args: Args, @@ -8,12 +67,37 @@ pub(super) struct CoordinatorControlledProcessRunner { pub(super) debug_control: Arc, pub(super) command_status: Arc>>, pub(super) timeout: Duration, + pub(super) configured_secrets: Vec, +} + +#[cfg(test)] +mod tests { + use super::redact_safe_live_log_prefix; + + #[test] + fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() { + let secrets = vec!["correct-horse".to_owned()]; + let mut pending = b"prefix correct-".to_vec(); + let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap(); + pending.drain(..consumed); + pending.extend_from_slice(b"horse suffix"); + let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap(); + + let combined = format!("{first}{second}"); + assert_eq!(combined, "prefix [REDACTED] suffix"); + assert!(!combined.contains("correct-")); + assert!(!combined.contains("horse")); + } } impl CoordinatorControlledProcessRunner { const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1; - pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self { + pub(super) fn new( + host: &CoordinatorWasmTaskHost, + timeout: Duration, + configured_secrets: Vec, + ) -> Self { Self { args: host.args.clone(), process: host.process.clone(), @@ -22,6 +106,7 @@ impl CoordinatorControlledProcessRunner { debug_control: Arc::clone(&host.debug_control), command_status: Arc::clone(&host.command_status), timeout, + configured_secrets, } } @@ -201,10 +286,17 @@ impl CoordinatorControlledProcessRunner { fn drain_bounded( mut reader: impl Read + Send + 'static, maximum: usize, + stream: &'static str, + sender: SyncSender, + configured_secrets: Vec, ) -> thread::JoinHandle, String>> { thread::spawn(move || { let mut captured = Vec::new(); let mut buffer = [0_u8; 16 * 1024]; + let mut source_offset = 0_u64; + let mut pending_offset = 0_u64; + let mut pending = Vec::new(); + let mut discarded = false; loop { let count = reader .read(&mut buffer) @@ -213,11 +305,129 @@ impl CoordinatorControlledProcessRunner { break; } let remaining = maximum.saturating_sub(captured.len()); - captured.extend_from_slice(&buffer[..count.min(remaining)]); + let retained = count.min(remaining); + if retained > 0 { + captured.extend_from_slice(&buffer[..retained]); + pending.extend_from_slice(&buffer[..retained]); + if let Some((consumed, text)) = + redact_safe_live_log_prefix(&pending, &configured_secrets, false) + { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: pending_offset, + source_bytes: consumed as u64, + bytes: text.into_bytes(), + truncated: false, + }); + pending.drain(..consumed); + pending_offset = pending_offset.saturating_add(consumed as u64); + } + } + if retained < count { + discarded = true; + } + source_offset = source_offset.saturating_add(count as u64); + } + if let Some((consumed, text)) = + redact_safe_live_log_prefix(&pending, &configured_secrets, true) + { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: pending_offset, + source_bytes: consumed as u64, + bytes: text.into_bytes(), + truncated: false, + }); + } + if discarded { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: maximum as u64, + source_bytes: 0, + bytes: b"[log output truncated at node capture limit]".to_vec(), + truncated: true, + }); } Ok(captured) }) } + + fn spawn_live_log_reporter(&self, receiver: Receiver) -> thread::JoinHandle<()> { + let args = self.args.clone(); + let process = self.process.clone(); + let task = self.task.clone(); + let node_private_key = self.node_private_key.clone(); + let configured_secrets = self.configured_secrets.clone(); + let command_status = Arc::clone(&self.command_status); + thread::spawn(move || { + let mut log_session = None; + let mut delivery_available = true; + while let Ok(chunk) = receiver.recv() { + if !delivery_available { + continue; + } + let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned(); + text = redact_configured_values(text, &configured_secrets); + let mut delivered = false; + for _ in 0..2 { + if log_session.is_none() { + log_session = CoordinatorSession::connect_with_timeouts( + &args.coordinator, + Duration::from_millis(500), + Duration::from_millis(500), + ) + .ok(); + } + let Some(session) = log_session.as_mut() else { + continue; + }; + let request = signed_node_request_json( + &args, + &node_private_key, + "report_task_log_chunk", + serde_json::json!({ + "type": "report_task_log_chunk", + "tenant": &args.tenant, + "project": &args.project, + "process": &process, + "node": &args.node, + "task": &task, + "stream": chunk.stream, + "offset": chunk.offset, + "source_bytes": chunk.source_bytes, + "text": &text, + "truncated": chunk.truncated, + }), + ); + let result = match request { + Ok(request) => session + .request(request) + .map(|_| ()) + .map_err(|error| error.to_string()), + Err(error) => Err(error.to_string()), + }; + match result { + Ok(()) => { + delivered = true; + break; + } + Err(_) => { + log_session = None; + } + } + } + if !delivered { + delivery_available = false; + if let Ok(mut current) = command_status.lock() { + *current = Some( + "live log delivery was interrupted; final bounded output remains available" + .to_owned(), + ); + } + } + } + }) + } } impl ProcessRunner for CoordinatorControlledProcessRunner { @@ -245,12 +455,16 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { command.program, command.args.join(" ") )); + let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64); let stdout = Self::drain_bounded( child .stdout .take() .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, + "stdout", + live_log_sender.clone(), + self.configured_secrets.clone(), ); let stderr = Self::drain_bounded( child @@ -258,11 +472,18 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .take() .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, + "stderr", + live_log_sender, + self.configured_secrets.clone(), ); + let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver); let mut session = match CoordinatorSession::connect(&self.args.coordinator) { Ok(session) => session, Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); + let _ = stdout.join(); + let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "establish execution control channel: {error}" ))); @@ -277,6 +498,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Ok(None) => {} Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); + let _ = stdout.join(); + let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(error.to_string())); } } @@ -289,6 +513,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "native command exceeded wall-clock timeout of {} ms", self.timeout.as_millis() @@ -303,6 +528,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Cancelled( "coordinator requested cancellation or abort".to_owned(), )); @@ -312,6 +538,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(error); } } @@ -360,6 +587,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .join() .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))? .map_err(BackendError::Command)?; + live_log_reporter + .join() + .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?; self.set_command_status(format!( "native command exited with status {:?}", status.code() diff --git a/crates/clusterflux-node/src/assignment_runner/tests.rs b/crates/clusterflux-node/src/assignment_runner/tests.rs index 4d2d864..35147d3 100644 --- a/crates/clusterflux-node/src/assignment_runner/tests.rs +++ b/crates/clusterflux-node/src/assignment_runner/tests.rs @@ -49,6 +49,7 @@ fn test_controlled_runner( debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), timeout, + configured_secrets: Vec::new(), } } @@ -90,6 +91,7 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() { debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), timeout: Duration::from_secs(30), + configured_secrets: Vec::new(), }; let started = Instant::now(); let error = runner diff --git a/crates/clusterflux-node/src/coordinator_session.rs b/crates/clusterflux-node/src/coordinator_session.rs index 997e0f0..13e64ec 100644 --- a/crates/clusterflux-node/src/coordinator_session.rs +++ b/crates/clusterflux-node/src/coordinator_session.rs @@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity; use clusterflux_control::ControlSession; use clusterflux_core::coordinator_wire_request; use serde_json::Value; +use std::time::Duration; pub(crate) struct CoordinatorSession { inner: ControlSession, @@ -15,6 +16,16 @@ impl CoordinatorSession { }) } + pub(crate) fn connect_with_timeouts( + addr: &str, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result> { + Ok(Self { + inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?, + }) + } + pub(crate) fn request(&mut self, value: Value) -> Result> { let request_id = format!("node-{}", self.inner.requests() + 1); let wire_request = coordinator_wire_request(request_id, value); diff --git a/docs/artifacts.md b/docs/artifacts.md index 9320564..a1521a7 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context. The reverse stream counts framing, base64 expansion, failed bytes, and abandoned transfers. The CLI verifies the final digest. Hosted policy may impose per-project, per-tenant, and per-account size, concurrency, and period limits. -Operators may disable relay traffic as an emergency safety control; one tenant's -period usage does not consume a shared customer byte quota. +There is no hosted global circuit breaker, and one tenant's period usage does +not consume a shared customer byte quota.