michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Update public backend API surface

Browse source 
Private source commit: ba3f7ce2b6d9
This commit is contained in:
Clusterflux Release 2026-07-26 17:22:33 +02:00
parent 784463622c
commit 26fdcb9d84
34 changed files with 5473 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

186
crates/clusterflux-coordinator/src/service.rs
View file

@ -7,9 +7,10 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
RateLimit, TenantId, TransportError, UserId,
Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
TenantId, TransportError, UserId,
};
use thiserror::Error;
@ -34,6 +35,7 @@ mod quota;
mod relay;
mod routing;
mod signed_nodes;
mod summaries;
mod tcp;
mod wire_protocol;
use authorization::authorize_authenticated_user_operation;
@ -43,12 +45,14 @@ use keys::{
ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
};
pub use protocol::{
ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
TaskTerminalState, VirtualProcessStatus, WorkflowActor,
ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
VirtualProcessStatus, WorkflowActor,
};
pub use quota::CoordinatorQuotaConfiguration;
pub use relay::{
@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
requested.clamp(1, maximum)
@ -111,6 +121,24 @@ pub struct CoordinatorAdmission {
pub max_artifact_download_ttl_seconds: u64,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CoordinatorOperationalMetrics {
pub tenants: usize,
pub users: usize,
pub projects: usize,
pub enrolled_nodes: usize,
pub reported_nodes: usize,
pub live_nodes: usize,
pub active_processes: usize,
pub active_coordinator_mains: usize,
pub max_active_coordinator_mains: usize,
pub active_tasks: usize,
pub queued_tasks: usize,
pub artifacts: usize,
pub retained_download_links: usize,
pub relay: ArtifactRelayUsage,
}
impl Default for CoordinatorAdmission {
fn default() -> Self {
Self {
@ -151,6 +179,98 @@ pub enum CoordinatorServiceError {
Durable(String),
}
impl CoordinatorServiceError {
pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
let request_id = request_id.into();
let message = self.to_string();
let (code, category, retryable) = match self {
Self::Io(_) => (
ApiErrorCode::TemporaryCapacity,
ApiErrorCategory::Availability,
true,
),
Self::Json(_)
| Self::CapabilityReport(_)
| Self::InvalidArtifactPath(_)
| Self::InvalidTaskLogTail(_) => (
ApiErrorCode::ValidationError,
ApiErrorCategory::Validation,
false,
),
Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
return ApiError::from_message(request_id, message);
}
Self::Coordinator(CoordinatorError::UnknownNode) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Coordinator(CoordinatorError::Enrollment(_)) => (
ApiErrorCode::Unauthenticated,
ApiErrorCategory::Authentication,
false,
),
Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
(ApiErrorCode::Conflict, ApiErrorCategory::State, true)
}
Self::Download(DownloadError::NotFound) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Download(DownloadError::Unavailable)
| Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
ApiErrorCode::ArtifactUnavailable,
ApiErrorCategory::Availability,
true,
),
Self::Download(DownloadError::LimitExceeded { .. }) => (
ApiErrorCode::ArtifactLimitExceeded,
ApiErrorCategory::Resource,
false,
),
Self::Download(DownloadError::Unauthorized(_))
| Self::Download(DownloadError::InvalidToken)
| Self::Download(DownloadError::Expired)
| Self::Download(DownloadError::Revoked) => (
ApiErrorCode::Forbidden,
ApiErrorCategory::Authorization,
false,
),
Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
ApiErrorCode::QuotaExceeded,
ApiErrorCategory::Resource,
true,
),
Self::Scheduler(_) => (
ApiErrorCode::NoCapableNode,
ApiErrorCategory::Availability,
true,
),
Self::Transport(_) => (
ApiErrorCode::NodeOffline,
ApiErrorCategory::Availability,
true,
),
Self::Panel(PanelError::RateLimited) => (
ApiErrorCode::QuotaExceeded,
ApiErrorCategory::Resource,
true,
),
Self::Panel(PanelError::UnknownWidget(_)) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Panel(_) => (
ApiErrorCode::Forbidden,
ApiErrorCategory::Authorization,
false,
),
Self::Durable(_) => (
ApiErrorCode::InternalError,
ApiErrorCategory::Internal,
true,
),
};
ApiError::new(code, category, message, retryable, request_id)
}
}
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
@ -161,6 +281,22 @@ pub struct CoordinatorService {
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
task_events: VecDeque<TaskCompletionEvent>,
process_scope_history: VecDeque<ProcessControlKey>,
process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
process_summary_order: VecDeque<ProcessControlKey>,
next_process_summary_order: u64,
recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
recent_log_offsets: BTreeMap<
(
TenantId,
ProjectId,
ProcessId,
clusterflux_core::TaskInstanceId,
String,
),
u64,
>,
next_recent_log_sequence: u64,
debug_audit_events: VecDeque<DebugAuditEvent>,
debug_epochs: BTreeMap<ProcessControlKey, u64>,
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@ -215,6 +351,29 @@ impl CoordinatorService {
self.artifact_relay.usage()
}
pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
CoordinatorOperationalMetrics {
tenants: self.coordinator.tenant_count(),
users: self.coordinator.user_count(),
projects: self.coordinator.project_count(),
enrolled_nodes: self.coordinator.node_identity_count(),
reported_nodes: self.node_descriptors.len(),
live_nodes: self
.node_descriptors
.keys()
.filter(|scope| self.node_is_live(scope))
.count(),
active_processes: self.coordinator.active_process_count(),
active_coordinator_mains: self.main_runtime.active_main_count(),
max_active_coordinator_mains: self.main_runtime.max_active_mains(),
active_tasks: self.active_tasks.len(),
queued_tasks: self.pending_task_launches.len(),
artifacts: self.artifact_registry.artifact_count(),
retained_download_links: self.artifact_registry.retained_download_link_count(),
relay: self.artifact_relay.usage(),
}
}
fn commit_artifact_relay(
&mut self,
candidate: relay::ArtifactRelayLedger,
@ -404,6 +563,13 @@ impl CoordinatorService {
enrollment_grants: BTreeMap::new(),
task_events: VecDeque::new(),
process_scope_history: VecDeque::new(),
process_summaries: BTreeMap::new(),
process_summary_order: VecDeque::new(),
next_process_summary_order: 1,
recent_logs: BTreeMap::new(),
recent_log_dropped_through: BTreeMap::new(),
recent_log_offsets: BTreeMap::new(),
next_recent_log_sequence: 1,
debug_audit_events: VecDeque::new(),
debug_epochs: BTreeMap::new(),
debug_epoch_runtime: BTreeMap::new(),