michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-e47f9c27bbeb

Browse source 
Source commit: e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d

Public tree identity: sha256:4c1af1dfd67d3f2b5088531ea8727b11124b013d2251a4d5088f51a6424c0196
This commit is contained in:
Clusterflux release 2026-07-24 15:52:30 +02:00
parent 3a4d4fa7ae
commit 9223c54939
30 changed files with 4195 additions and 434 deletions
Download patch file Download diff file Expand all files Collapse all files

24
crates/clusterflux-coordinator/src/service/debug.rs
View file

@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime {
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct DebugBreakpointPlan {
pub(super) actor: UserId,
pub(super) revision: u64,
pub(super) probe_symbols: BTreeSet<String>,
pub(super) hit_epoch: Option<u64>,
pub(super) hit_task: Option<TaskInstanceId>,
@ -85,6 +86,7 @@ impl CoordinatorService {
project: String,
actor_user: String,
process: String,
revision: u64,
probe_symbols: Vec<String>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let probe_symbols = validate_probe_symbols(probe_symbols)?;
@ -115,10 +117,28 @@ impl CoordinatorService {
))
.into());
}
let key = process_control_key(&tenant, &project, &process);
if let Some(current) = self.debug_breakpoints.get(&key) {
if current.actor == actor && revision < current.revision {
return Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision: current.revision,
probe_symbols: current.probe_symbols.iter().cloned().collect(),
hit_epoch: current.hit_epoch,
hit_task: current.hit_task.clone(),
hit_probe_symbol: current.hit_probe_symbol.clone(),
charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
used_debug_read_bytes: audit_event.used_debug_read_bytes,
audit_event,
});
}
}
self.debug_breakpoints.insert(
process_control_key(&tenant, &project, &process),
key,
DebugBreakpointPlan {
actor: actor.clone(),
revision,
probe_symbols: probe_symbols.iter().cloned().collect(),
hit_epoch: None,
hit_task: None,
@ -128,6 +148,7 @@ impl CoordinatorService {
Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision,
probe_symbols,
hit_epoch: None,
hit_task: None,
@ -190,6 +211,7 @@ impl CoordinatorService {
Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision: plan.revision,
probe_symbols: plan.probe_symbols.into_iter().collect(),
hit_epoch: plan.hit_epoch,
hit_task: plan.hit_task,

4
crates/clusterflux-coordinator/src/service/debug_requests.rs
View file

@ -33,12 +33,14 @@ impl CoordinatorService {
project,
actor_user,
process,
revision,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant,
project,
actor_user,
process,
revision,
probe_symbols,
),
CoordinatorRequest::InspectDebugBreakpoints {
@ -96,12 +98,14 @@ impl CoordinatorService {
),
AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
process,
revision,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
revision,
probe_symbols,
),
AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self

74
crates/clusterflux-coordinator/src/service/main_runtime.rs
View file

@ -30,14 +30,14 @@ use super::{
};
#[derive(Clone)]
struct MainScope {
tenant: TenantId,
project: ProjectId,
process: ProcessId,
task_definition: TaskDefinitionId,
task_instance: TaskInstanceId,
epoch: u64,
launch_id: u64,
pub(super) struct MainScope {
pub(super) tenant: TenantId,
pub(super) project: ProjectId,
pub(super) process: ProcessId,
pub(super) task_definition: TaskDefinitionId,
pub(super) task_instance: TaskInstanceId,
pub(super) epoch: u64,
pub(super) launch_id: u64,
}
enum MainCommand {
@ -889,7 +889,7 @@ impl CoordinatorService {
}
}
fn record_coordinator_main_completion(
pub(super) fn record_coordinator_main_completion(
&mut self,
scope: MainScope,
result: Result<WasmTaskResult, String>,
@ -1229,4 +1229,60 @@ mod tests {
assert!(service.active_tasks.contains(&child_key));
assert!(!service.task_aborts.contains(&child_key));
}
#[test]
fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() {
let mut service = CoordinatorService::new(7);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("vp-failed-main");
let main_task = TaskInstanceId::from("ti:vp-failed-main:main");
let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1");
let child_node = NodeId::from("worker");
let scope = MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main_task.clone(),
epoch: 7,
launch_id: 1,
};
service
.coordinator
.start_process(tenant.clone(), project.clone(), process.clone());
let process_key = process_control_key(&tenant, &project, &process);
service.main_runtime.controls.insert(
process_key.clone(),
CoordinatorMainControl {
task_definition: scope.task_definition.clone(),
task_instance: main_task,
abort: Arc::new(AtomicBool::new(false)),
debug: Arc::new(WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: Arc::new(Mutex::new(HashMap::new())),
launch_id: 1,
},
);
let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
service.active_tasks.insert(child_key.clone());
service.debug_epochs.insert(process_key.clone(), 2);
service.record_coordinator_main_completion(scope, Err("main crashed".to_owned()));
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
assert!(service.active_tasks.contains(&child_key));
assert!(service.task_aborts.contains(&child_key));
assert!(service.process_aborts.contains(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(service.task_events.iter().any(|event| {
event.process == process
&& event.executor == TaskExecutor::CoordinatorMain
&& event.terminal_state == TaskTerminalState::Failed
}));
}
}

1157
crates/clusterflux-coordinator/src/service/protocol.rs
View file

File diff suppressed because it is too large Load diff

1
crates/clusterflux-coordinator/src/service/protocol/responses.rs
View file

@ -430,6 +430,7 @@ pub enum CoordinatorResponse {
DebugBreakpoints {
process: ProcessId,
actor: UserId,
revision: u64,
probe_symbols: Vec<String>,
hit_epoch: Option<u64>,
hit_task: Option<TaskInstanceId>,

27
crates/clusterflux-coordinator/src/service/signed_nodes.rs
View file

@ -19,7 +19,9 @@ impl CoordinatorService {
))
})?;
let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
let signed_node = NodeId::new(signed_node);
let signed_node = NodeId::try_new(signed_node).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
})?;
if request_node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
@ -144,6 +146,19 @@ impl CoordinatorService {
provider,
source_snapshot,
),
CoordinatorRequest::RequestRendezvous {
scope,
source,
destination,
direct_connectivity,
failure_reason,
} => self.handle_request_rendezvous(
scope,
source,
destination,
direct_connectivity,
failure_reason,
),
CoordinatorRequest::ReconnectNode {
node,
process,
@ -322,6 +337,7 @@ fn signed_node_request_kind(
CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"),
CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
@ -356,7 +372,14 @@ fn signed_node_request_node(
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
| CoordinatorRequest::TaskCompleted { node, .. } => {
NodeId::try_new(node.clone()).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"invalid wrapped node identifier: {error}"
))
})
}
CoordinatorRequest::RequestRendezvous { source, .. } => Ok(source.node.clone()),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)

97
crates/clusterflux-coordinator/src/service/tcp.rs
View file

@ -189,6 +189,11 @@ fn authorize_client_request(
#[cfg(test)]
mod transport_boundary_tests {
use std::io::{BufRead as _, BufReader};
use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId};
use serde_json::json;
use super::*;
#[test]
@ -197,4 +202,96 @@ mod transport_boundary_tests {
let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
assert!(error.to_string().contains("restricted to loopback"));
}
#[test]
fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() {
let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
let mut coordinator = CoordinatorService::new(11);
coordinator
.issue_cli_session(
TenantId::from("tenant"),
ProjectId::from("project"),
UserId::from("user"),
"healthy-session",
None,
)
.unwrap();
let shared = Arc::new(Mutex::new(coordinator));
let server_shared = Arc::clone(&shared);
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap();
});
let mut stream = TcpStream::connect(addr).unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
for (index, malformed_process) in [
String::new(),
" ".to_owned(),
"bad\0process".to_owned(),
"bad process!".to_owned(),
"x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1),
]
.into_iter()
.enumerate()
{
let malformed = coordinator_wire_request(
format!("malformed-{index}"),
json!({
"type": "authenticated",
"session_secret": "healthy-session",
"request": {
"type": "abort_process",
"process": malformed_process,
"launch_attempt": "valid-attempt"
}
}),
);
serde_json::to_writer(&mut stream, &malformed).unwrap();
stream.write_all(b"\n").unwrap();
stream.flush().unwrap();
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let CoordinatorResponse::Error { message } =
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
else {
panic!("malformed identifier request unexpectedly succeeded");
};
assert!(
message.contains("malformed external identifier")
&& message.contains("request.request.process"),
"unexpected malformed identifier response: {message}"
);
let valid = coordinator_wire_request(
format!("healthy-{index}"),
json!({
"type": "authenticated",
"session_secret": "healthy-session",
"request": { "type": "auth_status" }
}),
);
serde_json::to_writer(&mut stream, &valid).unwrap();
stream.write_all(b"\n").unwrap();
stream.flush().unwrap();
line.clear();
reader.read_line(&mut line).unwrap();
assert!(
matches!(
serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
CoordinatorResponse::AuthStatus {
authenticated: true,
..
}
),
"valid authenticated traffic failed after malformed request {index}"
);
}
stream.shutdown(std::net::Shutdown::Both).unwrap();
server.join().unwrap();
assert!(!shared.is_poisoned());
}
}

237
crates/clusterflux-coordinator/src/service/tests.rs
View file

@ -12,8 +12,8 @@ use clusterflux_core::{
AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability,
DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements,
LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits,
SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec,
VfsPath, WasmExportAbi,
SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId,
TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult,
};
use serde_json::json;
@ -521,6 +521,9 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest {
CoordinatorRequest::CompleteSourcePreparation { node, .. } => {
(node.clone(), "complete_source_preparation")
}
CoordinatorRequest::RequestRendezvous { source, .. } => {
(source.node.to_string(), "request_rendezvous")
}
CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"),
CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"),
CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"),
@ -2752,6 +2755,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
},
);
let CoordinatorResponse::DebugBreakpoints {
revision,
probe_symbols,
hit_epoch,
..
@ -2761,15 +2765,37 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
project: "project".to_owned(),
actor_user: "user".to_owned(),
process: "process".to_owned(),
revision: 1,
probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()],
})
.unwrap()
else {
panic!("expected debug breakpoints response");
};
assert_eq!(revision, 1);
assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
assert_eq!(hit_epoch, None);
let CoordinatorResponse::DebugBreakpoints {
revision,
probe_symbols,
..
} = service
.handle_request(CoordinatorRequest::SetDebugBreakpoints {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
process: "process".to_owned(),
revision: 0,
probe_symbols: vec!["clusterflux.probe.stale".to_owned()],
})
.unwrap()
else {
panic!("expected stale breakpoint response");
};
assert_eq!(revision, 1);
assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
let CoordinatorResponse::DebugProbeHit {
breakpoint_matched,
debug_epoch,
@ -3583,6 +3609,184 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
assert!(!service.main_runtime.controls.contains_key(&process_key));
}
#[test]
fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() {
let mut service = CoordinatorService::new(31);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("process-main-before-child");
let child = TaskInstanceId::from("child-active");
let process_key = process_control_key(&tenant, &project, &process);
service
.handle_request(CoordinatorRequest::AttachNode {
tenant: tenant.to_string(),
project: project.to_string(),
node: "worker".to_owned(),
public_key: test_node_public_key("worker"),
})
.unwrap();
service
.handle_request(CoordinatorRequest::StartProcess {
launch_attempt: None,
tenant: tenant.to_string(),
project: project.to_string(),
actor_user: None,
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
process: process.to_string(),
restart: false,
})
.unwrap();
service
.handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
node: "worker".to_owned(),
process: process.to_string(),
epoch: 31,
})
.unwrap();
register_test_task_assignment(
&mut service,
tenant.as_str(),
project.as_str(),
process.as_str(),
"worker",
"child-definition",
child.as_str(),
31,
);
let main = TaskInstanceId::from("main-instance");
service.main_runtime.controls.insert(
process_key.clone(),
super::main_runtime::CoordinatorMainControl {
task_definition: TaskDefinitionId::from("build"),
task_instance: main.clone(),
abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
launch_id: 1,
},
);
service.debug_epochs.insert(process_key.clone(), 9);
service.record_coordinator_main_completion(
super::main_runtime::MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main,
epoch: 31,
launch_id: 1,
},
Ok(WasmTaskResult::completed(
TaskInstanceId::from("main-instance"),
TaskBoundaryValue::SmallJson(json!("main completed")),
)),
);
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_some());
assert!(service.debug_epochs.contains_key(&process_key));
assert!(service
.active_tasks
.iter()
.any(|(_, _, retained_process, _, task)| {
retained_process == &process && task == &child
}));
let blocked_next = service
.handle_request(CoordinatorRequest::StartProcess {
launch_attempt: None,
tenant: tenant.to_string(),
project: project.to_string(),
actor_user: None,
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
process: "process-too-early".to_owned(),
restart: false,
})
.unwrap_err();
assert!(blocked_next
.to_string()
.contains("already has active virtual process"));
let artifact_bytes = b"child artifact survives terminal cleanup";
service
.handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
tenant: tenant.to_string(),
project: project.to_string(),
process: process.to_string(),
node: "worker".to_owned(),
task: child.to_string(),
terminal_state: Some(TaskTerminalState::Completed),
status_code: Some(0),
stdout_bytes: artifact_bytes.len() as u64,
stderr_bytes: 0,
stdout_tail: "child completed".to_owned(),
stderr_tail: String::new(),
stdout_truncated: false,
stderr_truncated: false,
artifact_path: Some("/vfs/artifacts/child-output".to_owned()),
artifact_digest: Some(Digest::sha256(artifact_bytes)),
artifact_size_bytes: Some(artifact_bytes.len() as u64),
result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))),
})
.unwrap();
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
assert!(!service.debug_epochs.contains_key(&process_key));
let CoordinatorResponse::TaskEvents { events } = service
.handle_request(CoordinatorRequest::ListTaskEvents {
tenant: tenant.to_string(),
project: project.to_string(),
actor_user: "user".to_owned(),
process: Some(process.to_string()),
})
.unwrap()
else {
panic!("expected retained task events");
};
assert_eq!(events.len(), 2);
assert!(events.iter().any(|event| {
event.executor == TaskExecutor::CoordinatorMain
&& event.terminal_state == TaskTerminalState::Completed
}));
assert!(events.iter().any(|event| {
event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes))
}));
let metadata = service
.artifact_registry
.metadata(&ArtifactId::from("child-output"))
.expect("artifact metadata must survive terminal cleanup");
assert_eq!(metadata.process, process);
assert_eq!(metadata.digest, Digest::sha256(artifact_bytes));
let next = service
.handle_request(CoordinatorRequest::StartProcess {
launch_attempt: None,
tenant: tenant.to_string(),
project: project.to_string(),
actor_user: None,
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
process: "process-after-cleanup".to_owned(),
restart: false,
})
.unwrap();
assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. }));
}
#[test]
fn quiescent_cooperative_cancel_releases_slot_immediately() {
let mut service = CoordinatorService::new(17);
@ -6084,6 +6288,35 @@ fn service_meters_rendezvous_before_direct_transfer_plan() {
assert!(quota.to_string().contains("RendezvousAttempt"));
}
#[test]
fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() {
let mut service = CoordinatorService::new(7);
service
.handle_request(CoordinatorRequest::AttachNode {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "node-a".to_owned(),
public_key: test_node_public_key("node-a"),
})
.unwrap();
let response = service
.handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous {
scope: data_plane_scope("project"),
source: endpoint("node-a"),
destination: endpoint("node-b"),
direct_connectivity: true,
failure_reason: String::new(),
})
.unwrap();
let CoordinatorResponse::RendezvousPlan { plan, .. } = response else {
panic!("expected signed rendezvous plan");
};
assert_eq!(plan.source.node, NodeId::from("node-a"));
assert_eq!(plan.destination.node, NodeId::from("node-b"));
}
#[test]
fn service_rejects_task_completion_outside_node_scope() {
let mut service = CoordinatorService::new(1);