Public release release-e47f9c27bbeb
Source commit: e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d Public tree identity: sha256:4c1af1dfd67d3f2b5088531ea8727b11124b013d2251a4d5088f51a6424c0196
This commit is contained in:
Clusterflux release
parent
3a4d4fa7ae
commit
9223c54939
30 changed files with 4195 additions and 434 deletions
|
|
@ -12,8 +12,8 @@ use clusterflux_core::{
|
|||
AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability,
|
||||
DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements,
|
||||
LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits,
|
||||
SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec,
|
||||
VfsPath, WasmExportAbi,
|
||||
SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId,
|
||||
TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -521,6 +521,9 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest {
|
|||
CoordinatorRequest::CompleteSourcePreparation { node, .. } => {
|
||||
(node.clone(), "complete_source_preparation")
|
||||
}
|
||||
CoordinatorRequest::RequestRendezvous { source, .. } => {
|
||||
(source.node.to_string(), "request_rendezvous")
|
||||
}
|
||||
CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"),
|
||||
CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"),
|
||||
CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"),
|
||||
|
|
@ -2752,6 +2755,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
|
|||
},
|
||||
);
|
||||
let CoordinatorResponse::DebugBreakpoints {
|
||||
revision,
|
||||
probe_symbols,
|
||||
hit_epoch,
|
||||
..
|
||||
|
|
@ -2761,15 +2765,37 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
|
|||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
revision: 1,
|
||||
probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()],
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected debug breakpoints response");
|
||||
};
|
||||
assert_eq!(revision, 1);
|
||||
assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
|
||||
assert_eq!(hit_epoch, None);
|
||||
|
||||
let CoordinatorResponse::DebugBreakpoints {
|
||||
revision,
|
||||
probe_symbols,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::SetDebugBreakpoints {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
revision: 0,
|
||||
probe_symbols: vec!["clusterflux.probe.stale".to_owned()],
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected stale breakpoint response");
|
||||
};
|
||||
assert_eq!(revision, 1);
|
||||
assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
|
||||
|
||||
let CoordinatorResponse::DebugProbeHit {
|
||||
breakpoint_matched,
|
||||
debug_epoch,
|
||||
|
|
@ -3583,6 +3609,184 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
assert!(!service.main_runtime.controls.contains_key(&process_key));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() {
|
||||
let mut service = CoordinatorService::new(31);
|
||||
let tenant = TenantId::from("tenant");
|
||||
let project = ProjectId::from("project");
|
||||
let process = ProcessId::from("process-main-before-child");
|
||||
let child = TaskInstanceId::from("child-active");
|
||||
let process_key = process_control_key(&tenant, &project, &process);
|
||||
|
||||
service
|
||||
.handle_request(CoordinatorRequest::AttachNode {
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
node: "worker".to_owned(),
|
||||
public_key: test_node_public_key("worker"),
|
||||
})
|
||||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
actor_user: None,
|
||||
actor_agent: None,
|
||||
agent_public_key_fingerprint: None,
|
||||
agent_signature: None,
|
||||
process: process.to_string(),
|
||||
restart: false,
|
||||
})
|
||||
.unwrap();
|
||||
service
|
||||
.handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
|
||||
node: "worker".to_owned(),
|
||||
process: process.to_string(),
|
||||
epoch: 31,
|
||||
})
|
||||
.unwrap();
|
||||
register_test_task_assignment(
|
||||
&mut service,
|
||||
tenant.as_str(),
|
||||
project.as_str(),
|
||||
process.as_str(),
|
||||
"worker",
|
||||
"child-definition",
|
||||
child.as_str(),
|
||||
31,
|
||||
);
|
||||
|
||||
let main = TaskInstanceId::from("main-instance");
|
||||
service.main_runtime.controls.insert(
|
||||
process_key.clone(),
|
||||
super::main_runtime::CoordinatorMainControl {
|
||||
task_definition: TaskDefinitionId::from("build"),
|
||||
task_instance: main.clone(),
|
||||
abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
|
||||
state: "running".to_owned(),
|
||||
stopped_probe_symbol: None,
|
||||
handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
|
||||
launch_id: 1,
|
||||
},
|
||||
);
|
||||
service.debug_epochs.insert(process_key.clone(), 9);
|
||||
|
||||
service.record_coordinator_main_completion(
|
||||
super::main_runtime::MainScope {
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: process.clone(),
|
||||
task_definition: TaskDefinitionId::from("build"),
|
||||
task_instance: main,
|
||||
epoch: 31,
|
||||
launch_id: 1,
|
||||
},
|
||||
Ok(WasmTaskResult::completed(
|
||||
TaskInstanceId::from("main-instance"),
|
||||
TaskBoundaryValue::SmallJson(json!("main completed")),
|
||||
)),
|
||||
);
|
||||
|
||||
assert!(service
|
||||
.coordinator
|
||||
.active_process(&tenant, &project, &process)
|
||||
.is_some());
|
||||
assert!(service.debug_epochs.contains_key(&process_key));
|
||||
assert!(service
|
||||
.active_tasks
|
||||
.iter()
|
||||
.any(|(_, _, retained_process, _, task)| {
|
||||
retained_process == &process && task == &child
|
||||
}));
|
||||
let blocked_next = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
actor_user: None,
|
||||
actor_agent: None,
|
||||
agent_public_key_fingerprint: None,
|
||||
agent_signature: None,
|
||||
process: "process-too-early".to_owned(),
|
||||
restart: false,
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(blocked_next
|
||||
.to_string()
|
||||
.contains("already has active virtual process"));
|
||||
|
||||
let artifact_bytes = b"child artifact survives terminal cleanup";
|
||||
service
|
||||
.handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
process: process.to_string(),
|
||||
node: "worker".to_owned(),
|
||||
task: child.to_string(),
|
||||
terminal_state: Some(TaskTerminalState::Completed),
|
||||
status_code: Some(0),
|
||||
stdout_bytes: artifact_bytes.len() as u64,
|
||||
stderr_bytes: 0,
|
||||
stdout_tail: "child completed".to_owned(),
|
||||
stderr_tail: String::new(),
|
||||
stdout_truncated: false,
|
||||
stderr_truncated: false,
|
||||
artifact_path: Some("/vfs/artifacts/child-output".to_owned()),
|
||||
artifact_digest: Some(Digest::sha256(artifact_bytes)),
|
||||
artifact_size_bytes: Some(artifact_bytes.len() as u64),
|
||||
result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(service
|
||||
.coordinator
|
||||
.active_process(&tenant, &project, &process)
|
||||
.is_none());
|
||||
assert!(!service.debug_epochs.contains_key(&process_key));
|
||||
let CoordinatorResponse::TaskEvents { events } = service
|
||||
.handle_request(CoordinatorRequest::ListTaskEvents {
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: Some(process.to_string()),
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected retained task events");
|
||||
};
|
||||
assert_eq!(events.len(), 2);
|
||||
assert!(events.iter().any(|event| {
|
||||
event.executor == TaskExecutor::CoordinatorMain
|
||||
&& event.terminal_state == TaskTerminalState::Completed
|
||||
}));
|
||||
assert!(events.iter().any(|event| {
|
||||
event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes))
|
||||
}));
|
||||
let metadata = service
|
||||
.artifact_registry
|
||||
.metadata(&ArtifactId::from("child-output"))
|
||||
.expect("artifact metadata must survive terminal cleanup");
|
||||
assert_eq!(metadata.process, process);
|
||||
assert_eq!(metadata.digest, Digest::sha256(artifact_bytes));
|
||||
|
||||
let next = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: tenant.to_string(),
|
||||
project: project.to_string(),
|
||||
actor_user: None,
|
||||
actor_agent: None,
|
||||
agent_public_key_fingerprint: None,
|
||||
agent_signature: None,
|
||||
process: "process-after-cleanup".to_owned(),
|
||||
restart: false,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quiescent_cooperative_cancel_releases_slot_immediately() {
|
||||
let mut service = CoordinatorService::new(17);
|
||||
|
|
@ -6084,6 +6288,35 @@ fn service_meters_rendezvous_before_direct_transfer_plan() {
|
|||
assert!(quota.to_string().contains("RendezvousAttempt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() {
|
||||
let mut service = CoordinatorService::new(7);
|
||||
service
|
||||
.handle_request(CoordinatorRequest::AttachNode {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
node: "node-a".to_owned(),
|
||||
public_key: test_node_public_key("node-a"),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let response = service
|
||||
.handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous {
|
||||
scope: data_plane_scope("project"),
|
||||
source: endpoint("node-a"),
|
||||
destination: endpoint("node-b"),
|
||||
direct_connectivity: true,
|
||||
failure_reason: String::new(),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let CoordinatorResponse::RendezvousPlan { plan, .. } = response else {
|
||||
panic!("expected signed rendezvous plan");
|
||||
};
|
||||
assert_eq!(plan.source.node, NodeId::from("node-a"));
|
||||
assert_eq!(plan.destination.node, NodeId::from("node-b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_rejects_task_completion_outside_node_scope() {
|
||||
let mut service = CoordinatorService::new(1);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue