michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-e7c2b3ac175d

Browse source 
Source commit: e7c2b3ac175db426791c02779841c5df1d0ffdff

Public tree identity: sha256:4f0b7cd828932c06d56f2406788f89b2050ef56c1e1a4f76f55341d387d78e46
This commit is contained in:
Clusterflux release 2026-07-29 02:53:56 +02:00
parent 9f43c6276a
commit cfd4f19da2
16 changed files with 1386 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

276
crates/clusterflux-dap/src/runtime_client.rs
View file

@ -742,14 +742,16 @@ fn fetch_current_process_status(
client_user_request(
state,
json!({
"type": "list_processes",
"type": "list_process_summaries",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
"cursor": null,
"limit": 100,
}),
),
)?;
let current = statuses
let summary = statuses
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
@ -758,6 +760,9 @@ fn fetch_current_process_status(
})
})
.cloned();
let current = merge_active_process_status(state, summary, |request| {
coordinator_request(coordinator, request)
})?;
Ok((statuses, current))
}
@ -768,13 +773,15 @@ fn fetch_current_process_status_in(
let statuses = session.request(client_user_request(
state,
json!({
"type": "list_processes",
"type": "list_process_summaries",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
"cursor": null,
"limit": 100,
}),
))?;
let current = statuses
let summary = statuses
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
@ -783,9 +790,51 @@ fn fetch_current_process_status_in(
})
})
.cloned();
let current = merge_active_process_status(state, summary, |request| session.request(request))?;
Ok((statuses, current))
}
fn merge_active_process_status<F>(
state: &AdapterState,
summary: Option<Value>,
mut request: F,
) -> Result<Option<Value>>
where
F: FnMut(Value) -> Result<Value>,
{
let Some(mut summary) = summary else {
return Ok(None);
};
if summary.get("lifecycle").and_then(Value::as_str) != Some("active") {
return Ok(Some(summary));
}
let active_statuses = request(client_user_request(
state,
json!({
"type": "list_processes",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
}),
))?;
let active = active_statuses
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
processes.iter().find(|process| {
process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
})
});
if let (Some(summary), Some(active)) =
(summary.as_object_mut(), active.and_then(Value::as_object))
{
for (key, value) in active {
summary.entry(key.clone()).or_insert_with(|| value.clone());
}
}
Ok(Some(summary))
}
pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> {
let coordinator =
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
@ -859,27 +908,7 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result<RuntimeLau
.count()
})
.unwrap_or(0);
let process_statuses = coordinator_request(
&coordinator,
client_user_request(
state,
json!({
"type": "list_processes",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
}),
),
)?;
let process_status = process_statuses
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
processes.iter().find(|process| {
process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
})
})
.cloned();
let (process_statuses, process_status) = fetch_current_process_status(&coordinator, state)?;
let node = process_status
.as_ref()
.and_then(|status| status.get("connected_nodes"))
@ -1079,9 +1108,8 @@ pub(crate) fn observe_services_runtime(
}
}
}
// Terminal task events remain inspectable after the active process slot is
// released. Read them before active-process debug state so a fast main exit
// cannot turn a successful continuation into an authorization error.
// Events remain useful for output and attempt details, but terminal
// authority comes from the durable process summary read below.
let current_session = session.as_mut().expect("observer session connected");
let events = match current_session.request(client_user_request(
state,
@ -1118,65 +1146,6 @@ pub(crate) fn observe_services_runtime(
std::thread::sleep(reconnect_delay);
continue;
}
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
snapshot_failure_injected = true;
Err(anyhow!("injected task snapshot transport failure"))
} else {
fetch_task_snapshots_in(current_session, state)
};
let task_snapshots = match snapshot_request {
Ok(snapshots) => snapshots,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime terminal snapshot observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
let process_status_request =
if inject_process_status_failure && !process_status_failure_injected {
process_status_failure_injected = true;
Err(anyhow!("injected process-status transport failure"))
} else {
fetch_current_process_status_in(current_session, state)
};
let (process_statuses, process_status) = match process_status_request {
Ok(status) => status,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime terminal process observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
if !has_current_runtime_task(&task_snapshots) {
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
record.status_code =
whole_process_status_code(record.status_code, &task_snapshots);
record.node_report = json!({
"terminal_event": record.node_report.get("terminal_event"),
"task_snapshots": task_snapshots,
"process_status": process_status,
"process_statuses": process_statuses,
});
}
emit(outcome);
return Ok(());
}
}
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
snapshot_failure_injected = true;
Err(anyhow!("injected task snapshot transport failure"))
@ -1221,6 +1190,22 @@ pub(crate) fn observe_services_runtime(
}
};
reconnect_delay = Duration::from_millis(100);
if !has_current_runtime_task(&task_snapshots) {
if let Some(mut outcome) =
terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
{
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
record.node_report = json!({
"terminal_event": record.node_report.get("terminal_event"),
"task_snapshots": task_snapshots,
"process_status": process_status,
"process_statuses": process_statuses,
});
}
emit(outcome);
return Ok(());
}
}
if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
let failed_task = failed_task.to_owned();
let failed_event = events
@ -1352,7 +1337,9 @@ pub(crate) fn observe_services_runtime(
continue;
}
};
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
if let Some(mut outcome) =
terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
{
let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
Ok(snapshots) => snapshots,
Err(snapshot_error) => {
@ -1370,8 +1357,6 @@ pub(crate) fn observe_services_runtime(
};
if !has_current_runtime_task(&task_snapshots) {
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
record.status_code =
whole_process_status_code(record.status_code, &task_snapshots);
record.node_report = json!({
"terminal_event": record.node_report.get("terminal_event"),
"task_snapshots": task_snapshots,
@ -1547,6 +1532,7 @@ fn has_current_runtime_task(task_snapshots: &Value) -> bool {
})
}
#[cfg(test)]
pub(crate) fn whole_process_status_code(
main_status_code: Option<i32>,
task_snapshots: &Value,
@ -1582,66 +1568,75 @@ pub(crate) fn whole_process_status_code(
fn terminal_runtime_outcome(
coordinator: &str,
state: &AdapterState,
_state: &AdapterState,
events: &Value,
process_status: Option<&Value>,
) -> Option<RuntimeContinuationOutcome> {
let final_result = process_status?
.get("final_result")
.and_then(Value::as_str)?;
let status_code = match final_result {
"completed" => 0,
"failed" | "cancelled" => 1,
_ => return None,
};
let event = events
.get("events")
.and_then(Value::as_array)?
.get(state.runtime_event_count..)?
.iter()
.rev()
.find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?;
let status_code = event
.get("status_code")
.and_then(Value::as_i64)
.map(|status| status as i32)
.or_else(
|| match event.get("terminal_state").and_then(Value::as_str) {
Some("completed") => Some(0),
Some("failed" | "cancelled") => Some(1),
_ => None,
},
);
.and_then(Value::as_array)
.and_then(|events| {
events.iter().rev().find(|event| {
event.get("executor").and_then(Value::as_str) == Some("coordinator_main")
})
});
Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord {
coordinator: coordinator.to_owned(),
node: event
.get("node")
.and_then(|event| event.get("node"))
.and_then(Value::as_str)
.or_else(|| {
process_status
.and_then(|status| status.get("connected_nodes"))
.and_then(Value::as_array)
.and_then(|nodes| nodes.first())
.and_then(Value::as_str)
})
.unwrap_or("coordinator-main")
.to_owned(),
node_report: json!({ "terminal_event": event }),
node_report: json!({
"terminal_event": event,
"process_status": process_status,
}),
task_events: events.clone(),
placed_task_launched: true,
status_code,
status_code: Some(status_code),
stdout_bytes: event
.get("stdout_bytes")
.and_then(|event| event.get("stdout_bytes"))
.and_then(Value::as_u64)
.unwrap_or(0),
stderr_bytes: event
.get("stderr_bytes")
.and_then(|event| event.get("stderr_bytes"))
.and_then(Value::as_u64)
.unwrap_or(0),
stdout_tail: event
.get("stdout_tail")
.and_then(|event| event.get("stdout_tail"))
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned(),
stderr_tail: event
.get("stderr_tail")
.and_then(|event| event.get("stderr_tail"))
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned(),
stdout_truncated: event
.get("stdout_truncated")
.and_then(|event| event.get("stdout_truncated"))
.and_then(Value::as_bool)
.unwrap_or(false),
stderr_truncated: event
.get("stderr_truncated")
.and_then(|event| event.get("stderr_truncated"))
.and_then(Value::as_bool)
.unwrap_or(false),
artifact_path: event
.get("artifact_path")
.and_then(|event| event.get("artifact_path"))
.and_then(Value::as_str)
.map(str::to_owned),
event_count: events
@ -1727,6 +1722,53 @@ mod transactional_launch_tests {
assert!(error.contains("no virtual process was created"));
}
#[test]
fn durable_process_summary_is_terminal_authority_after_event_rotation() {
let state = AdapterState {
runtime_event_count: 10_000,
..AdapterState::default()
};
let completed = terminal_runtime_outcome(
"127.0.0.1:1",
&state,
&json!({ "events": [] }),
Some(&json!({
"process": state.process.as_str(),
"lifecycle": "recent_terminal",
"final_result": "completed",
"connected_nodes": []
})),
)
.expect("the durable summary should terminate observation");
let RuntimeContinuationOutcome::Terminal(completed) = completed else {
panic!("expected terminal outcome");
};
assert_eq!(completed.status_code, Some(0));
let failed = terminal_runtime_outcome(
"127.0.0.1:1",
&state,
&json!({
"events": [{
"executor": "coordinator_main",
"terminal_state": "completed",
"status_code": 0
}]
}),
Some(&json!({
"process": state.process.as_str(),
"lifecycle": "recent_terminal",
"final_result": "failed",
"connected_nodes": []
})),
)
.expect("the aggregate summary should override a successful main event");
let RuntimeContinuationOutcome::Terminal(failed) = failed else {
panic!("expected terminal outcome");
};
assert_eq!(failed.status_code, Some(1));
}
#[test]
fn failed_debug_launch_reconnects_and_aborts_the_process() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();