michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-09ca780bd67a

Browse source 
Source commit: 09ca780bd67a00467e78139cf38466b8201b66ca

Public tree identity: sha256:2f744c260b5fc2cf074ad9129f97f87f03714902e5b9fe174128afdc342ec2d0
This commit is contained in:
Clusterflux release 2026-07-19 18:38:40 +02:00
commit eb1077f380
221 changed files with 81144 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

666
crates/clusterflux-cli/src/process_events.rs Normal file
View file

@ -0,0 +1,666 @@
use std::path::Path;
use serde_json::{json, Value};
use crate::errors::{
cli_error_summary, cli_error_summary_for_category, cli_error_summary_with_default,
message_mentions_locality_failure,
};
fn task_event_values(task_events: Option<&Value>) -> Vec<&Value> {
task_events
.and_then(|task_events| task_events.pointer("/response/events"))
.and_then(Value::as_array)
.map(|events| events.iter().collect())
.unwrap_or_default()
}
fn event_string(event: &Value, field: &str) -> Option<String> {
event.get(field).and_then(Value::as_str).map(str::to_owned)
}
fn event_u64(event: &Value, field: &str) -> Option<u64> {
event.get(field).and_then(Value::as_u64)
}
pub(crate) fn task_summaries(task_events: Option<&Value>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.map(|event| {
let task = event_string(event, "task").unwrap_or_else(|| "unknown".to_owned());
let terminal_state =
event_string(event, "terminal_state").unwrap_or_else(|| "unknown".to_owned());
let node = event_string(event, "node");
let placement = event.get("placement");
let placement_reasons = placement
.and_then(|placement| placement.get("reasons"))
.cloned()
.unwrap_or_else(|| json!([]));
let placement_score = placement
.and_then(|placement| placement.get("score"))
.cloned()
.unwrap_or(Value::Null);
let failure_reason = task_failure_reason(event);
let locality_failure = task_locality_failure_from_reason(&failure_reason);
let machine_error = task_failure_machine_error_from_reason(event, &failure_reason);
json!({
"process": event_string(event, "process"),
"task": task,
"state": terminal_state,
"environment": event.get("environment").cloned().unwrap_or_else(|| json!("unknown_from_task_event")),
"environment_digest": event.get("environment_digest").cloned().unwrap_or(Value::Null),
"node_placement": {
"node": node,
"source": "coordinator_task_event",
"score": placement_score,
"reasons": placement_reasons,
"explanation_available": placement.is_some(),
},
"failure_reason": failure_reason,
"locality_failure": locality_failure,
"machine_error": machine_error,
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
})
})
.collect(),
)
}
fn task_failure_reason(event: &Value) -> Value {
match event.get("terminal_state").and_then(Value::as_str) {
Some("failed") => {
if let Some(stderr) = event.get("stderr_tail").and_then(Value::as_str) {
if !stderr.is_empty() {
return json!(redact_secret_like_text(stderr).0);
}
}
if let Some(status_code) = event.get("status_code").and_then(Value::as_i64) {
return json!(format!("task exited with status {status_code}"));
}
json!("task failed")
}
Some("cancelled") => json!("task cancelled"),
_ => Value::Null,
}
}
fn task_failure_machine_error_from_reason(event: &Value, reason: &Value) -> Value {
match event.get("terminal_state").and_then(Value::as_str) {
Some("failed") => {
let reason = reason.as_str().unwrap_or("task failed").to_owned();
let mut summary = cli_error_summary_with_default(&reason, "program");
if let Some(object) = summary.as_object_mut() {
if message_mentions_locality_failure(&reason.to_ascii_lowercase()) {
object.insert("locality_failure".to_owned(), json!(true));
object.insert(
"next_actions".to_owned(),
json!(locality_failure_next_actions(&reason)),
);
}
}
summary
}
Some("cancelled") => cli_error_summary_for_category("program", "task cancelled"),
_ => Value::Null,
}
}
fn task_locality_failure_from_reason(reason: &Value) -> Value {
let Some(reason) = reason.as_str() else {
return Value::Null;
};
let lower = reason.to_ascii_lowercase();
if !message_mentions_locality_failure(&lower) {
return Value::Null;
}
let affected_data = if lower.contains("source snapshot") {
"source_snapshot"
} else if lower.contains("artifact") {
"artifact"
} else {
"direct_transfer"
};
json!({
"category": "connectivity",
"affected_data": affected_data,
"reason": reason,
"coordinator_bulk_relay_used": false,
"safe_failure": true,
"safe_next_actions": locality_failure_next_actions(reason),
})
}
fn locality_failure_next_actions(reason: &str) -> Vec<&'static str> {
let lower = reason.to_ascii_lowercase();
if lower.contains("source snapshot") {
return vec![
"attach or select a node that already has the required source snapshot",
"rerun source preparation on an attached node",
"restore direct node-to-node connectivity and retry",
"do not rely on coordinator bulk source relay",
];
}
if lower.contains("artifact") {
return vec![
"attach or select a node that already has the required artifact",
"explicitly export or download the artifact before retrying",
"restore direct node-to-node connectivity and retry",
"do not rely on coordinator bulk artifact relay",
];
}
vec![
"check node direct connectivity and NAT traversal",
"attach a node with the needed source/artifact locality",
"retry after node connectivity is restored",
"do not rely on coordinator bulk relay",
]
}
pub(crate) fn process_state_from_tasks(task_events: Option<&Value>) -> &'static str {
let events = task_event_values(task_events);
if events.is_empty() {
return "no_tasks_observed";
}
if events.iter().any(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "failed")
}) {
return "has_failed_tasks";
}
if events.iter().any(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "cancelled")
}) {
return "has_cancelled_tasks";
}
if events.iter().all(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "completed")
}) {
return "completed_tasks_observed";
}
"tasks_observed"
}
pub(crate) fn log_entries(task_events: Option<&Value>, task_filter: Option<&str>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.filter(|event| {
task_filter.is_none_or( |task_filter| {
event
.get("task")
.and_then(Value::as_str)
.is_some_and(|task| task == task_filter)
})
})
.map(|event| {
let stdout_tail = event
.get("stdout_tail")
.and_then(Value::as_str)
.unwrap_or("");
let stderr_tail = event
.get("stderr_tail")
.and_then(Value::as_str)
.unwrap_or("");
let (stdout_tail, stdout_tail_redacted) = redact_secret_like_text(stdout_tail);
let (stderr_tail, stderr_tail_redacted) = redact_secret_like_text(stderr_tail);
json!({
"process": event_string(event, "process"),
"task": event_string(event, "task"),
"node": event_string(event, "node"),
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
"stdout_truncated": event.get("stdout_truncated").and_then(Value::as_bool).unwrap_or(false),
"stderr_truncated": event.get("stderr_truncated").and_then(Value::as_bool).unwrap_or(false),
"capped": true,
"secret_like_values_redacted": stdout_tail_redacted || stderr_tail_redacted,
"redacted_fields": redacted_log_fields(stdout_tail_redacted, stderr_tail_redacted),
})
})
.collect(),
)
}
fn redacted_log_fields(
stdout_tail_redacted: bool,
stderr_tail_redacted: bool,
) -> Vec<&'static str> {
let mut fields = Vec::new();
if stdout_tail_redacted {
fields.push("stdout_tail");
}
if stderr_tail_redacted {
fields.push("stderr_tail");
}
fields
}
fn redact_secret_like_text(text: &str) -> (String, bool) {
let markers = [
"access_token=",
"access_token:",
"refresh_token=",
"refresh_token:",
"id_token=",
"id_token:",
"api_key=",
"api_key:",
"api-key=",
"api-key:",
"token=",
"token:",
"secret=",
"secret:",
"password=",
"password:",
"passwd=",
"passwd:",
"bearer ",
];
let mut output = text.to_owned();
let mut redacted = false;
for marker in markers {
let (updated, changed) = redact_marker_values(output, marker);
output = updated;
redacted |= changed;
}
(output, redacted)
}
fn redact_marker_values(mut text: String, marker: &str) -> (String, bool) {
let mut changed = false;
let mut search_start = 0;
loop {
let lower = text.to_ascii_lowercase();
let Some(relative) = lower[search_start..].find(marker) else {
break;
};
let value_start = search_start + relative + marker.len();
let value_end = text[value_start..]
.char_indices()
.find_map(|(offset, character)| {
(character.is_whitespace()
|| matches!(
character,
'&' | '"' | '\'' | '`' | '<' | '>' | ',' | ';' | ')' | ']'
))
.then_some(value_start + offset)
})
.unwrap_or(text.len());
if value_start == value_end {
search_start = value_end;
if search_start >= text.len() {
break;
}
continue;
}
if text[value_start..value_end].starts_with("[redacted") {
search_start = value_end;
if search_start >= text.len() {
break;
}
continue;
}
text.replace_range(value_start..value_end, "[redacted]");
changed = true;
search_start = value_start + "[redacted]".len();
if search_start >= text.len() {
break;
}
}
(text, changed)
}
pub(crate) fn artifact_summaries(task_events: Option<&Value>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.filter_map(|event| {
let path = event.get("artifact_path").and_then(Value::as_str)?;
let node = event_string(event, "node");
Some(json!({
"artifact": artifact_name_from_path(path),
"path": path,
"producer_task": event_string(event, "task"),
"producer_node": node,
"process": event_string(event, "process"),
"digest": event.get("artifact_digest").cloned().unwrap_or(Value::Null),
"size_bytes": event.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
"state": if event.get("artifact_digest").is_some() { "metadata_flushed" } else { "metadata_without_digest" },
"known_locations": node.into_iter().collect::<Vec<_>>(),
"durable_storage": false,
}))
})
.collect(),
)
}
fn artifact_name_from_path(path: &str) -> String {
path.rsplit('/').next().unwrap_or(path).to_owned()
}
fn response_error_message(response: &Value, fallback: &str) -> String {
response
.get("message")
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| fallback.to_owned())
}
pub(crate) fn artifact_response_machine_error(
response: &Value,
fallback: &str,
default_category: &'static str,
) -> Value {
let message = response_error_message(response, fallback);
cli_error_summary_with_default(&message, default_category)
}
pub(crate) fn process_restart_request_summary(
response: &Value,
requires_confirmation: bool,
) -> Value {
if response.get("type").and_then(Value::as_str) != Some("process_started") {
let message = response_error_message(response, "coordinator rejected process restart");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "restart_virtual_process",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
json!({
"status": "process_started",
"operation": "restart_virtual_process",
"accepted": true,
"process": response.get("process").cloned().unwrap_or(Value::Null),
"coordinator_epoch": response.get("epoch").cloned().unwrap_or(Value::Null),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"website_required": false,
"single_active_process_boundary": true,
})
}
pub(crate) fn process_cancel_request_summary(
response: &Value,
requires_confirmation: bool,
) -> Value {
if response.get("type").and_then(Value::as_str) != Some("process_cancellation_requested") {
let message = response_error_message(response, "coordinator rejected process cancel");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "cancel_virtual_process",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"whole_process_cancel_available": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
let cancelled_tasks = response
.get("cancelled_tasks")
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0);
json!({
"status": "process_cancellation_requested",
"operation": "cancel_virtual_process",
"accepted": true,
"process": response.get("process").cloned().unwrap_or(Value::Null),
"cancelled_task_count": cancelled_tasks,
"cancelled_tasks": response.get("cancelled_tasks").cloned().unwrap_or_else(|| json!([])),
"affected_nodes": response.get("affected_nodes").cloned().unwrap_or_else(|| json!([])),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"website_required": false,
"whole_process_cancel_available": true,
"node_must_poll_task_control": true,
"new_task_launches_blocked": true,
"surviving_state_visibility": "task and artifact state remains visible after terminal task events are reported",
})
}
pub(crate) fn task_restart_request_summary(response: &Value, requires_confirmation: bool) -> Value {
if response.get("type").and_then(Value::as_str) != Some("task_restart") {
let message = response_error_message(response, "coordinator rejected task restart");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "restart_selected_task",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"clean_boundary_required": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
json!({
"status": "task_restart",
"operation": "restart_selected_task",
"accepted": response.get("accepted").cloned().unwrap_or(json!(false)),
"process": response.get("process").cloned().unwrap_or(Value::Null),
"task": response.get("task").cloned().unwrap_or(Value::Null),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"clean_boundary_required": true,
"clean_boundary_available": response
.get("clean_boundary_available")
.cloned()
.unwrap_or(json!(false)),
"active_task": response
.get("active_task")
.cloned()
.unwrap_or(json!(false)),
"completed_event_observed": response
.get("completed_event_observed")
.cloned()
.unwrap_or(json!(false)),
"requires_whole_process_restart": response
.get("requires_whole_process_restart")
.cloned()
.unwrap_or(json!(true)),
"message": response.get("message").cloned().unwrap_or(Value::Null),
"audit_event": response.get("audit_event").cloned().unwrap_or(Value::Null),
"charged_debug_read_bytes": response
.get("charged_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"used_debug_read_bytes": response
.get("used_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"debug_reads_quota_limited": true,
"website_required": false,
})
}
pub(crate) fn artifact_download_session_summary(response: &Value) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
let message = response_error_message(response, "coordinator rejected artifact download");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"link_issued": false,
"explicit_user_action_required": true,
"error": message.clone(),
"machine_error": cli_error_summary_with_default(&message, "connectivity"),
});
}
let link = response.get("link").unwrap_or(&Value::Null);
json!({
"status": "download_link_issued",
"link_issued": true,
"explicit_user_action_required": true,
"coordinator_preflight": "completed_before_link_issued",
"tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
"project": link.get("project").cloned().unwrap_or(Value::Null),
"process": link.get("process").cloned().unwrap_or(Value::Null),
"artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
"actor": link.get("actor").cloned().unwrap_or(Value::Null),
"source": link.get("source").cloned().unwrap_or(Value::Null),
"url_path": link.get("url_path").cloned().unwrap_or(Value::Null),
"expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
"max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
"token_material_returned": false,
"scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
"policy_context_digest_present": link.get("policy_context_digest").is_some(),
"authorization_required": true,
"short_lived": link.get("expires_at_epoch_seconds").is_some(),
"guessable_public_url": false,
"cross_tenant_usable": false,
"unauthorized_project_usable": false,
"default_durable_store_assumed": false,
})
}
pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
return json!([]);
}
let link = response.get("link").unwrap_or(&Value::Null);
json!([{
"grant": "artifact_download",
"description": "download scoped artifact bytes to the requesting machine",
"risk": "authorized artifact bytes leave the retaining node or explicit storage through an explicit download/export operation",
"coordinator_policy_limited": true,
"authorization_required": true,
"explicit_user_action_required": true,
"tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
"project": link.get("project").cloned().unwrap_or(Value::Null),
"process": link.get("process").cloned().unwrap_or(Value::Null),
"artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
"actor": link.get("actor").cloned().unwrap_or(Value::Null),
"source": link.get("source").cloned().unwrap_or(Value::Null),
"max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
"expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
"short_lived": link.get("expires_at_epoch_seconds").is_some(),
"scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
"token_material_returned": false,
"guessable_public_url": false,
"cross_tenant_reuse_allowed": false,
"unauthorized_project_reuse_allowed": false,
"default_durable_store_assumed": false,
"private_website_required": false,
}])
}
pub(crate) fn artifact_export_plan_summary(response: &Value, to: &Path) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_export_plan") {
let message = response_error_message(response, "coordinator rejected artifact export");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"explicit_user_action": true,
"local_path": to,
"local_bytes_written_by_cli": false,
"default_durable_store_assumed": false,
"error": message.clone(),
"machine_error": cli_error_summary_with_default(&message, "connectivity"),
});
}
let plan = response.get("plan").unwrap_or(&Value::Null);
json!({
"status": "transfer_plan_created",
"explicit_user_action": true,
"local_path": to,
"local_bytes_written_by_cli": false,
"writes_require_data_plane_followup": true,
"default_durable_store_assumed": false,
"artifact_size_bytes": response.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
"source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
"receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
"transport": plan.get("transport").cloned().unwrap_or(Value::Null),
"artifact": plan.pointer("/scope/object/Artifact").cloned().unwrap_or(Value::Null),
"tenant": plan.pointer("/scope/tenant").cloned().unwrap_or(Value::Null),
"project": plan.pointer("/scope/project").cloned().unwrap_or(Value::Null),
"process": plan.pointer("/scope/process").cloned().unwrap_or(Value::Null),
"coordinator_assisted_rendezvous": plan.get("coordinator_assisted_rendezvous").cloned().unwrap_or(Value::Null),
"coordinator_bulk_relay_allowed": plan.get("coordinator_bulk_relay_allowed").cloned().unwrap_or(Value::Null),
"authorization_digest_present": plan.get("authorization_digest").is_some(),
})
}
pub(crate) fn task_event_count(task_events: Option<&Value>) -> usize {
task_events
.and_then(|task_events| task_events.pointer("/response/events"))
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0)
}
pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
let current_usage = quota_current_usage(attached_nodes, task_events);
let next_blocked_action = quota_next_blocked_action(&current_usage);
json!({
"source": "cli_project_status_summary",
"current_usage": current_usage,
"limits": quota_limits_value(),
"next_blocked_action": next_blocked_action,
"private_abuse_heuristics_exposed": false,
})
}
pub(crate) fn quota_current_usage(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
let attached_node_count = attached_nodes
.get("count")
.and_then(Value::as_u64)
.unwrap_or(0);
let online_node_count = attached_nodes
.get("online")
.and_then(Value::as_u64)
.unwrap_or(0);
json!({
"attached_nodes": attached_node_count,
"online_nodes": online_node_count,
"observed_task_events": task_event_count(task_events),
"artifact_download_bytes": 0,
"rendezvous_attempts": 0,
"hosted_wasm_processes": 0,
})
}
pub(crate) fn quota_limits_value() -> Value {
json!({
"source": "coordinator",
"configured": false,
"message": "connect to the selected coordinator to read its scoped quota configuration",
})
}
pub(crate) fn quota_next_blocked_action(current_usage: &Value) -> Value {
if current_usage
.get("online_nodes")
.and_then(Value::as_u64)
.unwrap_or(0)
== 0
{
return json!({
"action": "node_work_requires_online_attached_node",
"category": "capability",
"quota_related": false,
"message": "no online attached node is visible for work that requires a user node",
"machine_error": cli_error_summary_for_category(
"capability",
"no online attached node is visible for work that requires a user node"
)
});
}
Value::Null
}