Classify CLI failure reports
This commit is contained in:
Michel Paulissen
parent
218ab65e47
commit
50d298ead7
5 changed files with 370 additions and 6 deletions
|
|
@ -1807,6 +1807,8 @@ fn human_report(value: &Value) -> String {
|
|||
));
|
||||
}
|
||||
}
|
||||
push_machine_error_line(&mut lines, value.get("machine_error"));
|
||||
push_machine_error_line(&mut lines, value.pointer("/run_start/machine_error"));
|
||||
if let Some(flow) = value.get("human_flow") {
|
||||
if let Some(device) = flow.get("Device") {
|
||||
lines.push("flow: device".to_owned());
|
||||
|
|
@ -2013,10 +2015,221 @@ fn push_nested_string_field(lines: &mut Vec<String>, value: &Value, key: &str, l
|
|||
}
|
||||
}
|
||||
|
||||
fn push_machine_error_line(lines: &mut Vec<String>, machine_error: Option<&Value>) {
|
||||
let Some(machine_error) = machine_error else {
|
||||
return;
|
||||
};
|
||||
if machine_error.is_null() {
|
||||
return;
|
||||
}
|
||||
let category = machine_error
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("unknown");
|
||||
let exit_code = machine_error
|
||||
.get("stable_exit_code")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(1);
|
||||
lines.push(format!("error category: {category} (exit {exit_code})"));
|
||||
if let Some(next_actions) = machine_error.get("next_actions").and_then(Value::as_array) {
|
||||
let actions = next_actions
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.collect::<Vec<_>>();
|
||||
if !actions.is_empty() {
|
||||
lines.push(format!("error next: {}", actions.join("; ")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compact_json(value: &Value) -> String {
|
||||
serde_json::to_string(value).unwrap_or_else(|_| "<unprintable>".to_owned())
|
||||
}
|
||||
|
||||
fn cli_error_summary(message: &str) -> Value {
|
||||
let category = classify_cli_error_message(message);
|
||||
cli_error_summary_for_category(category, message)
|
||||
}
|
||||
|
||||
fn cli_error_summary_with_default(message: &str, default_category: &'static str) -> Value {
|
||||
let category = classify_cli_error_message(message);
|
||||
let category = if category == "unknown" {
|
||||
default_category
|
||||
} else {
|
||||
category
|
||||
};
|
||||
cli_error_summary_for_category(category, message)
|
||||
}
|
||||
|
||||
fn cli_error_summary_for_category(category: &'static str, message: &str) -> Value {
|
||||
json!({
|
||||
"category": category,
|
||||
"stable_exit_code": cli_error_exit_code(category),
|
||||
"process_exit_code_applied": false,
|
||||
"retryable_after_user_action": cli_error_retryable_after_user_action(category),
|
||||
"message": message,
|
||||
"safe_failure": true,
|
||||
"next_actions": cli_error_next_actions(category),
|
||||
})
|
||||
}
|
||||
|
||||
fn classify_cli_error_message(message: &str) -> &'static str {
|
||||
let message = message.to_ascii_lowercase();
|
||||
if message.contains("already has active virtual process") || message.contains("active process")
|
||||
{
|
||||
return "active_process";
|
||||
}
|
||||
if message.contains("no capable node")
|
||||
|| message.contains("missing capability")
|
||||
|| message.contains("capability")
|
||||
|| message.contains("placement failed")
|
||||
|| message.contains("cmd.run")
|
||||
|| message.contains("node required")
|
||||
{
|
||||
return "capability";
|
||||
}
|
||||
if message.contains("quota")
|
||||
|| message.contains("resource limit")
|
||||
|| message.contains("limit exceeded")
|
||||
|| message.contains("metering")
|
||||
{
|
||||
return "quota";
|
||||
}
|
||||
if message.contains("policy")
|
||||
|| message.contains("disallowed")
|
||||
|| message.contains("not allowed")
|
||||
|| message.contains("suspended")
|
||||
|| message.contains("blocked by")
|
||||
|| message.contains("denied")
|
||||
{
|
||||
return "policy";
|
||||
}
|
||||
if message.contains("unauthenticated")
|
||||
|| message.contains("not authenticated")
|
||||
|| message.contains("login required")
|
||||
|| message.contains("browser login required")
|
||||
|| message.contains("missing session")
|
||||
|| message.contains("no cli session")
|
||||
|| message.contains("401")
|
||||
{
|
||||
return "authentication";
|
||||
}
|
||||
if message.contains("unauthorized")
|
||||
|| message.contains("not authorized")
|
||||
|| message.contains("forbidden")
|
||||
|| message.contains("permission")
|
||||
|| message.contains("tenant mismatch")
|
||||
|| message.contains("project mismatch")
|
||||
{
|
||||
return "authorization";
|
||||
}
|
||||
if message.contains("failed to connect")
|
||||
|| message.contains("connection refused")
|
||||
|| message.contains("closed session")
|
||||
|| message.contains("timed out")
|
||||
|| message.contains("timeout")
|
||||
|| message.contains("name resolution")
|
||||
|| message.contains("network unreachable")
|
||||
|| message.contains("dns")
|
||||
{
|
||||
return "connectivity";
|
||||
}
|
||||
if message.contains("missing environment")
|
||||
|| message.contains("environment mismatch")
|
||||
|| message.contains("incompatible environment")
|
||||
|| message.contains("containerfile")
|
||||
|| message.contains("dockerfile")
|
||||
|| message.contains("envs/")
|
||||
{
|
||||
return "environment";
|
||||
}
|
||||
if message.contains("task exited")
|
||||
|| message.contains("exit status")
|
||||
|| message.contains("status code")
|
||||
|| message.contains("stderr")
|
||||
|| message.contains("panic")
|
||||
|| message.contains("program error")
|
||||
|| message.contains("command failed")
|
||||
{
|
||||
return "program";
|
||||
}
|
||||
"unknown"
|
||||
}
|
||||
|
||||
fn cli_error_exit_code(category: &str) -> i64 {
|
||||
match category {
|
||||
"authentication" => 20,
|
||||
"authorization" => 21,
|
||||
"quota" => 22,
|
||||
"policy" => 23,
|
||||
"capability" => 24,
|
||||
"connectivity" => 25,
|
||||
"environment" => 26,
|
||||
"program" => 27,
|
||||
"active_process" => 28,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn cli_error_retryable_after_user_action(category: &str) -> bool {
|
||||
matches!(
|
||||
category,
|
||||
"authentication"
|
||||
| "authorization"
|
||||
| "quota"
|
||||
| "policy"
|
||||
| "capability"
|
||||
| "connectivity"
|
||||
| "environment"
|
||||
| "program"
|
||||
| "active_process"
|
||||
)
|
||||
}
|
||||
|
||||
fn cli_error_next_actions(category: &str) -> Vec<&'static str> {
|
||||
match category {
|
||||
"authentication" => vec!["disasmer login --browser", "disasmer auth status"],
|
||||
"authorization" => vec![
|
||||
"disasmer auth status",
|
||||
"check tenant/project selection",
|
||||
"ask an admin to grant access",
|
||||
],
|
||||
"quota" => vec![
|
||||
"disasmer quota status",
|
||||
"reduce concurrent work or wait for usage to fall",
|
||||
],
|
||||
"policy" => vec![
|
||||
"disasmer doctor",
|
||||
"check coordinator policy for this action",
|
||||
],
|
||||
"capability" => vec![
|
||||
"disasmer node list",
|
||||
"attach a node with the required capabilities",
|
||||
"check tenant/project on the attached node",
|
||||
],
|
||||
"connectivity" => vec![
|
||||
"disasmer doctor",
|
||||
"check the coordinator endpoint and network reachability",
|
||||
],
|
||||
"environment" => vec![
|
||||
"disasmer inspect",
|
||||
"check envs/<name>/Containerfile or envs/<name>/Dockerfile",
|
||||
],
|
||||
"program" => vec!["disasmer logs", "fix the program or task command and rerun"],
|
||||
"active_process" => vec![
|
||||
"disasmer process status",
|
||||
"disasmer logs",
|
||||
"disasmer process restart --yes",
|
||||
"disasmer process cancel --yes",
|
||||
"wait for the active process to finish",
|
||||
],
|
||||
_ => vec![
|
||||
"disasmer doctor",
|
||||
"rerun with --json for machine-readable details",
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
match cli.command {
|
||||
|
|
@ -2568,6 +2781,7 @@ fn task_summaries(task_events: Option<&Value>) -> Value {
|
|||
"explanation_available": placement.is_some(),
|
||||
},
|
||||
"failure_reason": task_failure_reason(event),
|
||||
"machine_error": task_failure_machine_error(event),
|
||||
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
|
||||
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
|
||||
})
|
||||
|
|
@ -2594,6 +2808,20 @@ fn task_failure_reason(event: &Value) -> Value {
|
|||
}
|
||||
}
|
||||
|
||||
fn task_failure_machine_error(event: &Value) -> Value {
|
||||
match event.get("terminal_state").and_then(Value::as_str) {
|
||||
Some("failed") => {
|
||||
let reason = task_failure_reason(event)
|
||||
.as_str()
|
||||
.unwrap_or("task failed")
|
||||
.to_owned();
|
||||
cli_error_summary_with_default(&reason, "program")
|
||||
}
|
||||
Some("cancelled") => cli_error_summary_for_category("program", "task cancelled"),
|
||||
_ => Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
fn process_state_from_tasks(task_events: Option<&Value>) -> &'static str {
|
||||
let events = task_event_values(task_events);
|
||||
if events.is_empty() {
|
||||
|
|
@ -2933,7 +3161,11 @@ fn quota_next_blocked_action(current_usage: &Value) -> Value {
|
|||
"action": "node_work_requires_online_attached_node",
|
||||
"category": "capability",
|
||||
"quota_related": false,
|
||||
"message": "no online attached node is visible for work that requires a user node"
|
||||
"message": "no online attached node is visible for work that requires a user node",
|
||||
"machine_error": cli_error_summary_for_category(
|
||||
"capability",
|
||||
"no online attached node is visible for work that requires a user node"
|
||||
)
|
||||
});
|
||||
}
|
||||
Value::Null
|
||||
|
|
@ -3568,10 +3800,26 @@ fn run_start_summary(response: &Value) -> Value {
|
|||
.and_then(Value::as_str)
|
||||
.unwrap_or("coordinator rejected run");
|
||||
let active_conflict = message.contains("already has active virtual process");
|
||||
let machine_error = if active_conflict {
|
||||
cli_error_summary_for_category("active_process", message)
|
||||
} else {
|
||||
cli_error_summary(message)
|
||||
};
|
||||
let error_category = machine_error
|
||||
.get("category")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("unknown"));
|
||||
let stable_exit_code = machine_error
|
||||
.get("stable_exit_code")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!(1));
|
||||
json!({
|
||||
"status": if active_conflict { "blocked_active_process" } else { "coordinator_rejected" },
|
||||
"accepted": false,
|
||||
"category": if active_conflict { "active_process_already_running" } else { "coordinator" },
|
||||
"error_category": error_category,
|
||||
"stable_exit_code": stable_exit_code,
|
||||
"machine_error": machine_error,
|
||||
"message": message,
|
||||
"restart": false,
|
||||
"single_active_process_boundary": true,
|
||||
|
|
@ -4034,7 +4282,21 @@ impl JsonLineSession {
|
|||
fn request(&mut self, value: Value) -> Result<Value> {
|
||||
let response = self.request_allow_error(value)?;
|
||||
if response.get("type").and_then(Value::as_str) == Some("error") {
|
||||
anyhow::bail!("coordinator error: {response}");
|
||||
let message = response
|
||||
.get("message")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| response.to_string());
|
||||
let machine_error = cli_error_summary(&message);
|
||||
let category = machine_error
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("unknown");
|
||||
let exit_code = machine_error
|
||||
.get("stable_exit_code")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(1);
|
||||
anyhow::bail!("coordinator error ({category}, exit {exit_code}): {response}");
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
|
@ -4111,6 +4373,48 @@ mod tests {
|
|||
Cli::parse_from(args)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_error_classifier_distinguishes_mvp_failure_categories() {
|
||||
for (message, category, exit_code) in [
|
||||
(
|
||||
"login required before running this command",
|
||||
"authentication",
|
||||
20,
|
||||
),
|
||||
("unauthorized tenant action", "authorization", 21),
|
||||
("quota unavailable: resource limit exceeded", "quota", 22),
|
||||
("policy denied native command execution", "policy", 23),
|
||||
(
|
||||
"scheduler placement failed: no capable node for placement: project mismatch",
|
||||
"capability",
|
||||
24,
|
||||
),
|
||||
(
|
||||
"failed to connect to coordinator: connection refused",
|
||||
"connectivity",
|
||||
25,
|
||||
),
|
||||
(
|
||||
"missing environment envs/linux/Containerfile",
|
||||
"environment",
|
||||
26,
|
||||
),
|
||||
("task exited with status 101 after panic", "program", 27),
|
||||
(
|
||||
"project already has active virtual process vp-current",
|
||||
"active_process",
|
||||
28,
|
||||
),
|
||||
] {
|
||||
let summary = cli_error_summary(message);
|
||||
assert_eq!(summary["category"], category, "{message}");
|
||||
assert_eq!(summary["stable_exit_code"], exit_code, "{message}");
|
||||
assert_eq!(summary["safe_failure"], true);
|
||||
assert_eq!(summary["process_exit_code_applied"], false);
|
||||
assert!(summary["next_actions"].as_array().unwrap().len() >= 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_version_is_available() {
|
||||
let command = Cli::command();
|
||||
|
|
@ -4417,6 +4721,16 @@ mod tests {
|
|||
blocked["run_start"]["category"],
|
||||
"active_process_already_running"
|
||||
);
|
||||
assert_eq!(blocked["run_start"]["error_category"], "active_process");
|
||||
assert_eq!(blocked["run_start"]["stable_exit_code"], 28);
|
||||
assert_eq!(
|
||||
blocked["run_start"]["machine_error"]["category"],
|
||||
"active_process"
|
||||
);
|
||||
assert_eq!(
|
||||
blocked["run_start"]["machine_error"]["stable_exit_code"],
|
||||
28
|
||||
);
|
||||
assert_eq!(blocked["run_start"]["safe_failure"], true);
|
||||
assert!(blocked["run_start"]["next_actions"]
|
||||
.as_array()
|
||||
|
|
@ -4425,6 +4739,24 @@ mod tests {
|
|||
.any(|action| action == "disasmer process restart --yes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_rejection_reports_machine_readable_error_category() {
|
||||
let rejected = run_start_summary(&json!({
|
||||
"type": "error",
|
||||
"message": "quota unavailable: resource limit exceeded for api_calls"
|
||||
}));
|
||||
|
||||
assert_eq!(rejected["status"], "coordinator_rejected");
|
||||
assert_eq!(rejected["error_category"], "quota");
|
||||
assert_eq!(rejected["stable_exit_code"], 22);
|
||||
assert_eq!(rejected["machine_error"]["category"], "quota");
|
||||
assert!(rejected["machine_error"]["next_actions"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|action| action == "disasmer quota status"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_only_run_executes_ephemeral_local_services() {
|
||||
let Cli {
|
||||
|
|
@ -5312,6 +5644,14 @@ mod tests {
|
|||
status["next_blocked_action"]["action"],
|
||||
"node_work_requires_online_attached_node"
|
||||
);
|
||||
assert_eq!(
|
||||
status["next_blocked_action"]["machine_error"]["category"],
|
||||
"capability"
|
||||
);
|
||||
assert_eq!(
|
||||
status["next_blocked_action"]["machine_error"]["stable_exit_code"],
|
||||
24
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -5449,6 +5789,8 @@ mod tests {
|
|||
assert!(rendered_tasks.contains("placement task-a: node-a"));
|
||||
assert!(rendered_tasks.contains("source snapshot already local"));
|
||||
assert_eq!(tasks["tasks"][1]["failure_reason"], "boom");
|
||||
assert_eq!(tasks["tasks"][1]["machine_error"]["category"], "program");
|
||||
assert_eq!(tasks["tasks"][1]["machine_error"]["stable_exit_code"], 27);
|
||||
assert_eq!(logs["log_entries"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(logs["log_entries"][0]["task"], "task-a");
|
||||
assert_eq!(logs["log_entries"][0]["stdout_tail"], "ok");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue