Update public CLI log redaction
This commit is contained in:
parent
7a80d87376
commit
eb8dc3bf12
3 changed files with 173 additions and 5 deletions
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"kind": "disasmer-filtered-public-tree",
|
"kind": "disasmer-filtered-public-tree",
|
||||||
"source_commit": "611a91087c5a78ed23617674f2664f7cf07af020",
|
"source_commit": "f4707862d419b08f13d072a31f8f5e3cb9ddda66",
|
||||||
"release_name": "dryrun-611a91087c5a",
|
"release_name": "dryrun-f4707862d419",
|
||||||
"filtered_out": [
|
"filtered_out": [
|
||||||
"private/**",
|
"private/**",
|
||||||
"experiments/**",
|
"experiments/**",
|
||||||
|
|
|
||||||
|
|
@ -3583,7 +3583,7 @@ fn task_failure_reason(event: &Value) -> Value {
|
||||||
Some("failed") => {
|
Some("failed") => {
|
||||||
if let Some(stderr) = event.get("stderr_tail").and_then(Value::as_str) {
|
if let Some(stderr) = event.get("stderr_tail").and_then(Value::as_str) {
|
||||||
if !stderr.is_empty() {
|
if !stderr.is_empty() {
|
||||||
return json!(stderr);
|
return json!(redact_secret_like_text(stderr).0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(status_code) = event.get("status_code").and_then(Value::as_i64) {
|
if let Some(status_code) = event.get("status_code").and_then(Value::as_i64) {
|
||||||
|
|
@ -3713,23 +3713,125 @@ fn log_entries(task_events: Option<&Value>, task_filter: Option<&str>) -> Value
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.map(|event| {
|
.map(|event| {
|
||||||
|
let stdout_tail = event
|
||||||
|
.get("stdout_tail")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("");
|
||||||
|
let stderr_tail = event
|
||||||
|
.get("stderr_tail")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("");
|
||||||
|
let (stdout_tail, stdout_tail_redacted) = redact_secret_like_text(stdout_tail);
|
||||||
|
let (stderr_tail, stderr_tail_redacted) = redact_secret_like_text(stderr_tail);
|
||||||
json!({
|
json!({
|
||||||
"process": event_string(event, "process"),
|
"process": event_string(event, "process"),
|
||||||
"task": event_string(event, "task"),
|
"task": event_string(event, "task"),
|
||||||
"node": event_string(event, "node"),
|
"node": event_string(event, "node"),
|
||||||
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
|
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
|
||||||
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
|
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
|
||||||
"stdout_tail": event.get("stdout_tail").cloned().unwrap_or_else(|| json!("")),
|
"stdout_tail": stdout_tail,
|
||||||
"stderr_tail": event.get("stderr_tail").cloned().unwrap_or_else(|| json!("")),
|
"stderr_tail": stderr_tail,
|
||||||
"stdout_truncated": event.get("stdout_truncated").and_then(Value::as_bool).unwrap_or(false),
|
"stdout_truncated": event.get("stdout_truncated").and_then(Value::as_bool).unwrap_or(false),
|
||||||
"stderr_truncated": event.get("stderr_truncated").and_then(Value::as_bool).unwrap_or(false),
|
"stderr_truncated": event.get("stderr_truncated").and_then(Value::as_bool).unwrap_or(false),
|
||||||
"capped": true,
|
"capped": true,
|
||||||
|
"secret_like_values_redacted": stdout_tail_redacted || stderr_tail_redacted,
|
||||||
|
"redacted_fields": redacted_log_fields(stdout_tail_redacted, stderr_tail_redacted),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn redacted_log_fields(
|
||||||
|
stdout_tail_redacted: bool,
|
||||||
|
stderr_tail_redacted: bool,
|
||||||
|
) -> Vec<&'static str> {
|
||||||
|
let mut fields = Vec::new();
|
||||||
|
if stdout_tail_redacted {
|
||||||
|
fields.push("stdout_tail");
|
||||||
|
}
|
||||||
|
if stderr_tail_redacted {
|
||||||
|
fields.push("stderr_tail");
|
||||||
|
}
|
||||||
|
fields
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redact_secret_like_text(text: &str) -> (String, bool) {
|
||||||
|
let markers = [
|
||||||
|
"access_token=",
|
||||||
|
"access_token:",
|
||||||
|
"refresh_token=",
|
||||||
|
"refresh_token:",
|
||||||
|
"id_token=",
|
||||||
|
"id_token:",
|
||||||
|
"api_key=",
|
||||||
|
"api_key:",
|
||||||
|
"api-key=",
|
||||||
|
"api-key:",
|
||||||
|
"token=",
|
||||||
|
"token:",
|
||||||
|
"secret=",
|
||||||
|
"secret:",
|
||||||
|
"password=",
|
||||||
|
"password:",
|
||||||
|
"passwd=",
|
||||||
|
"passwd:",
|
||||||
|
"bearer ",
|
||||||
|
];
|
||||||
|
let mut output = text.to_owned();
|
||||||
|
let mut redacted = false;
|
||||||
|
for marker in markers {
|
||||||
|
let (updated, changed) = redact_marker_values(output, marker);
|
||||||
|
output = updated;
|
||||||
|
redacted |= changed;
|
||||||
|
}
|
||||||
|
(output, redacted)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redact_marker_values(mut text: String, marker: &str) -> (String, bool) {
|
||||||
|
let mut changed = false;
|
||||||
|
let mut search_start = 0;
|
||||||
|
loop {
|
||||||
|
let lower = text.to_ascii_lowercase();
|
||||||
|
let Some(relative) = lower[search_start..].find(marker) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let value_start = search_start + relative + marker.len();
|
||||||
|
let value_end = text[value_start..]
|
||||||
|
.char_indices()
|
||||||
|
.find_map(|(offset, character)| {
|
||||||
|
(character.is_whitespace()
|
||||||
|
|| matches!(
|
||||||
|
character,
|
||||||
|
'&' | '"' | '\'' | '`' | '<' | '>' | ',' | ';' | ')' | ']'
|
||||||
|
))
|
||||||
|
.then_some(value_start + offset)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| text.len());
|
||||||
|
if value_start == value_end {
|
||||||
|
search_start = value_end;
|
||||||
|
if search_start >= text.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if text[value_start..value_end].starts_with("[redacted") {
|
||||||
|
search_start = value_end;
|
||||||
|
if search_start >= text.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
text.replace_range(value_start..value_end, "[redacted]");
|
||||||
|
changed = true;
|
||||||
|
search_start = value_start + "[redacted]".len();
|
||||||
|
if search_start >= text.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(text, changed)
|
||||||
|
}
|
||||||
|
|
||||||
fn artifact_summaries(task_events: Option<&Value>) -> Value {
|
fn artifact_summaries(task_events: Option<&Value>) -> Value {
|
||||||
Value::Array(
|
Value::Array(
|
||||||
task_event_values(task_events)
|
task_event_values(task_events)
|
||||||
|
|
@ -7796,6 +7898,61 @@ mod tests {
|
||||||
assert_eq!(artifacts["default_durable_store_assumed"], false);
|
assert_eq!(artifacts["default_durable_store_assumed"], false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn log_and_task_reports_redact_secret_like_values() {
|
||||||
|
let events = json!({
|
||||||
|
"response": {
|
||||||
|
"type": "task_events",
|
||||||
|
"events": [{
|
||||||
|
"tenant": "tenant",
|
||||||
|
"project": "project",
|
||||||
|
"process": "vp",
|
||||||
|
"node": "node-a",
|
||||||
|
"task": "task-secret",
|
||||||
|
"terminal_state": "failed",
|
||||||
|
"status_code": 1,
|
||||||
|
"stdout_bytes": 128,
|
||||||
|
"stderr_bytes": 64,
|
||||||
|
"stdout_tail": "upload token=abc123 Authorization: Bearer bearer-secret",
|
||||||
|
"stderr_tail": "failed password=hunter2 access_token=provider-secret",
|
||||||
|
"stdout_truncated": true,
|
||||||
|
"stderr_truncated": false
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let entries = log_entries(Some(&events), Some("task-secret"));
|
||||||
|
let entry = &entries.as_array().unwrap()[0];
|
||||||
|
assert_eq!(
|
||||||
|
entry["stdout_tail"],
|
||||||
|
"upload token=[redacted] Authorization: Bearer [redacted]"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
entry["stderr_tail"],
|
||||||
|
"failed password=[redacted] access_token=[redacted]"
|
||||||
|
);
|
||||||
|
assert_eq!(entry["stdout_bytes"], 128);
|
||||||
|
assert_eq!(entry["stdout_truncated"], true);
|
||||||
|
assert_eq!(entry["secret_like_values_redacted"], true);
|
||||||
|
assert_eq!(entry["redacted_fields"][0], "stdout_tail");
|
||||||
|
assert_eq!(entry["redacted_fields"][1], "stderr_tail");
|
||||||
|
|
||||||
|
let tasks = task_summaries(Some(&events));
|
||||||
|
let task = &tasks.as_array().unwrap()[0];
|
||||||
|
assert_eq!(
|
||||||
|
task["failure_reason"],
|
||||||
|
"failed password=[redacted] access_token=[redacted]"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
task["machine_error"]["message"],
|
||||||
|
"failed password=[redacted] access_token=[redacted]"
|
||||||
|
);
|
||||||
|
assert!(!serde_json::to_string(&entries)
|
||||||
|
.unwrap()
|
||||||
|
.contains("provider-secret"));
|
||||||
|
assert!(!serde_json::to_string(&tasks).unwrap().contains("hunter2"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn artifact_download_and_export_reports_expose_safe_session_boundaries() {
|
fn artifact_download_and_export_reports_expose_safe_session_boundaries() {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,11 @@ expect(
|
||||||
"mutating commands require confirmation",
|
"mutating commands require confirmation",
|
||||||
/Mutating or dangerous commands support `--yes`[\s\S]*confirmation-required safe failure[\s\S]*do not send coordinator requests/
|
/Mutating or dangerous commands support `--yes`[\s\S]*confirmation-required safe failure[\s\S]*do not send coordinator requests/
|
||||||
);
|
);
|
||||||
|
expect(
|
||||||
|
criteria,
|
||||||
|
"log redaction criteria",
|
||||||
|
/Logs are capped, truncated honestly[\s\S]*preserve byte counts and truncation flags[\s\S]*Secret-like values are redacted[\s\S]*common token\/password\/bearer patterns/
|
||||||
|
);
|
||||||
expect(
|
expect(
|
||||||
cliFirstAcceptance,
|
cliFirstAcceptance,
|
||||||
"CLI-first acceptance report",
|
"CLI-first acceptance report",
|
||||||
|
|
@ -219,6 +224,7 @@ for (const [name, pattern] of [
|
||||||
["quota coordinator usage coverage", /fn quota_status_queries_public_coordinator_usage\(\)/],
|
["quota coordinator usage coverage", /fn quota_status_queries_public_coordinator_usage\(\)/],
|
||||||
["task event summary coverage", /fn process_task_log_and_artifact_reports_summarize_task_events\(\)/],
|
["task event summary coverage", /fn process_task_log_and_artifact_reports_summarize_task_events\(\)/],
|
||||||
["task locality failure summary coverage", /fn process_task_log_and_artifact_reports_summarize_task_events\(\)[\s\S]*source snapshot unavailable and direct connectivity unavailable[\s\S]*locality_failure/],
|
["task locality failure summary coverage", /fn process_task_log_and_artifact_reports_summarize_task_events\(\)[\s\S]*source snapshot unavailable and direct connectivity unavailable[\s\S]*locality_failure/],
|
||||||
|
["log secret redaction coverage", /fn log_and_task_reports_redact_secret_like_values\(\)/],
|
||||||
["artifact download/export report coverage", /fn artifact_download_and_export_reports_expose_safe_session_boundaries\(\)/],
|
["artifact download/export report coverage", /fn artifact_download_and_export_reports_expose_safe_session_boundaries\(\)/],
|
||||||
["process control report coverage", /fn process_restart_and_cancel_reports_expose_control_boundaries\(\)/],
|
["process control report coverage", /fn process_restart_and_cancel_reports_expose_control_boundaries\(\)/],
|
||||||
["task restart report coverage", /fn task_restart_reports_clean_boundary_requirements\(\)/],
|
["task restart report coverage", /fn task_restart_reports_clean_boundary_requirements\(\)/],
|
||||||
|
|
@ -348,6 +354,11 @@ expect(
|
||||||
"CLI renders task placement reasons",
|
"CLI renders task placement reasons",
|
||||||
/fn push_task_placement_reasons[\s\S]*placement \{task_name\}: \{node\}/
|
/fn push_task_placement_reasons[\s\S]*placement \{task_name\}: \{node\}/
|
||||||
);
|
);
|
||||||
|
expect(
|
||||||
|
cli,
|
||||||
|
"CLI redacts secret-like log values",
|
||||||
|
/fn log_entries[\s\S]*redact_secret_like_text[\s\S]*secret_like_values_redacted[\s\S]*redacted_fields[\s\S]*fn redact_secret_like_text[\s\S]*access_token=[\s\S]*password=[\s\S]*bearer /
|
||||||
|
);
|
||||||
expect(
|
expect(
|
||||||
cli,
|
cli,
|
||||||
"CLI classifies machine-readable error categories",
|
"CLI classifies machine-readable error categories",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue