Public release release-e7c2b3ac175d
Source commit: e7c2b3ac175db426791c02779841c5df1d0ffdff Public tree identity: sha256:4f0b7cd828932c06d56f2406788f89b2050ef56c1e1a4f76f55341d387d78e46
This commit is contained in:
Clusterflux release
parent
9f43c6276a
commit
cfd4f19da2
16 changed files with 1386 additions and 313 deletions
|
|
@ -2501,10 +2501,10 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() {
|
|||
process: "process".to_owned(),
|
||||
node: "node".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
stdout_bytes: 1,
|
||||
stderr_bytes: 0,
|
||||
stdout_tail: "x".to_owned(),
|
||||
stderr_tail: String::new(),
|
||||
stdout_bytes: 4,
|
||||
stderr_bytes: 1,
|
||||
stdout_tail: "outx".to_owned(),
|
||||
stderr_tail: "e".to_owned(),
|
||||
stdout_truncated: false,
|
||||
stderr_truncated: false,
|
||||
backpressured: true,
|
||||
|
|
@ -4474,6 +4474,63 @@ fn completed_main_unpolled_final_assignment_completion_retires_process() {
|
|||
.expect("unpolled terminal completion must release the one-process slot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_main_retires_from_authoritative_state_after_event_history_rotates() {
|
||||
let mut service =
|
||||
service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast);
|
||||
let tenant = TenantId::from("tenant");
|
||||
let project = ProjectId::from("project");
|
||||
let process = ProcessId::from("terminal-matrix");
|
||||
|
||||
for index in 0..=MAX_TASK_EVENTS_PER_PROCESS {
|
||||
service.record_task_completion_event(TaskCompletionEvent {
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: process.clone(),
|
||||
node: NodeId::from("worker"),
|
||||
executor: TaskExecutor::Node,
|
||||
task_definition: TaskDefinitionId::from("historical"),
|
||||
task: TaskInstanceId::new(format!("historical-{index}")),
|
||||
attempt_id: None,
|
||||
placement: None,
|
||||
terminal_state: TaskTerminalState::Completed,
|
||||
status_code: Some(0),
|
||||
stdout_bytes: 0,
|
||||
stderr_bytes: 0,
|
||||
stdout_tail: String::new(),
|
||||
stderr_tail: String::new(),
|
||||
stdout_truncated: false,
|
||||
stderr_truncated: false,
|
||||
artifact_path: None,
|
||||
artifact_digest: None,
|
||||
artifact_size_bytes: None,
|
||||
result: None,
|
||||
});
|
||||
}
|
||||
assert!(
|
||||
service.task_events.iter().all(|event| {
|
||||
event.process != process || event.executor != TaskExecutor::CoordinatorMain
|
||||
}),
|
||||
"the regression requires bounded history to have rotated the main event"
|
||||
);
|
||||
|
||||
complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed);
|
||||
|
||||
assert!(service
|
||||
.coordinator
|
||||
.active_process(&tenant, &project, &process)
|
||||
.is_none());
|
||||
let summary = service
|
||||
.process_summaries
|
||||
.get(&process_control_key(&tenant, &project, &process))
|
||||
.expect("the terminal process summary must remain authoritative");
|
||||
assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed));
|
||||
assert_eq!(
|
||||
summary.main_terminal_state,
|
||||
Some(TaskTerminalState::Completed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_main_await_operator_blocks_retirement_until_each_resolution() {
|
||||
for resolution in [
|
||||
|
|
@ -4634,6 +4691,14 @@ fn completed_main_failed_child_restarted_successfully_retires_with_successful_cu
|
|||
assert!(attempts
|
||||
.iter()
|
||||
.any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed));
|
||||
assert_eq!(
|
||||
service
|
||||
.process_summaries
|
||||
.get(&process_control_key(&tenant, &project, &process))
|
||||
.and_then(|summary| summary.final_result.clone()),
|
||||
Some(ProcessFinalResult::Completed),
|
||||
"a successful current retry must override the superseded failed attempt"
|
||||
);
|
||||
assert_eq!(
|
||||
service
|
||||
.task_join_result(tenant, project, process, task)
|
||||
|
|
@ -8182,6 +8247,54 @@ fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_
|
|||
assert!(oversized.to_string().contains("100"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_summary_eviction_releases_live_log_accounting_state() {
|
||||
let mut service = CoordinatorService::new(7);
|
||||
let tenant = TenantId::from("tenant-summary-bound");
|
||||
let project = ProjectId::from("project-summary-bound");
|
||||
let task = TaskInstanceId::from("task");
|
||||
|
||||
for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT {
|
||||
let process = ProcessId::new(format!("process-{index:03}"));
|
||||
service.record_process_started(&tenant, &project, &process, index as u64);
|
||||
service.record_process_terminal(
|
||||
&tenant,
|
||||
&project,
|
||||
&process,
|
||||
ProcessFinalResult::Completed,
|
||||
index as u64 + 1,
|
||||
);
|
||||
let key = (
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
process,
|
||||
task.clone(),
|
||||
"stdout".to_owned(),
|
||||
);
|
||||
service.recent_log_accounted_bytes.insert(key.clone(), 10);
|
||||
service.recent_log_truncated_streams.insert(key);
|
||||
}
|
||||
|
||||
let evicted = ProcessId::from("process-000");
|
||||
service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000);
|
||||
|
||||
assert!(!service.process_summaries.contains_key(&(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
evicted.clone()
|
||||
)));
|
||||
assert!(!service.recent_log_accounted_bytes.keys().any(
|
||||
|(entry_tenant, entry_project, process, _, _)| {
|
||||
entry_tenant == &tenant && entry_project == &project && process == &evicted
|
||||
}
|
||||
));
|
||||
assert!(!service.recent_log_truncated_streams.iter().any(
|
||||
|(entry_tenant, entry_project, process, _, _)| {
|
||||
entry_tenant == &tenant && entry_project == &project && process == &evicted
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_node_summaries_are_scoped_paginated_and_hard_bounded() {
|
||||
let mut service = CoordinatorService::new(7);
|
||||
|
|
@ -8560,6 +8673,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
|
|||
else {
|
||||
panic!("expected first live log sequence");
|
||||
};
|
||||
assert_eq!(
|
||||
service.quota.used_log_bytes(
|
||||
&TenantId::from("tenant-a"),
|
||||
&ProjectId::from("project-a"),
|
||||
100,
|
||||
),
|
||||
5,
|
||||
"live bytes must be charged when accepted"
|
||||
);
|
||||
let retry = service
|
||||
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
|
||||
tenant: "tenant-a".to_owned(),
|
||||
|
|
@ -8578,6 +8700,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
|
|||
retry,
|
||||
CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. }
|
||||
));
|
||||
assert_eq!(
|
||||
service.quota.used_log_bytes(
|
||||
&TenantId::from("tenant-a"),
|
||||
&ProjectId::from("project-a"),
|
||||
100,
|
||||
),
|
||||
5,
|
||||
"a retried chunk must not be charged twice"
|
||||
);
|
||||
service
|
||||
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
|
||||
tenant: "tenant-a".to_owned(),
|
||||
|
|
@ -8592,6 +8723,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
|
|||
truncated: false,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
service.quota.used_log_bytes(
|
||||
&TenantId::from("tenant-a"),
|
||||
&ProjectId::from("project-a"),
|
||||
100,
|
||||
),
|
||||
10,
|
||||
"a gap and the delivered bytes must both count toward source-byte usage"
|
||||
);
|
||||
|
||||
let CoordinatorResponse::RecentLogs {
|
||||
entries,
|
||||
|
|
@ -8633,6 +8773,113 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
|
|||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].text, "ok");
|
||||
|
||||
service
|
||||
.handle_report_task_log(
|
||||
"tenant-a".to_owned(),
|
||||
"project-a".to_owned(),
|
||||
"process-shared".to_owned(),
|
||||
"node-a".to_owned(),
|
||||
"task-one".to_owned(),
|
||||
12,
|
||||
0,
|
||||
"hello???okZZ".to_owned(),
|
||||
String::new(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
service.quota.used_log_bytes(
|
||||
&TenantId::from("tenant-a"),
|
||||
&ProjectId::from("project-a"),
|
||||
100,
|
||||
),
|
||||
12,
|
||||
"the final summary must charge only source bytes not already charged live"
|
||||
);
|
||||
assert_eq!(
|
||||
service
|
||||
.recent_logs
|
||||
.get(&(TenantId::from("tenant-a"), ProjectId::from("project-a")))
|
||||
.unwrap()
|
||||
.back()
|
||||
.unwrap()
|
||||
.text,
|
||||
"ZZ",
|
||||
"final-tail reconciliation must append only the nonduplicating suffix"
|
||||
);
|
||||
service
|
||||
.handle_report_task_log(
|
||||
"tenant-a".to_owned(),
|
||||
"project-a".to_owned(),
|
||||
"process-shared".to_owned(),
|
||||
"node-a".to_owned(),
|
||||
"task-one".to_owned(),
|
||||
12,
|
||||
0,
|
||||
"hello???okZZ".to_owned(),
|
||||
String::new(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
service.quota.used_log_bytes(
|
||||
&TenantId::from("tenant-a"),
|
||||
&ProjectId::from("project-a"),
|
||||
100,
|
||||
),
|
||||
12,
|
||||
"replayed final accounting must be idempotent"
|
||||
);
|
||||
|
||||
let marker = service
|
||||
.handle_report_task_log_chunk(
|
||||
"tenant-a".to_owned(),
|
||||
"project-a".to_owned(),
|
||||
"process-shared".to_owned(),
|
||||
"node-a".to_owned(),
|
||||
"task-one".to_owned(),
|
||||
TaskLogStream::Stdout,
|
||||
12,
|
||||
0,
|
||||
"[log output truncated at node capture limit]".to_owned(),
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
marker,
|
||||
CoordinatorResponse::TaskLogChunkRecorded {
|
||||
sequence: Some(_),
|
||||
next_offset: 12,
|
||||
..
|
||||
}
|
||||
));
|
||||
let repeated_marker = service
|
||||
.handle_report_task_log_chunk(
|
||||
"tenant-a".to_owned(),
|
||||
"project-a".to_owned(),
|
||||
"process-shared".to_owned(),
|
||||
"node-a".to_owned(),
|
||||
"task-one".to_owned(),
|
||||
TaskLogStream::Stdout,
|
||||
12,
|
||||
0,
|
||||
"[log output truncated at node capture limit]".to_owned(),
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
repeated_marker,
|
||||
CoordinatorResponse::TaskLogChunkRecorded {
|
||||
sequence: None,
|
||||
next_offset: 12,
|
||||
..
|
||||
}
|
||||
));
|
||||
|
||||
let cross_tenant = service
|
||||
.handle_request(CoordinatorRequest::Authenticated {
|
||||
session_secret: "session-b".to_owned(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue