michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-e7c2b3ac175d

Browse source 
Source commit: e7c2b3ac175db426791c02779841c5df1d0ffdff

Public tree identity: sha256:4f0b7cd828932c06d56f2406788f89b2050ef56c1e1a4f76f55341d387d78e46
This commit is contained in:
Clusterflux release 2026-07-29 02:53:56 +02:00
parent 9f43c6276a
commit cfd4f19da2
16 changed files with 1386 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/clusterflux-coordinator/src/service.rs
View file

@ -284,9 +284,10 @@ pub struct CoordinatorService {
process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
process_summary_order: VecDeque<ProcessControlKey>,
next_process_summary_order: u64,
task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
recent_log_offsets: BTreeMap<
recent_log_accounted_bytes: BTreeMap<
(
TenantId,
ProjectId,
@ -296,6 +297,13 @@ pub struct CoordinatorService {
),
u64,
>,
recent_log_truncated_streams: BTreeSet<(
TenantId,
ProjectId,
ProcessId,
clusterflux_core::TaskInstanceId,
String,
)>,
next_recent_log_sequence: u64,
debug_audit_events: VecDeque<DebugAuditEvent>,
debug_epochs: BTreeMap<ProcessControlKey, u64>,
@ -566,9 +574,11 @@ impl CoordinatorService {
process_summaries: BTreeMap::new(),
process_summary_order: VecDeque::new(),
next_process_summary_order: 1,
task_terminal_states: BTreeMap::new(),
recent_logs: BTreeMap::new(),
recent_log_dropped_through: BTreeMap::new(),
recent_log_offsets: BTreeMap::new(),
recent_log_accounted_bytes: BTreeMap::new(),
recent_log_truncated_streams: BTreeSet::new(),
next_recent_log_sequence: 1,
debug_audit_events: VecDeque::new(),
debug_epochs: BTreeMap::new(),

386
crates/clusterflux-coordinator/src/service/logs.rs
View file

@ -39,40 +39,41 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
validate_task_log_tail("stdout_tail", &stdout_tail)?;
validate_task_log_tail("stderr_tail", &stderr_tail)?;
let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
let unaccounted_bytes = self.unaccounted_final_log_bytes(
&tenant,
&project,
&process,
&task,
stdout_bytes,
stderr_bytes,
)?;
self.quota
.can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
.can_charge_log_bytes(&tenant, &project, unaccounted_bytes, now_epoch_seconds)?;
self.quota
.charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
let stdout_key =
recent_log_offset_key(&tenant, &project, &process, &task, &TaskLogStream::Stdout);
let stderr_key =
recent_log_offset_key(&tenant, &project, &process, &task, &TaskLogStream::Stderr);
if !stdout_tail.is_empty() && !self.recent_log_offsets.contains_key(&stdout_key) {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
TaskLogStream::Stdout,
stdout_tail.clone(),
stdout_truncated,
now_epoch_seconds,
);
}
if !stderr_tail.is_empty() && !self.recent_log_offsets.contains_key(&stderr_key) {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
TaskLogStream::Stderr,
stderr_tail.clone(),
stderr_truncated,
now_epoch_seconds,
);
}
.charge_log_bytes(&tenant, &project, unaccounted_bytes, now_epoch_seconds)?;
self.reconcile_final_log_stream(
&tenant,
&project,
&process,
&task,
TaskLogStream::Stdout,
stdout_bytes,
&stdout_tail,
stdout_truncated,
now_epoch_seconds,
);
self.reconcile_final_log_stream(
&tenant,
&project,
&process,
&task,
TaskLogStream::Stderr,
stderr_bytes,
&stderr_tail,
stderr_truncated,
now_epoch_seconds,
);
Ok(CoordinatorResponse::TaskLogRecorded {
process,
task,
@ -129,13 +130,18 @@ impl CoordinatorService {
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
let expected = self.recent_log_offsets.get(&key).copied().unwrap_or(0);
let expected = self
.recent_log_accounted_bytes
.get(&key)
.copied()
.unwrap_or(0);
let end = offset.checked_add(source_bytes).ok_or_else(|| {
CoordinatorServiceError::Protocol(
"live log chunk offset exceeds the supported range".to_owned(),
)
})?;
if end <= expected {
let state_marker = source_bytes == 0 && truncated;
if end < expected || (end == expected && !state_marker) {
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
@ -144,6 +150,11 @@ impl CoordinatorService {
});
}
let now_epoch_seconds = self.current_epoch_seconds()?;
let newly_accounted = end.saturating_sub(expected);
self.quota
.can_charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)?;
self.quota
.charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)?;
if offset > expected {
self.record_recent_log(
tenant.clone(),
@ -156,26 +167,46 @@ impl CoordinatorService {
now_epoch_seconds,
);
} else if offset < expected {
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
sequence: None,
next_offset: expected,
});
}
let sequence = (!text.is_empty()).then(|| {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream,
text,
truncated,
stream.clone(),
format!(
"[log output overlap omitted: {} new source bytes]",
end - expected
),
true,
now_epoch_seconds,
)
});
self.recent_log_offsets.insert(key, end);
);
}
let marker_is_new = if truncated {
self.recent_log_truncated_streams.insert(key.clone())
} else {
false
};
let text = if state_marker && text.is_empty() {
"[log output truncated at source]".to_owned()
} else {
text
};
let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
.then(|| {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream,
text,
truncated,
now_epoch_seconds,
)
});
if end > expected {
self.recent_log_accounted_bytes.insert(key, end);
}
Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
@ -302,20 +333,49 @@ impl CoordinatorService {
artifact_size_bytes,
result,
};
let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
let unaccounted_bytes = self.unaccounted_final_log_bytes(
&event.tenant,
&event.project,
&event.process,
&event.task,
event.stdout_bytes,
event.stderr_bytes,
)?;
self.quota.can_charge_log_bytes(
&event.tenant,
&event.project,
reported_bytes,
unaccounted_bytes,
now_epoch_seconds,
)?;
self.quota.charge_log_bytes(
&event.tenant,
&event.project,
reported_bytes,
unaccounted_bytes,
now_epoch_seconds,
)?;
self.reconcile_final_log_stream(
&event.tenant,
&event.project,
&event.process,
&event.task,
TaskLogStream::Stdout,
event.stdout_bytes,
&event.stdout_tail,
event.stdout_truncated,
now_epoch_seconds,
);
self.reconcile_final_log_stream(
&event.tenant,
&event.project,
&event.process,
&event.task,
TaskLogStream::Stderr,
event.stderr_bytes,
&event.stderr_tail,
event.stderr_truncated,
now_epoch_seconds,
);
let task_key = task_control_key(
&event.tenant,
&event.project,
@ -648,6 +708,22 @@ impl CoordinatorService {
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
match event.executor {
super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
&event.tenant,
&event.project,
&event.process,
event.task_definition.clone(),
event.task.clone(),
event.terminal_state.clone(),
),
super::TaskExecutor::Node => {
self.task_terminal_states.insert(
task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
event.terminal_state.clone(),
);
}
}
let process_scope = (
event.tenant.clone(),
event.project.clone(),
@ -756,6 +832,171 @@ impl CoordinatorService {
sequence
}
#[allow(clippy::too_many_arguments)]
fn unaccounted_final_log_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stdout_bytes: u64,
stderr_bytes: u64,
) -> Result<u64, CoordinatorServiceError> {
let stdout_accounted = self
.recent_log_accounted_bytes
.get(&recent_log_offset_key(
tenant,
project,
process,
task,
&TaskLogStream::Stdout,
))
.copied()
.unwrap_or(0);
let stderr_accounted = self
.recent_log_accounted_bytes
.get(&recent_log_offset_key(
tenant,
project,
process,
task,
&TaskLogStream::Stderr,
))
.copied()
.unwrap_or(0);
let stdout_remaining = stdout_bytes.checked_sub(stdout_accounted).ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"final stdout byte count {stdout_bytes} is below the {stdout_accounted} live bytes already accounted"
))
})?;
let stderr_remaining = stderr_bytes.checked_sub(stderr_accounted).ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"final stderr byte count {stderr_bytes} is below the {stderr_accounted} live bytes already accounted"
))
})?;
checked_reported_log_bytes(stdout_remaining, stderr_remaining)
}
#[allow(clippy::too_many_arguments)]
fn reconcile_final_log_stream(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stream: TaskLogStream,
total_source_bytes: u64,
final_tail: &str,
source_truncated: bool,
now_epoch_seconds: u64,
) {
let key = recent_log_offset_key(tenant, project, process, task, &stream);
let accounted = self
.recent_log_accounted_bytes
.get(&key)
.copied()
.unwrap_or(0);
let mut visible_truncation = false;
if total_source_bytes > accounted {
let missing = total_source_bytes - accounted;
if final_tail.is_empty() {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!("[log output unavailable: {missing} source bytes]"),
true,
now_epoch_seconds,
);
visible_truncation = true;
} else if (final_tail.len() as u64) <= total_source_bytes {
let tail_source_start = total_source_bytes - final_tail.len() as u64;
if accounted < tail_source_start {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!(
"[log output lost before final tail: {} source bytes]",
tail_source_start - accounted
),
true,
now_epoch_seconds,
);
visible_truncation = true;
}
let source_start = accounted.max(tail_source_start) - tail_source_start;
let mut byte_start = usize::try_from(source_start)
.unwrap_or(final_tail.len())
.min(final_tail.len());
while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
byte_start += 1;
}
let suffix = &final_tail[byte_start..];
if !suffix.is_empty() {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
suffix.to_owned(),
source_truncated || visible_truncation,
now_epoch_seconds,
);
visible_truncation |= source_truncated;
}
} else if accounted == 0 {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
final_tail.to_owned(),
source_truncated,
now_epoch_seconds,
);
visible_truncation |= source_truncated;
} else {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!(
"[{missing} additional source bytes could not be merged without duplicating redacted output]"
),
true,
now_epoch_seconds,
);
visible_truncation = true;
}
self.recent_log_accounted_bytes
.insert(key.clone(), total_source_bytes);
}
let marker_is_new = (source_truncated || visible_truncation)
&& self.recent_log_truncated_streams.insert(key);
if marker_is_new && !visible_truncation {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream,
"[log output truncated at source]".to_owned(),
true,
now_epoch_seconds,
);
}
}
pub(super) fn clear_recent_log_offsets_for_task(
&mut self,
tenant: &TenantId,
@ -763,7 +1004,7 @@ impl CoordinatorService {
process: &ProcessId,
task: &TaskInstanceId,
) {
self.recent_log_offsets.retain(
self.recent_log_accounted_bytes.retain(
|(entry_tenant, entry_project, entry_process, entry_task, _), _| {
entry_tenant != tenant
|| entry_project != project
@ -771,6 +1012,14 @@ impl CoordinatorService {
|| entry_task != task
},
);
self.recent_log_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, entry_task, _)| {
entry_tenant != tenant
|| entry_project != project
|| entry_process != process
|| entry_task != task
},
);
}
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
@ -856,13 +1105,11 @@ impl CoordinatorService {
return Ok(false);
}
let main_completed = self.task_events.iter().rev().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& matches!(event.executor, super::TaskExecutor::CoordinatorMain)
&& matches!(event.terminal_state, TaskTerminalState::Completed)
});
let main_completed = self
.process_summaries
.get(&process_key)
.and_then(|summary| summary.main_terminal_state.as_ref())
.is_some_and(|state| matches!(state, TaskTerminalState::Completed));
let cancellation_completed = self.process_cancellations.contains(&process_key);
if !main_completed && !cancellation_completed {
return Ok(false);
@ -879,13 +1126,24 @@ impl CoordinatorService {
}
let final_result = if cancellation_completed {
super::ProcessFinalResult::Cancelled
} else if self.task_events.iter().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& event.terminal_state == TaskTerminalState::Failed
}) {
} else if self.task_terminal_states.iter().any(
|((task_tenant, task_project, task_process, _), terminal_state)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& terminal_state == &TaskTerminalState::Failed
},
) {
super::ProcessFinalResult::Failed
} else if self.task_terminal_states.iter().any(
|((task_tenant, task_project, task_process, _), terminal_state)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& terminal_state == &TaskTerminalState::Cancelled
},
) {
super::ProcessFinalResult::Cancelled
} else {
super::ProcessFinalResult::Completed
};

2
crates/clusterflux-coordinator/src/service/process_launch.rs
View file

@ -671,7 +671,9 @@ impl CoordinatorService {
));
};
self.task_attempts.remove(&removable);
self.task_terminal_states.remove(&removable);
}
self.task_terminal_states.remove(&key);
let attempts = self.task_attempts.entry(key).or_default();
for attempt in attempts.iter_mut() {
attempt.current = false;

26
crates/clusterflux-coordinator/src/service/processes.rs
View file

@ -382,6 +382,10 @@ impl CoordinatorService {
|| attempt_project != &project
|| attempt_process != &process
});
self.task_terminal_states
.retain(|(task_tenant, task_project, task_process, _), _| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.restart_launches
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
attempt_tenant != &tenant
@ -671,10 +675,18 @@ impl CoordinatorService {
.map(|active| {
let process_key = process_control_key(&active.tenant, &active.project, &active.id);
let main = self.main_runtime.controls.get(&process_key);
let stored = self.process_summaries.get(&process_key);
let stored_main_state = stored
.and_then(|summary| summary.main_terminal_state.as_ref())
.map(|state| match state {
super::TaskTerminalState::Completed => "completed",
super::TaskTerminalState::Failed => "failed",
super::TaskTerminalState::Cancelled => "cancelled",
});
let state = if self.process_cancellations.contains(&process_key) {
"cancelling"
} else {
main.map_or("running", |main| main.state.as_str())
"running"
};
let main_wait_state = main.and_then(|main| {
if main.state != "running" {
@ -699,9 +711,15 @@ impl CoordinatorService {
VirtualProcessStatus {
process: active.id,
state: state.to_owned(),
main_task_definition: main.map(|main| main.task_definition.clone()),
main_task_instance: main.map(|main| main.task_instance.clone()),
main_state: main.map(|main| main.state.clone()),
main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
|| stored.and_then(|summary| summary.main_task_definition.clone()),
),
main_task_instance: main
.map(|main| main.task_instance.clone())
.or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
main_state: main
.map(|main| main.state.clone())
.or_else(|| stored_main_state.map(str::to_owned)),
main_wait_state,
main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
connected_nodes: active.connected_nodes.into_iter().collect(),

80
crates/clusterflux-coordinator/src/service/summaries.rs
View file

@ -2,7 +2,8 @@ use std::collections::BTreeSet;
use std::time::Instant;
use clusterflux_core::{
ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TenantId, UserId,
ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
TenantId, UserId,
};
use super::keys::{process_control_key, ProcessControlKey};
@ -19,6 +20,9 @@ pub(super) struct StoredProcessSummary {
pub(super) ended_at_epoch_seconds: Option<u64>,
pub(super) final_result: Option<ProcessFinalResult>,
pub(super) connected_nodes: Vec<NodeId>,
pub(super) main_task_definition: Option<TaskDefinitionId>,
pub(super) main_task_instance: Option<TaskInstanceId>,
pub(super) main_terminal_state: Option<super::TaskTerminalState>,
pub(super) order: u64,
}
@ -84,10 +88,16 @@ impl CoordinatorService {
}
}
self.recent_log_dropped_through.remove(&key);
self.recent_log_offsets
.retain(|(entry_tenant, entry_project, entry_process, _, _), _| {
self.recent_log_accounted_bytes.retain(
|(entry_tenant, entry_project, entry_process, _, _), _| {
entry_tenant != tenant || entry_project != project || entry_process != process
});
},
);
self.recent_log_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, _, _)| {
entry_tenant != tenant || entry_project != project || entry_process != process
},
);
self.process_summary_order
.retain(|retained| retained != &key);
self.evict_process_summaries_for_project(tenant, project);
@ -101,6 +111,9 @@ impl CoordinatorService {
ended_at_epoch_seconds: None,
final_result: None,
connected_nodes: Vec::new(),
main_task_definition: None,
main_task_instance: None,
main_terminal_state: None,
order,
},
);
@ -133,6 +146,9 @@ impl CoordinatorService {
ended_at_epoch_seconds: None,
final_result: None,
connected_nodes: Vec::new(),
main_task_definition: None,
main_task_instance: None,
main_terminal_state: None,
order,
}
});
@ -141,6 +157,31 @@ impl CoordinatorService {
entry.connected_nodes = connected_nodes;
}
pub(super) fn record_main_terminal_state(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task_definition: TaskDefinitionId,
task_instance: TaskInstanceId,
terminal_state: super::TaskTerminalState,
) {
let key = process_control_key(tenant, project, process);
if !self.process_summaries.contains_key(&key) {
self.record_process_started(
tenant,
project,
process,
self.liveness_now_epoch_seconds(),
);
}
if let Some(summary) = self.process_summaries.get_mut(&key) {
summary.main_task_definition = Some(task_definition);
summary.main_task_instance = Some(task_instance);
summary.main_terminal_state = Some(terminal_state);
}
}
pub(super) fn handle_list_process_summaries(
&mut self,
tenant: String,
@ -449,10 +490,7 @@ impl CoordinatorService {
let Some(candidate) = candidate.cloned() else {
break;
};
self.process_summaries.remove(&candidate);
self.recent_log_dropped_through.remove(&candidate);
self.process_summary_order
.retain(|retained| retained != &candidate);
self.remove_process_summary_state(&candidate);
}
}
@ -466,12 +504,30 @@ impl CoordinatorService {
let Some(candidate) = candidate.cloned() else {
break;
};
self.process_summaries.remove(&candidate);
self.recent_log_dropped_through.remove(&candidate);
self.process_summary_order
.retain(|retained| retained != &candidate);
self.remove_process_summary_state(&candidate);
}
}
fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
self.process_summaries.remove(key);
self.recent_log_dropped_through.remove(key);
if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
logs.retain(|entry| entry.process != key.2);
if logs.is_empty() {
self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
}
}
self.recent_log_accounted_bytes
.retain(|(tenant, project, process, _, _), _| {
tenant != &key.0 || project != &key.1 || process != &key.2
});
self.recent_log_truncated_streams
.retain(|(tenant, project, process, _, _)| {
tenant != &key.0 || project != &key.1 || process != &key.2
});
self.process_summary_order
.retain(|retained| retained != key);
}
}
fn parse_order_cursor(

255
crates/clusterflux-coordinator/src/service/tests.rs
View file

@ -2501,10 +2501,10 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() {
process: "process".to_owned(),
node: "node".to_owned(),
task: "task".to_owned(),
stdout_bytes: 1,
stderr_bytes: 0,
stdout_tail: "x".to_owned(),
stderr_tail: String::new(),
stdout_bytes: 4,
stderr_bytes: 1,
stdout_tail: "outx".to_owned(),
stderr_tail: "e".to_owned(),
stdout_truncated: false,
stderr_truncated: false,
backpressured: true,
@ -4474,6 +4474,63 @@ fn completed_main_unpolled_final_assignment_completion_retires_process() {
.expect("unpolled terminal completion must release the one-process slot");
}
#[test]
fn completed_main_retires_from_authoritative_state_after_event_history_rotates() {
let mut service =
service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("terminal-matrix");
for index in 0..=MAX_TASK_EVENTS_PER_PROCESS {
service.record_task_completion_event(TaskCompletionEvent {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
node: NodeId::from("worker"),
executor: TaskExecutor::Node,
task_definition: TaskDefinitionId::from("historical"),
task: TaskInstanceId::new(format!("historical-{index}")),
attempt_id: None,
placement: None,
terminal_state: TaskTerminalState::Completed,
status_code: Some(0),
stdout_bytes: 0,
stderr_bytes: 0,
stdout_tail: String::new(),
stderr_tail: String::new(),
stdout_truncated: false,
stderr_truncated: false,
artifact_path: None,
artifact_digest: None,
artifact_size_bytes: None,
result: None,
});
}
assert!(
service.task_events.iter().all(|event| {
event.process != process || event.executor != TaskExecutor::CoordinatorMain
}),
"the regression requires bounded history to have rotated the main event"
);
complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed);
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
let summary = service
.process_summaries
.get(&process_control_key(&tenant, &project, &process))
.expect("the terminal process summary must remain authoritative");
assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed));
assert_eq!(
summary.main_terminal_state,
Some(TaskTerminalState::Completed)
);
}
#[test]
fn completed_main_await_operator_blocks_retirement_until_each_resolution() {
for resolution in [
@ -4634,6 +4691,14 @@ fn completed_main_failed_child_restarted_successfully_retires_with_successful_cu
assert!(attempts
.iter()
.any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed));
assert_eq!(
service
.process_summaries
.get(&process_control_key(&tenant, &project, &process))
.and_then(|summary| summary.final_result.clone()),
Some(ProcessFinalResult::Completed),
"a successful current retry must override the superseded failed attempt"
);
assert_eq!(
service
.task_join_result(tenant, project, process, task)
@ -8182,6 +8247,54 @@ fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_
assert!(oversized.to_string().contains("100"));
}
#[test]
fn process_summary_eviction_releases_live_log_accounting_state() {
let mut service = CoordinatorService::new(7);
let tenant = TenantId::from("tenant-summary-bound");
let project = ProjectId::from("project-summary-bound");
let task = TaskInstanceId::from("task");
for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT {
let process = ProcessId::new(format!("process-{index:03}"));
service.record_process_started(&tenant, &project, &process, index as u64);
service.record_process_terminal(
&tenant,
&project,
&process,
ProcessFinalResult::Completed,
index as u64 + 1,
);
let key = (
tenant.clone(),
project.clone(),
process,
task.clone(),
"stdout".to_owned(),
);
service.recent_log_accounted_bytes.insert(key.clone(), 10);
service.recent_log_truncated_streams.insert(key);
}
let evicted = ProcessId::from("process-000");
service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000);
assert!(!service.process_summaries.contains_key(&(
tenant.clone(),
project.clone(),
evicted.clone()
)));
assert!(!service.recent_log_accounted_bytes.keys().any(
|(entry_tenant, entry_project, process, _, _)| {
entry_tenant == &tenant && entry_project == &project && process == &evicted
}
));
assert!(!service.recent_log_truncated_streams.iter().any(
|(entry_tenant, entry_project, process, _, _)| {
entry_tenant == &tenant && entry_project == &project && process == &evicted
}
));
}
#[test]
fn web_node_summaries_are_scoped_paginated_and_hard_bounded() {
let mut service = CoordinatorService::new(7);
@ -8560,6 +8673,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
else {
panic!("expected first live log sequence");
};
assert_eq!(
service.quota.used_log_bytes(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
100,
),
5,
"live bytes must be charged when accepted"
);
let retry = service
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
tenant: "tenant-a".to_owned(),
@ -8578,6 +8700,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
retry,
CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. }
));
assert_eq!(
service.quota.used_log_bytes(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
100,
),
5,
"a retried chunk must not be charged twice"
);
service
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
tenant: "tenant-a".to_owned(),
@ -8592,6 +8723,15 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
truncated: false,
})
.unwrap();
assert_eq!(
service.quota.used_log_bytes(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
100,
),
10,
"a gap and the delivered bytes must both count toward source-byte usage"
);
let CoordinatorResponse::RecentLogs {
entries,
@ -8633,6 +8773,113 @@ fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].text, "ok");
service
.handle_report_task_log(
"tenant-a".to_owned(),
"project-a".to_owned(),
"process-shared".to_owned(),
"node-a".to_owned(),
"task-one".to_owned(),
12,
0,
"hello???okZZ".to_owned(),
String::new(),
false,
false,
false,
)
.unwrap();
assert_eq!(
service.quota.used_log_bytes(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
100,
),
12,
"the final summary must charge only source bytes not already charged live"
);
assert_eq!(
service
.recent_logs
.get(&(TenantId::from("tenant-a"), ProjectId::from("project-a")))
.unwrap()
.back()
.unwrap()
.text,
"ZZ",
"final-tail reconciliation must append only the nonduplicating suffix"
);
service
.handle_report_task_log(
"tenant-a".to_owned(),
"project-a".to_owned(),
"process-shared".to_owned(),
"node-a".to_owned(),
"task-one".to_owned(),
12,
0,
"hello???okZZ".to_owned(),
String::new(),
false,
false,
false,
)
.unwrap();
assert_eq!(
service.quota.used_log_bytes(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
100,
),
12,
"replayed final accounting must be idempotent"
);
let marker = service
.handle_report_task_log_chunk(
"tenant-a".to_owned(),
"project-a".to_owned(),
"process-shared".to_owned(),
"node-a".to_owned(),
"task-one".to_owned(),
TaskLogStream::Stdout,
12,
0,
"[log output truncated at node capture limit]".to_owned(),
true,
)
.unwrap();
assert!(matches!(
marker,
CoordinatorResponse::TaskLogChunkRecorded {
sequence: Some(_),
next_offset: 12,
..
}
));
let repeated_marker = service
.handle_report_task_log_chunk(
"tenant-a".to_owned(),
"project-a".to_owned(),
"process-shared".to_owned(),
"node-a".to_owned(),
"task-one".to_owned(),
TaskLogStream::Stdout,
12,
0,
"[log output truncated at node capture limit]".to_owned(),
true,
)
.unwrap();
assert!(matches!(
repeated_marker,
CoordinatorResponse::TaskLogChunkRecorded {
sequence: None,
next_offset: 12,
..
}
));
let cross_tenant = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-b".to_owned(),