use std::collections::{BTreeMap, BTreeSet}; use std::time::Instant; use clusterflux_core::{ Actor, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId, WasmHostDebugProbeResult, WASM_TASK_ABI_VERSION, }; use crate::{CoordinatorError, CoordinatorServiceError}; use super::keys::{process_control_key, task_control_key, task_restart_key, TaskControlKey}; use super::{ CoordinatorResponse, CoordinatorService, DebugAcknowledgementState, DebugAuditEvent, DebugParticipantAcknowledgement, TaskCancellationTarget, DEBUG_CONTROL_READ_BYTES, }; mod validation; use validation::{runtime_all_in_state, validate_debug_snapshot, validate_probe_symbols}; #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugPendingCommand { pub(super) epoch: u64, pub(super) command: String, } #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugEpochRuntime { pub(super) epoch: u64, pub(super) command: String, pub(super) expected: BTreeSet, pub(super) acknowledgements: BTreeMap, pub(super) deadline: Instant, } #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugBreakpointPlan { pub(super) actor: UserId, pub(super) probe_symbols: BTreeSet, pub(super) hit_epoch: Option, pub(super) hit_task: Option, pub(super) hit_probe_symbol: Option, } impl CoordinatorService { pub(super) fn handle_debug_attach( &mut self, tenant: String, project: String, actor_user: String, process: String, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let actor = UserId::new(actor_user); let process = ProcessId::new(process); let context = clusterflux_core::AuthContext { tenant: tenant.clone(), project: project.clone(), actor: Actor::User(actor.clone()), }; let authorization = self.coordinator.authorize_debug_attach(&context, &process); let audit_event = self.record_debug_audit_event( tenant, project, process.clone(), None, actor.clone(), "debug_attach", authorization.allowed, authorization.reason.clone(), )?; Ok(CoordinatorResponse::DebugAttach { process, actor, authorization, charged_debug_read_bytes: audit_event.charged_debug_read_bytes, used_debug_read_bytes: audit_event.used_debug_read_bytes, audit_event, }) } pub(super) fn handle_set_debug_breakpoints( &mut self, tenant: String, project: String, actor_user: String, process: String, probe_symbols: Vec, ) -> Result { let probe_symbols = validate_probe_symbols(probe_symbols)?; let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let actor = UserId::new(actor_user); let process = ProcessId::new(process); let context = clusterflux_core::AuthContext { tenant: tenant.clone(), project: project.clone(), actor: Actor::User(actor.clone()), }; let authorization = self.coordinator.authorize_debug_attach(&context, &process); let audit_event = self.record_debug_audit_event( tenant.clone(), project.clone(), process.clone(), None, actor.clone(), "set_debug_breakpoints", authorization.allowed, authorization.reason.clone(), )?; if !authorization.allowed { return Err(CoordinatorError::Unauthorized(format!( "set_debug_breakpoints denied: {}", authorization.reason )) .into()); } self.debug_breakpoints.insert( process_control_key(&tenant, &project, &process), DebugBreakpointPlan { actor: actor.clone(), probe_symbols: probe_symbols.iter().cloned().collect(), hit_epoch: None, hit_task: None, hit_probe_symbol: None, }, ); Ok(CoordinatorResponse::DebugBreakpoints { process, actor, probe_symbols, hit_epoch: None, hit_task: None, hit_probe_symbol: None, charged_debug_read_bytes: audit_event.charged_debug_read_bytes, used_debug_read_bytes: audit_event.used_debug_read_bytes, audit_event, }) } pub(super) fn handle_inspect_debug_breakpoints( &mut self, tenant: String, project: String, actor_user: String, process: String, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let actor = UserId::new(actor_user); let process = ProcessId::new(process); let context = clusterflux_core::AuthContext { tenant: tenant.clone(), project: project.clone(), actor: Actor::User(actor.clone()), }; let authorization = self.coordinator.authorize_debug_attach(&context, &process); let plan = self .debug_breakpoints .get(&process_control_key(&tenant, &project, &process)) .cloned() .ok_or_else(|| { CoordinatorServiceError::Protocol(format!( "no debug breakpoints are configured for {process}" )) })?; if plan.actor != actor { return Err(CoordinatorError::Unauthorized( "debug breakpoint inspection belongs to another authenticated user".to_owned(), ) .into()); } let audit_event = self.record_debug_audit_event( tenant, project, process.clone(), plan.hit_task.clone(), actor.clone(), "inspect_debug_breakpoints", authorization.allowed, authorization.reason.clone(), )?; if !authorization.allowed { return Err(CoordinatorError::Unauthorized(format!( "inspect_debug_breakpoints denied: {}", authorization.reason )) .into()); } Ok(CoordinatorResponse::DebugBreakpoints { process, actor, probe_symbols: plan.probe_symbols.into_iter().collect(), hit_epoch: plan.hit_epoch, hit_task: plan.hit_task, hit_probe_symbol: plan.hit_probe_symbol, charged_debug_read_bytes: audit_event.charged_debug_read_bytes, used_debug_read_bytes: audit_event.used_debug_read_bytes, audit_event, }) } pub(super) fn handle_create_debug_epoch( &mut self, tenant: String, project: String, actor_user: String, process: String, stopped_task: String, reason: String, ) -> Result { self.handle_debug_epoch_control( tenant, project, actor_user, process, Some(stopped_task), None, "create_debug_epoch", "freeze", true, reason, ) } pub(super) fn handle_resume_debug_epoch( &mut self, tenant: String, project: String, actor_user: String, process: String, epoch: u64, ) -> Result { self.handle_debug_epoch_control( tenant, project, actor_user, process, None, Some(epoch), "resume_debug_epoch", "resume", false, "continue requested by debugger", ) } pub(super) fn handle_poll_debug_command( &mut self, tenant: String, project: String, process: String, node: String, task: String, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let process = ProcessId::new(process); let node = NodeId::new(node); let task = TaskInstanceId::new(task); self.coordinator .authorize_node_for_process(&node, &tenant, &project, &process)?; let pending = self .debug_commands .remove(&task_control_key(&tenant, &project, &process, &node, &task)); Ok(CoordinatorResponse::DebugCommand { process, task, epoch: pending.as_ref().map(|command| command.epoch), command: pending.map(|command| command.command), }) } #[allow(clippy::too_many_arguments)] pub(super) fn handle_report_debug_state( &mut self, tenant: String, project: String, process: String, node: String, task: String, epoch: u64, state: DebugAcknowledgementState, stack_frames: Vec, local_values: Vec<(String, String)>, task_args: Vec<(String, String)>, handles: Vec<(String, String)>, command_status: Option, recent_output: Vec, message: Option, ) -> Result { validate_debug_snapshot( &stack_frames, &local_values, &task_args, &handles, command_status.as_deref(), &recent_output, message.as_deref(), )?; let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let process = ProcessId::new(process); let node = NodeId::new(node); let task = TaskInstanceId::new(task); self.coordinator .authorize_node_for_process(&node, &tenant, &project, &process)?; let participant_key = task_control_key(&tenant, &project, &process, &node, &task); let process_key = process_control_key(&tenant, &project, &process); let runtime = self .debug_epoch_runtime .get_mut(&process_key) .ok_or_else(|| { CoordinatorServiceError::Protocol(format!( "cannot acknowledge debug epoch {epoch} for {process}: no active debug epoch" )) })?; if runtime.epoch != epoch { return Err(CoordinatorServiceError::Protocol(format!( "cannot acknowledge debug epoch {epoch} for {process}: current debug epoch is {}", runtime.epoch ))); } if !runtime.expected.contains(&participant_key) { return Err(CoordinatorError::Unauthorized( "debug acknowledgement is not from an expected active task participant".to_owned(), ) .into()); } let task_definition = self .task_restart_checkpoints .get(&task_restart_key(&tenant, &project, &process, &task)) .map(|checkpoint| checkpoint.assignment.task_spec.task_definition.clone()) .ok_or_else(|| { CoordinatorError::Unauthorized( "debug acknowledgement does not name a coordinator-issued task instance" .to_owned(), ) })?; let valid_state = matches!( (runtime.command.as_str(), &state), ("freeze", DebugAcknowledgementState::Frozen) | ("resume", DebugAcknowledgementState::Running) | (_, DebugAcknowledgementState::Failed) ); if !valid_state { return Err(CoordinatorServiceError::Protocol(format!( "debug epoch {epoch} command `{}` cannot be acknowledged as {state:?}", runtime.command ))); } runtime.acknowledgements.insert( participant_key, DebugParticipantAcknowledgement { node: node.clone(), task_definition, task: task.clone(), epoch, state: state.clone(), stack_frames, local_values, task_args, handles, command_status, recent_output, message, }, ); Ok(CoordinatorResponse::DebugStateRecorded { process, node, task, epoch, state, }) } pub(super) fn handle_report_debug_probe_hit( &mut self, tenant: String, project: String, process: String, node: String, task: String, probe_symbol: String, ) -> Result { let probe_symbol = validate_probe_symbols(vec![probe_symbol])? .into_iter() .next() .expect("one validated probe symbol remains one symbol"); let tenant_id = TenantId::new(tenant.clone()); let project_id = ProjectId::new(project.clone()); let process_id = ProcessId::new(process.clone()); let node_id = NodeId::new(node.clone()); let task_id = TaskInstanceId::new(task.clone()); self.coordinator.authorize_node_for_process( &node_id, &tenant_id, &project_id, &process_id, )?; let participant_key = task_control_key(&tenant_id, &project_id, &process_id, &node_id, &task_id); if !self.active_tasks.contains(&participant_key) { return Err(CoordinatorError::Unauthorized( "debug probe hit is not from an active task participant".to_owned(), ) .into()); } let attempt_key = task_restart_key(&tenant_id, &project_id, &process_id, &task_id); if let Some(attempt) = self .task_attempts .get_mut(&attempt_key) .and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current)) { attempt.probe_symbol = Some(probe_symbol.clone()); } let process_key = process_control_key(&tenant_id, &project_id, &process_id); let Some(plan) = self.debug_breakpoints.get(&process_key).cloned() else { return Ok(CoordinatorResponse::DebugProbeHit { process: process_id, node: node_id, task: task_id, probe_symbol, breakpoint_matched: false, debug_epoch: None, }); }; if !plan.probe_symbols.contains(&probe_symbol) { return Ok(CoordinatorResponse::DebugProbeHit { process: process_id, node: node_id, task: task_id, probe_symbol, breakpoint_matched: false, debug_epoch: None, }); } let response = self.handle_debug_epoch_control( tenant, project, plan.actor.as_str().to_owned(), process, Some(task.clone()), None, "wasm_debug_probe_hit", "freeze", true, format!("executing Wasm reached probe `{probe_symbol}`"), )?; let CoordinatorResponse::DebugEpoch { epoch, .. } = response else { unreachable!("debug epoch control always returns DebugEpoch") }; if let Some(plan) = self.debug_breakpoints.get_mut(&process_key) { plan.hit_epoch = Some(epoch); plan.hit_task = Some(task_id.clone()); plan.hit_probe_symbol = Some(probe_symbol.clone()); } Ok(CoordinatorResponse::DebugProbeHit { process: process_id, node: node_id, task: task_id, probe_symbol, breakpoint_matched: true, debug_epoch: Some(epoch), }) } pub(super) fn handle_coordinator_main_debug_probe( &mut self, tenant: TenantId, project: ProjectId, process: ProcessId, task: TaskInstanceId, probe_symbol: String, ) -> Result { let probe_symbol = validate_probe_symbols(vec![probe_symbol])? .into_iter() .next() .expect("one validated probe symbol remains one symbol"); let process_key = process_control_key(&tenant, &project, &process); let main_matches = self .main_runtime .controls .get(&process_key) .is_some_and(|control| control.task_instance == task); if !main_matches { return Err(CoordinatorError::Unauthorized( "debug probe does not belong to the active coordinator main participant".to_owned(), ) .into()); } let Some(plan) = self.debug_breakpoints.get(&process_key).cloned() else { return Ok(WasmHostDebugProbeResult { abi_version: WASM_TASK_ABI_VERSION, breakpoint_matched: false, debug_epoch: None, }); }; if !plan.probe_symbols.contains(&probe_symbol) { return Ok(WasmHostDebugProbeResult { abi_version: WASM_TASK_ABI_VERSION, breakpoint_matched: false, debug_epoch: None, }); } if let Some(control) = self.main_runtime.controls.get_mut(&process_key) { control.stopped_probe_symbol = Some(probe_symbol.clone()); } let response = self.handle_debug_epoch_control( tenant.as_str().to_owned(), project.as_str().to_owned(), plan.actor.as_str().to_owned(), process.as_str().to_owned(), Some(task.as_str().to_owned()), None, "wasm_debug_probe_hit", "freeze", true, format!("coordinator main reached probe `{probe_symbol}`"), )?; let CoordinatorResponse::DebugEpoch { epoch, .. } = response else { unreachable!("debug epoch control always returns DebugEpoch") }; if let Some(plan) = self.debug_breakpoints.get_mut(&process_key) { plan.hit_epoch = Some(epoch); plan.hit_task = Some(task); plan.hit_probe_symbol = Some(probe_symbol); } Ok(WasmHostDebugProbeResult { abi_version: WASM_TASK_ABI_VERSION, breakpoint_matched: true, debug_epoch: Some(epoch), }) } pub(super) fn handle_inspect_debug_epoch( &mut self, tenant: String, project: String, actor_user: String, process: String, epoch: u64, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let actor = UserId::new(actor_user); let process = ProcessId::new(process); let context = clusterflux_core::AuthContext { tenant: tenant.clone(), project: project.clone(), actor: Actor::User(actor.clone()), }; let authorization = self.coordinator.authorize_debug_attach(&context, &process); let process_key = process_control_key(&tenant, &project, &process); self.sync_coordinator_main_debug_acknowledgement(&process_key, epoch); let runtime = self .debug_epoch_runtime .get(&process_key) .filter(|runtime| runtime.epoch == epoch) .cloned() .ok_or_else(|| { CoordinatorServiceError::Protocol(format!( "debug epoch {epoch} is not active for {process}" )) })?; let audit_event = self.record_debug_audit_event( tenant, project, process.clone(), None, actor.clone(), "inspect_debug_epoch", authorization.allowed, authorization.reason.clone(), )?; if !authorization.allowed { return Err(CoordinatorError::Unauthorized(format!( "inspect_debug_epoch denied: {}", authorization.reason )) .into()); } let acknowledgements = runtime .acknowledgements .values() .cloned() .collect::>(); let all_acknowledged = !runtime.expected.is_empty() && runtime .expected .iter() .all(|key| runtime.acknowledgements.contains_key(key)); let fully_frozen = runtime.command == "freeze" && all_acknowledged && acknowledgements .iter() .all(|ack| ack.state == DebugAcknowledgementState::Frozen); let frozen_count = acknowledgements .iter() .filter(|ack| ack.state == DebugAcknowledgementState::Frozen) .count(); let freeze_deadline_elapsed = runtime.command == "freeze" && Instant::now() >= runtime.deadline; let partially_frozen = runtime.command == "freeze" && freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen; let fully_resumed = runtime.command == "resume" && all_acknowledged && acknowledgements .iter() .all(|ack| ack.state == DebugAcknowledgementState::Running); let missing = runtime .expected .iter() .filter(|key| !runtime.acknowledgements.contains_key(*key)) .cloned() .collect::>(); let failed = acknowledgements .iter() .any(|ack| ack.state == DebugAcknowledgementState::Failed) || (freeze_deadline_elapsed && !missing.is_empty()); let mut failure_messages = acknowledgements .iter() .filter(|ack| ack.state == DebugAcknowledgementState::Failed) .map(|ack| { ack.message .clone() .unwrap_or_else(|| format!("task {} failed debug control", ack.task)) }) .collect::>(); if freeze_deadline_elapsed { failure_messages.extend(missing.iter().map(|(_, _, _, node, task)| { format!( "task {task} on node {node} did not acknowledge frozen state within {} ms", self.debug_freeze_timeout.as_millis() ) })); } let expected_tasks = runtime .expected .into_iter() .map(|(_, _, process, node, task)| TaskCancellationTarget { process, node, task, }) .collect(); Ok(CoordinatorResponse::DebugEpochStatus { process, actor, epoch, command: runtime.command, expected_tasks, acknowledgements, fully_frozen, partially_frozen, fully_resumed, failed, failure_messages, charged_debug_read_bytes: audit_event.charged_debug_read_bytes, used_debug_read_bytes: audit_event.used_debug_read_bytes, audit_event, }) } pub(super) fn clear_debug_state_for_process( &mut self, tenant: &TenantId, project: &ProjectId, process: &ProcessId, ) { self.debug_epochs .remove(&process_control_key(tenant, project, process)); self.debug_epoch_runtime .remove(&process_control_key(tenant, project, process)); self.debug_breakpoints .remove(&process_control_key(tenant, project, process)); self.debug_commands .retain(|(task_tenant, task_project, task_process, _, _), _| { task_tenant != tenant || task_project != project || task_process != process }); } fn handle_debug_epoch_control( &mut self, tenant: String, project: String, actor_user: String, process: String, stopped_task: Option, expected_epoch: Option, operation: &'static str, command: &'static str, all_stop_requested: bool, reason: impl Into, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let actor = UserId::new(actor_user); let process = ProcessId::new(process); let context = clusterflux_core::AuthContext { tenant: tenant.clone(), project: project.clone(), actor: Actor::User(actor.clone()), }; let authorization = self.coordinator.authorize_debug_attach(&context, &process); let reason = reason.into(); let audit_event = self.record_debug_audit_event( tenant.clone(), project.clone(), process.clone(), stopped_task.as_ref().map(TaskInstanceId::new), actor.clone(), operation, authorization.allowed, if authorization.allowed { reason } else { authorization.reason.clone() }, )?; if !authorization.allowed { return Err(CoordinatorError::Unauthorized(format!( "{operation} denied: {}", authorization.reason )) .into()); } let process_key = process_control_key(&tenant, &project, &process); let epoch = match expected_epoch { Some(expected) => { let current = self .debug_epochs .get(&process_key) .copied() .ok_or_else(|| { CoordinatorServiceError::Protocol(format!( "cannot resume debug epoch {expected} for {process}: no active debug epoch" )) })?; if current != expected { return Err(CoordinatorServiceError::Protocol(format!( "cannot resume debug epoch {expected} for {process}: current debug epoch is {current}" ))); } let runtime = self.debug_epoch_runtime.get(&process_key).ok_or_else(|| { CoordinatorServiceError::Protocol(format!( "cannot resume debug epoch {expected} for {process}: participant state is unavailable" )) })?; let has_frozen = runtime .acknowledgements .values() .any(|ack| ack.state == DebugAcknowledgementState::Frozen); let settled = runtime .expected .iter() .all(|key| runtime.acknowledgements.contains_key(key)) || Instant::now() >= runtime.deadline; if runtime.command != "freeze" || !has_frozen || !settled { return Err(CoordinatorServiceError::Protocol(format!( "cannot resume debug epoch {expected} for {process}: no settled frozen participant set is available" ))); } current } None => { if let Some(runtime) = self.debug_epoch_runtime.get(&process_key) { let previous_complete = runtime.command == "resume" && runtime_all_in_state(runtime, DebugAcknowledgementState::Running); if !previous_complete { return Err(CoordinatorServiceError::Protocol(format!( "cannot create a new debug epoch for {process}: epoch {} has not fully resumed", runtime.epoch ))); } } let next = self.debug_epochs.get(&process_key).copied().unwrap_or(0) + 1; self.debug_epochs.insert(process_key.clone(), next); next } }; let resumable = if command == "resume" { self.debug_epoch_runtime .get(&process_key) .map(|runtime| { runtime .acknowledgements .iter() .filter(|(_, ack)| ack.state == DebugAcknowledgementState::Frozen) .map(|(key, _)| key.clone()) .collect::>() }) .unwrap_or_default() } else { BTreeSet::new() }; let mut affected_tasks = self .enqueue_debug_command_for_active_tasks(&tenant, &project, &process, epoch, command); let mut expected = self .active_tasks .iter() .filter(|(task_tenant, task_project, task_process, _, _)| { task_tenant == &tenant && task_project == &project && task_process == &process }) .cloned() .collect::>(); if let Some(control) = self .main_runtime .controls .get(&process_key) .filter(|control| matches!(control.state.as_str(), "running" | "stopping")) { let key = task_control_key( &tenant, &project, &process, &NodeId::from("coordinator-main"), &control.task_instance, ); expected.insert(key); affected_tasks.push(TaskCancellationTarget { process: process.clone(), node: NodeId::from("coordinator-main"), task: control.task_instance.clone(), }); } if command == "resume" { expected.retain(|key| resumable.contains(key)); affected_tasks.retain(|target| { resumable .iter() .any(|(_, _, _, node, task)| node == &target.node && task == &target.task) }); self.debug_commands.retain(|key, pending| { pending.epoch != epoch || pending.command != "resume" || resumable.contains(key) }); } self.debug_epoch_runtime.insert( process_key.clone(), DebugEpochRuntime { epoch, command: command.to_owned(), expected, acknowledgements: BTreeMap::new(), deadline: Instant::now() + self.debug_freeze_timeout, }, ); if let Some(control) = self .main_runtime .controls .get(&process_key) .filter(|control| matches!(control.state.as_str(), "running" | "stopping")) { if command == "freeze" { control.debug.request_freeze(epoch); } else if resumable.contains(&task_control_key( &tenant, &project, &process, &NodeId::from("coordinator-main"), &control.task_instance, )) { control.debug.request_resume(epoch); } } self.sync_coordinator_main_debug_acknowledgement(&process_key, epoch); Ok(CoordinatorResponse::DebugEpoch { process, actor, epoch, command: command.to_owned(), affected_tasks, all_stop_requested, charged_debug_read_bytes: audit_event.charged_debug_read_bytes, used_debug_read_bytes: audit_event.used_debug_read_bytes, audit_event, }) } fn sync_coordinator_main_debug_acknowledgement( &mut self, process_key: &super::keys::ProcessControlKey, epoch: u64, ) { let Some(control) = self.main_runtime.controls.get(process_key) else { return; }; let Some(runtime) = self.debug_epoch_runtime.get_mut(process_key) else { return; }; if runtime.epoch != epoch { return; } let state = if runtime.command == "freeze" && control.debug.frozen_epoch() == Some(epoch) { Some(DebugAcknowledgementState::Frozen) } else if runtime.command == "resume" && control.debug.resume_requested(epoch) && control.debug.frozen_epoch() != Some(epoch) { Some(DebugAcknowledgementState::Running) } else { None }; let Some(state) = state else { return; }; let handles = control .handles .lock() .map(|handles| { let mut snapshot = handles .iter() .map(|(handle_id, spec)| { ( format!("task_handle_{handle_id}"), format!( "definition={} instance={} state=active", spec.task_definition, spec.task_instance ), ) }) .collect::>(); snapshot.sort_by(|left, right| left.0.cmp(&right.0)); snapshot }) .unwrap_or_else(|_| { vec![( "handle-registry-diagnostic".to_owned(), "coordinator main handle registry was unavailable".to_owned(), )] }); let node = NodeId::from("coordinator-main"); let key = task_control_key( &process_key.0, &process_key.1, &process_key.2, &node, &control.task_instance, ); if !runtime.expected.contains(&key) { return; } runtime.acknowledgements.insert( key, DebugParticipantAcknowledgement { node, task_definition: control.task_definition.clone(), task: control.task_instance.clone(), epoch, state, stack_frames: control .stopped_probe_symbol .as_deref() .and_then(|symbol| symbol.strip_prefix("clusterflux.probe.")) .map(|function| format!("{function}::wasm")) .into_iter() .collect(), local_values: Vec::new(), task_args: Vec::new(), handles, command_status: None, recent_output: vec![ "coordinator-hosted capless main acknowledged all-stop control".to_owned(), ], message: None, }, ); } fn enqueue_debug_command_for_active_tasks( &mut self, tenant: &TenantId, project: &ProjectId, process: &ProcessId, epoch: u64, command: &str, ) -> Vec { let task_keys = self .active_tasks .iter() .filter(|(task_tenant, task_project, task_process, _, _)| { task_tenant == tenant && task_project == project && task_process == process }) .cloned() .collect::>(); for key in &task_keys { self.debug_commands.insert( key.clone(), DebugPendingCommand { epoch, command: command.to_owned(), }, ); } task_keys .into_iter() .map(|(_, _, process, node, task)| TaskCancellationTarget { process, task, node, }) .collect() } pub(super) fn record_debug_audit_event( &mut self, tenant: TenantId, project: ProjectId, process: ProcessId, task: Option, actor: UserId, operation: &'static str, allowed: bool, reason: impl Into, ) -> Result { let now_epoch_seconds = self.current_epoch_seconds()?; let charged_debug_read_bytes = if allowed { self.quota.charge_debug_read( &tenant, &project, DEBUG_CONTROL_READ_BYTES, now_epoch_seconds, )?; DEBUG_CONTROL_READ_BYTES } else { 0 }; let used_debug_read_bytes = self.quota .used_debug_read_bytes(&tenant, &project, now_epoch_seconds); let event = DebugAuditEvent { tenant, project, process, task, actor, operation: operation.to_owned(), allowed, reason: reason.into(), charged_debug_read_bytes, used_debug_read_bytes, }; while self .debug_audit_events .iter() .filter(|retained| { retained.tenant == event.tenant && retained.project == event.project && retained.process == event.process }) .count() >= super::MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS { let Some(index) = self.debug_audit_events.iter().position(|retained| { retained.tenant == event.tenant && retained.project == event.project && retained.process == event.process }) else { break; }; self.debug_audit_events.remove(index); } while self.debug_audit_events.len() >= super::MAX_DEBUG_AUDIT_EVENTS_TOTAL { self.debug_audit_events.pop_front(); } self.debug_audit_events.push_back(event.clone()); Ok(event) } }