use clusterflux_core::{ Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind, ProcessId, ProjectId, RateLimit, TenantId, UserId, }; use crate::CoordinatorError; use super::artifact_id_from_path; use super::keys::panel_stop_key; use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError}; impl CoordinatorService { pub(super) fn clear_operator_panel_state( &mut self, tenant: &TenantId, project: &ProjectId, process: &ProcessId, ) { let stop_key = panel_stop_key(tenant, project, process); self.panel_snapshots.remove(&stop_key); self.stopped_panels.remove(&stop_key); self.panel_event_limits .retain(|(event_tenant, event_project, event_process, _), _| { event_tenant != tenant || event_project != project || event_process != process }); } pub(super) fn handle_render_operator_panel( &mut self, tenant: String, project: String, actor_user: String, process: String, max_download_bytes: u64, stopped: bool, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let process = ProcessId::new(process); let stop_key = panel_stop_key(&tenant, &project, &process); let panel = if stopped { self.ensure_operator_panel_scope(&tenant, &project, &process)?; self.stopped_panels.insert(stop_key); let stop_key = panel_stop_key(&tenant, &project, &process); let panel = match self.panel_snapshots.get(&stop_key).cloned() { Some(panel) => stopped_panel_snapshot(panel), None => self.render_operator_panel( tenant.clone(), project.clone(), process.clone(), UserId::new(actor_user), max_download_bytes, true, )?, }; self.panel_snapshots.insert(stop_key, panel.clone()); panel } else { self.stopped_panels.remove(&stop_key); let panel = self.render_operator_panel( tenant.clone(), project.clone(), process.clone(), UserId::new(actor_user), max_download_bytes, false, )?; self.panel_snapshots.insert(stop_key, panel.clone()); panel }; Ok(CoordinatorResponse::OperatorPanel { panel }) } pub(super) fn handle_submit_panel_event( &mut self, tenant: String, project: String, process: String, widget_id: String, kind: PanelEventKind, max_events: u64, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let process = ProcessId::new(process); let stop_key = panel_stop_key(&tenant, &project, &process); let stopped = self.stopped_panels.contains(&stop_key); let panel = if stopped { match self.panel_snapshots.get(&stop_key).cloned() { Some(panel) => stopped_panel_snapshot(panel), None => { let panel = self.render_operator_panel( tenant.clone(), project.clone(), process.clone(), UserId::from("panel-user"), self.quota.download_limit(), true, )?; self.panel_snapshots.insert(stop_key.clone(), panel.clone()); panel } } } else { let panel = self.render_operator_panel( tenant.clone(), project.clone(), process.clone(), UserId::from("panel-user"), self.quota.download_limit(), false, )?; self.panel_snapshots.insert(stop_key, panel.clone()); panel }; let event = PanelEvent { tenant: tenant.clone(), project: project.clone(), process: process.clone(), widget_id: widget_id.clone(), kind, }; let limit_key = (tenant, project, process, widget_id); let mut limit = self .panel_event_limits .get(&limit_key) .cloned() .unwrap_or(RateLimit { max_events, used_events: 0, }); limit.max_events = max_events; panel.accept_event(&event, &mut limit)?; self.panel_event_limits.insert(limit_key, limit.clone()); Ok(CoordinatorResponse::PanelEventAccepted { used_events: limit.used_events, max_events: limit.max_events, }) } fn render_operator_panel( &self, tenant: TenantId, project: ProjectId, process: ProcessId, actor_user: UserId, max_download_bytes: u64, stopped: bool, ) -> Result { self.ensure_operator_panel_scope(&tenant, &project, &process)?; let events = self .task_events .iter() .filter(|event| { event.tenant == tenant && event.project == project && event.process == process }) .collect::>(); let completed = events .iter() .filter(|event| event.status_code == Some(0)) .count() as u64; let total = events.len().max(1) as u64; let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::(); let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::(); let last_task = events.last().map(|event| event.task.clone()); let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone()); if stopped { panel.freeze_program_ui_events(); } panel.add_widget(PanelWidget { id: "process-status".to_owned(), label: "Process Status".to_owned(), kind: PanelWidgetKind::Text { value: if stopped { "stopped".to_owned() } else { "running".to_owned() }, }, })?; panel.add_widget(PanelWidget { id: "task-progress".to_owned(), label: "Tasks".to_owned(), kind: PanelWidgetKind::Progress { current: completed, total, }, })?; panel.add_widget(PanelWidget { id: "task-summary".to_owned(), label: "Task Summary".to_owned(), kind: PanelWidgetKind::Text { value: if events.is_empty() { "no task events recorded".to_owned() } else { events .iter() .map(|event| { format!( "{} [{}]:{:?}:{}", event.task_definition, event.task, event.status_code, event.node ) }) .collect::>() .join(", ") }, }, })?; panel.add_widget(PanelWidget { id: "recent-logs".to_owned(), label: "Recent Logs".to_owned(), kind: PanelWidgetKind::Text { value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"), }, })?; panel.add_widget(PanelWidget { id: "debug-process".to_owned(), label: "Debug Process".to_owned(), kind: PanelWidgetKind::Button { action: "debug-process".to_owned(), }, })?; panel.add_widget(PanelWidget { id: "cancel-process".to_owned(), label: "Cancel Process".to_owned(), kind: PanelWidgetKind::Button { action: "cancel-process".to_owned(), }, })?; if last_task.is_some() { panel.add_widget(PanelWidget { id: "restart-selected-task".to_owned(), label: "Restart Selected Task".to_owned(), kind: PanelWidgetKind::Button { action: "restart-task".to_owned(), }, })?; } let mut actions = vec![ clusterflux_core::ControlPlaneAction::DebugProcess, clusterflux_core::ControlPlaneAction::CancelProcess, ]; if let Some(task) = last_task.clone() { actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task)); } panel.set_control_plane_actions(actions); if let Some(path) = events .iter() .rev() .find_map(|event| event.artifact_path.as_ref()) { let artifact = artifact_id_from_path(path); let context = clusterflux_core::AuthContext { tenant, project, actor: Actor::User(actor_user), }; panel.add_download_widget_from_action( "download-artifact", "Download Artifact", self.artifact_registry.download_action( &context, &artifact, &DownloadPolicy { max_bytes: max_download_bytes, }, ), )?; } Ok(panel) } fn ensure_operator_panel_scope( &self, tenant: &TenantId, project: &ProjectId, process: &ProcessId, ) -> Result<(), CoordinatorServiceError> { let active = self .coordinator .active_process(tenant, project, process) .ok_or_else(|| { CoordinatorError::Unauthorized( "operator panel requires an active virtual process".to_owned(), ) })?; debug_assert_eq!(active.tenant, *tenant); debug_assert_eq!(active.project, *project); Ok(()) } } fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState { panel.freeze_program_ui_events(); if let Some(status) = panel.widgets.get_mut("process-status") { status.kind = PanelWidgetKind::Text { value: "stopped".to_owned(), }; } panel }