307 lines
11 KiB
Rust
307 lines
11 KiB
Rust
use clusterflux_core::{
|
|
Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
|
|
ProcessId, ProjectId, RateLimit, TenantId, UserId,
|
|
};
|
|
|
|
use crate::CoordinatorError;
|
|
|
|
use super::artifact_id_from_path;
|
|
use super::keys::panel_stop_key;
|
|
use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
|
|
|
|
impl CoordinatorService {
|
|
pub(super) fn clear_operator_panel_state(
|
|
&mut self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
process: &ProcessId,
|
|
) {
|
|
let stop_key = panel_stop_key(tenant, project, process);
|
|
self.panel_snapshots.remove(&stop_key);
|
|
self.stopped_panels.remove(&stop_key);
|
|
self.panel_event_limits
|
|
.retain(|(event_tenant, event_project, event_process, _), _| {
|
|
event_tenant != tenant || event_project != project || event_process != process
|
|
});
|
|
}
|
|
|
|
pub(super) fn handle_render_operator_panel(
|
|
&mut self,
|
|
tenant: String,
|
|
project: String,
|
|
actor_user: String,
|
|
process: String,
|
|
max_download_bytes: u64,
|
|
stopped: bool,
|
|
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
let tenant = TenantId::new(tenant);
|
|
let project = ProjectId::new(project);
|
|
let process = ProcessId::new(process);
|
|
let stop_key = panel_stop_key(&tenant, &project, &process);
|
|
|
|
let panel = if stopped {
|
|
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
|
|
self.stopped_panels.insert(stop_key);
|
|
let stop_key = panel_stop_key(&tenant, &project, &process);
|
|
let panel = match self.panel_snapshots.get(&stop_key).cloned() {
|
|
Some(panel) => stopped_panel_snapshot(panel),
|
|
None => self.render_operator_panel(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
process.clone(),
|
|
UserId::new(actor_user),
|
|
max_download_bytes,
|
|
true,
|
|
)?,
|
|
};
|
|
self.panel_snapshots.insert(stop_key, panel.clone());
|
|
panel
|
|
} else {
|
|
self.stopped_panels.remove(&stop_key);
|
|
let panel = self.render_operator_panel(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
process.clone(),
|
|
UserId::new(actor_user),
|
|
max_download_bytes,
|
|
false,
|
|
)?;
|
|
self.panel_snapshots.insert(stop_key, panel.clone());
|
|
panel
|
|
};
|
|
Ok(CoordinatorResponse::OperatorPanel { panel })
|
|
}
|
|
|
|
pub(super) fn handle_submit_panel_event(
|
|
&mut self,
|
|
tenant: String,
|
|
project: String,
|
|
process: String,
|
|
widget_id: String,
|
|
kind: PanelEventKind,
|
|
max_events: u64,
|
|
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
let tenant = TenantId::new(tenant);
|
|
let project = ProjectId::new(project);
|
|
let process = ProcessId::new(process);
|
|
let stop_key = panel_stop_key(&tenant, &project, &process);
|
|
let stopped = self.stopped_panels.contains(&stop_key);
|
|
let panel = if stopped {
|
|
match self.panel_snapshots.get(&stop_key).cloned() {
|
|
Some(panel) => stopped_panel_snapshot(panel),
|
|
None => {
|
|
let panel = self.render_operator_panel(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
process.clone(),
|
|
UserId::from("panel-user"),
|
|
self.quota.download_limit(),
|
|
true,
|
|
)?;
|
|
self.panel_snapshots.insert(stop_key.clone(), panel.clone());
|
|
panel
|
|
}
|
|
}
|
|
} else {
|
|
let panel = self.render_operator_panel(
|
|
tenant.clone(),
|
|
project.clone(),
|
|
process.clone(),
|
|
UserId::from("panel-user"),
|
|
self.quota.download_limit(),
|
|
false,
|
|
)?;
|
|
self.panel_snapshots.insert(stop_key, panel.clone());
|
|
panel
|
|
};
|
|
let event = PanelEvent {
|
|
tenant: tenant.clone(),
|
|
project: project.clone(),
|
|
process: process.clone(),
|
|
widget_id: widget_id.clone(),
|
|
kind,
|
|
};
|
|
let limit_key = (tenant, project, process, widget_id);
|
|
let mut limit = self
|
|
.panel_event_limits
|
|
.get(&limit_key)
|
|
.cloned()
|
|
.unwrap_or(RateLimit {
|
|
max_events,
|
|
used_events: 0,
|
|
});
|
|
limit.max_events = max_events;
|
|
panel.accept_event(&event, &mut limit)?;
|
|
self.panel_event_limits.insert(limit_key, limit.clone());
|
|
Ok(CoordinatorResponse::PanelEventAccepted {
|
|
used_events: limit.used_events,
|
|
max_events: limit.max_events,
|
|
})
|
|
}
|
|
|
|
fn render_operator_panel(
|
|
&self,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
process: ProcessId,
|
|
actor_user: UserId,
|
|
max_download_bytes: u64,
|
|
stopped: bool,
|
|
) -> Result<PanelState, CoordinatorServiceError> {
|
|
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
|
|
|
|
let events = self
|
|
.task_events
|
|
.iter()
|
|
.filter(|event| {
|
|
event.tenant == tenant && event.project == project && event.process == process
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let completed = events
|
|
.iter()
|
|
.filter(|event| event.status_code == Some(0))
|
|
.count() as u64;
|
|
let total = events.len().max(1) as u64;
|
|
let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::<u64>();
|
|
let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::<u64>();
|
|
let last_task = events.last().map(|event| event.task.clone());
|
|
|
|
let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone());
|
|
if stopped {
|
|
panel.freeze_program_ui_events();
|
|
}
|
|
panel.add_widget(PanelWidget {
|
|
id: "process-status".to_owned(),
|
|
label: "Process Status".to_owned(),
|
|
kind: PanelWidgetKind::Text {
|
|
value: if stopped {
|
|
"stopped".to_owned()
|
|
} else {
|
|
"running".to_owned()
|
|
},
|
|
},
|
|
})?;
|
|
panel.add_widget(PanelWidget {
|
|
id: "task-progress".to_owned(),
|
|
label: "Tasks".to_owned(),
|
|
kind: PanelWidgetKind::Progress {
|
|
current: completed,
|
|
total,
|
|
},
|
|
})?;
|
|
panel.add_widget(PanelWidget {
|
|
id: "task-summary".to_owned(),
|
|
label: "Task Summary".to_owned(),
|
|
kind: PanelWidgetKind::Text {
|
|
value: if events.is_empty() {
|
|
"no task events recorded".to_owned()
|
|
} else {
|
|
events
|
|
.iter()
|
|
.map(|event| {
|
|
format!(
|
|
"{} [{}]:{:?}:{}",
|
|
event.task_definition, event.task, event.status_code, event.node
|
|
)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
},
|
|
},
|
|
})?;
|
|
panel.add_widget(PanelWidget {
|
|
id: "recent-logs".to_owned(),
|
|
label: "Recent Logs".to_owned(),
|
|
kind: PanelWidgetKind::Text {
|
|
value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"),
|
|
},
|
|
})?;
|
|
panel.add_widget(PanelWidget {
|
|
id: "debug-process".to_owned(),
|
|
label: "Debug Process".to_owned(),
|
|
kind: PanelWidgetKind::Button {
|
|
action: "debug-process".to_owned(),
|
|
},
|
|
})?;
|
|
panel.add_widget(PanelWidget {
|
|
id: "cancel-process".to_owned(),
|
|
label: "Cancel Process".to_owned(),
|
|
kind: PanelWidgetKind::Button {
|
|
action: "cancel-process".to_owned(),
|
|
},
|
|
})?;
|
|
if last_task.is_some() {
|
|
panel.add_widget(PanelWidget {
|
|
id: "restart-selected-task".to_owned(),
|
|
label: "Restart Selected Task".to_owned(),
|
|
kind: PanelWidgetKind::Button {
|
|
action: "restart-task".to_owned(),
|
|
},
|
|
})?;
|
|
}
|
|
|
|
let mut actions = vec![
|
|
clusterflux_core::ControlPlaneAction::DebugProcess,
|
|
clusterflux_core::ControlPlaneAction::CancelProcess,
|
|
];
|
|
if let Some(task) = last_task.clone() {
|
|
actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
|
|
}
|
|
panel.set_control_plane_actions(actions);
|
|
|
|
if let Some(path) = events
|
|
.iter()
|
|
.rev()
|
|
.find_map(|event| event.artifact_path.as_ref())
|
|
{
|
|
let artifact = artifact_id_from_path(path);
|
|
let context = clusterflux_core::AuthContext {
|
|
tenant,
|
|
project,
|
|
actor: Actor::User(actor_user),
|
|
};
|
|
panel.add_download_widget_from_action(
|
|
"download-artifact",
|
|
"Download Artifact",
|
|
self.artifact_registry.download_action(
|
|
&context,
|
|
&artifact,
|
|
&DownloadPolicy {
|
|
max_bytes: max_download_bytes,
|
|
},
|
|
),
|
|
)?;
|
|
}
|
|
|
|
Ok(panel)
|
|
}
|
|
|
|
fn ensure_operator_panel_scope(
|
|
&self,
|
|
tenant: &TenantId,
|
|
project: &ProjectId,
|
|
process: &ProcessId,
|
|
) -> Result<(), CoordinatorServiceError> {
|
|
let active = self
|
|
.coordinator
|
|
.active_process(tenant, project, process)
|
|
.ok_or_else(|| {
|
|
CoordinatorError::Unauthorized(
|
|
"operator panel requires an active virtual process".to_owned(),
|
|
)
|
|
})?;
|
|
debug_assert_eq!(active.tenant, *tenant);
|
|
debug_assert_eq!(active.project, *project);
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState {
|
|
panel.freeze_program_ui_events();
|
|
if let Some(status) = panel.widgets.get_mut("process-status") {
|
|
status.kind = PanelWidgetKind::Text {
|
|
value: "stopped".to_owned(),
|
|
};
|
|
}
|
|
panel
|
|
}
|