Publish Clusterflux 1d0b6fa filtered source
This commit is contained in:
commit
e5d609cfb2
226 changed files with 86613 additions and 0 deletions
307
crates/clusterflux-coordinator/src/service/panels.rs
Normal file
307
crates/clusterflux-coordinator/src/service/panels.rs
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
use clusterflux_core::{
|
||||
Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
|
||||
ProcessId, ProjectId, RateLimit, TenantId, UserId,
|
||||
};
|
||||
|
||||
use crate::CoordinatorError;
|
||||
|
||||
use super::artifact_id_from_path;
|
||||
use super::keys::panel_stop_key;
|
||||
use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
|
||||
|
||||
impl CoordinatorService {
|
||||
pub(super) fn clear_operator_panel_state(
|
||||
&mut self,
|
||||
tenant: &TenantId,
|
||||
project: &ProjectId,
|
||||
process: &ProcessId,
|
||||
) {
|
||||
let stop_key = panel_stop_key(tenant, project, process);
|
||||
self.panel_snapshots.remove(&stop_key);
|
||||
self.stopped_panels.remove(&stop_key);
|
||||
self.panel_event_limits
|
||||
.retain(|(event_tenant, event_project, event_process, _), _| {
|
||||
event_tenant != tenant || event_project != project || event_process != process
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn handle_render_operator_panel(
|
||||
&mut self,
|
||||
tenant: String,
|
||||
project: String,
|
||||
actor_user: String,
|
||||
process: String,
|
||||
max_download_bytes: u64,
|
||||
stopped: bool,
|
||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
||||
let tenant = TenantId::new(tenant);
|
||||
let project = ProjectId::new(project);
|
||||
let process = ProcessId::new(process);
|
||||
let stop_key = panel_stop_key(&tenant, &project, &process);
|
||||
|
||||
let panel = if stopped {
|
||||
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
|
||||
self.stopped_panels.insert(stop_key);
|
||||
let stop_key = panel_stop_key(&tenant, &project, &process);
|
||||
let panel = match self.panel_snapshots.get(&stop_key).cloned() {
|
||||
Some(panel) => stopped_panel_snapshot(panel),
|
||||
None => self.render_operator_panel(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
process.clone(),
|
||||
UserId::new(actor_user),
|
||||
max_download_bytes,
|
||||
true,
|
||||
)?,
|
||||
};
|
||||
self.panel_snapshots.insert(stop_key, panel.clone());
|
||||
panel
|
||||
} else {
|
||||
self.stopped_panels.remove(&stop_key);
|
||||
let panel = self.render_operator_panel(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
process.clone(),
|
||||
UserId::new(actor_user),
|
||||
max_download_bytes,
|
||||
false,
|
||||
)?;
|
||||
self.panel_snapshots.insert(stop_key, panel.clone());
|
||||
panel
|
||||
};
|
||||
Ok(CoordinatorResponse::OperatorPanel { panel })
|
||||
}
|
||||
|
||||
pub(super) fn handle_submit_panel_event(
|
||||
&mut self,
|
||||
tenant: String,
|
||||
project: String,
|
||||
process: String,
|
||||
widget_id: String,
|
||||
kind: PanelEventKind,
|
||||
max_events: u64,
|
||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
||||
let tenant = TenantId::new(tenant);
|
||||
let project = ProjectId::new(project);
|
||||
let process = ProcessId::new(process);
|
||||
let stop_key = panel_stop_key(&tenant, &project, &process);
|
||||
let stopped = self.stopped_panels.contains(&stop_key);
|
||||
let panel = if stopped {
|
||||
match self.panel_snapshots.get(&stop_key).cloned() {
|
||||
Some(panel) => stopped_panel_snapshot(panel),
|
||||
None => {
|
||||
let panel = self.render_operator_panel(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
process.clone(),
|
||||
UserId::from("panel-user"),
|
||||
self.quota.download_limit(),
|
||||
true,
|
||||
)?;
|
||||
self.panel_snapshots.insert(stop_key.clone(), panel.clone());
|
||||
panel
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let panel = self.render_operator_panel(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
process.clone(),
|
||||
UserId::from("panel-user"),
|
||||
self.quota.download_limit(),
|
||||
false,
|
||||
)?;
|
||||
self.panel_snapshots.insert(stop_key, panel.clone());
|
||||
panel
|
||||
};
|
||||
let event = PanelEvent {
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: process.clone(),
|
||||
widget_id: widget_id.clone(),
|
||||
kind,
|
||||
};
|
||||
let limit_key = (tenant, project, process, widget_id);
|
||||
let mut limit = self
|
||||
.panel_event_limits
|
||||
.get(&limit_key)
|
||||
.cloned()
|
||||
.unwrap_or(RateLimit {
|
||||
max_events,
|
||||
used_events: 0,
|
||||
});
|
||||
limit.max_events = max_events;
|
||||
panel.accept_event(&event, &mut limit)?;
|
||||
self.panel_event_limits.insert(limit_key, limit.clone());
|
||||
Ok(CoordinatorResponse::PanelEventAccepted {
|
||||
used_events: limit.used_events,
|
||||
max_events: limit.max_events,
|
||||
})
|
||||
}
|
||||
|
||||
fn render_operator_panel(
|
||||
&self,
|
||||
tenant: TenantId,
|
||||
project: ProjectId,
|
||||
process: ProcessId,
|
||||
actor_user: UserId,
|
||||
max_download_bytes: u64,
|
||||
stopped: bool,
|
||||
) -> Result<PanelState, CoordinatorServiceError> {
|
||||
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
|
||||
|
||||
let events = self
|
||||
.task_events
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
event.tenant == tenant && event.project == project && event.process == process
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let completed = events
|
||||
.iter()
|
||||
.filter(|event| event.status_code == Some(0))
|
||||
.count() as u64;
|
||||
let total = events.len().max(1) as u64;
|
||||
let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::<u64>();
|
||||
let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::<u64>();
|
||||
let last_task = events.last().map(|event| event.task.clone());
|
||||
|
||||
let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone());
|
||||
if stopped {
|
||||
panel.freeze_program_ui_events();
|
||||
}
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "process-status".to_owned(),
|
||||
label: "Process Status".to_owned(),
|
||||
kind: PanelWidgetKind::Text {
|
||||
value: if stopped {
|
||||
"stopped".to_owned()
|
||||
} else {
|
||||
"running".to_owned()
|
||||
},
|
||||
},
|
||||
})?;
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "task-progress".to_owned(),
|
||||
label: "Tasks".to_owned(),
|
||||
kind: PanelWidgetKind::Progress {
|
||||
current: completed,
|
||||
total,
|
||||
},
|
||||
})?;
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "task-summary".to_owned(),
|
||||
label: "Task Summary".to_owned(),
|
||||
kind: PanelWidgetKind::Text {
|
||||
value: if events.is_empty() {
|
||||
"no task events recorded".to_owned()
|
||||
} else {
|
||||
events
|
||||
.iter()
|
||||
.map(|event| {
|
||||
format!(
|
||||
"{} [{}]:{:?}:{}",
|
||||
event.task_definition, event.task, event.status_code, event.node
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
},
|
||||
},
|
||||
})?;
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "recent-logs".to_owned(),
|
||||
label: "Recent Logs".to_owned(),
|
||||
kind: PanelWidgetKind::Text {
|
||||
value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"),
|
||||
},
|
||||
})?;
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "debug-process".to_owned(),
|
||||
label: "Debug Process".to_owned(),
|
||||
kind: PanelWidgetKind::Button {
|
||||
action: "debug-process".to_owned(),
|
||||
},
|
||||
})?;
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "cancel-process".to_owned(),
|
||||
label: "Cancel Process".to_owned(),
|
||||
kind: PanelWidgetKind::Button {
|
||||
action: "cancel-process".to_owned(),
|
||||
},
|
||||
})?;
|
||||
if last_task.is_some() {
|
||||
panel.add_widget(PanelWidget {
|
||||
id: "restart-selected-task".to_owned(),
|
||||
label: "Restart Selected Task".to_owned(),
|
||||
kind: PanelWidgetKind::Button {
|
||||
action: "restart-task".to_owned(),
|
||||
},
|
||||
})?;
|
||||
}
|
||||
|
||||
let mut actions = vec![
|
||||
clusterflux_core::ControlPlaneAction::DebugProcess,
|
||||
clusterflux_core::ControlPlaneAction::CancelProcess,
|
||||
];
|
||||
if let Some(task) = last_task.clone() {
|
||||
actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
|
||||
}
|
||||
panel.set_control_plane_actions(actions);
|
||||
|
||||
if let Some(path) = events
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|event| event.artifact_path.as_ref())
|
||||
{
|
||||
let artifact = artifact_id_from_path(path);
|
||||
let context = clusterflux_core::AuthContext {
|
||||
tenant,
|
||||
project,
|
||||
actor: Actor::User(actor_user),
|
||||
};
|
||||
panel.add_download_widget_from_action(
|
||||
"download-artifact",
|
||||
"Download Artifact",
|
||||
self.artifact_registry.download_action(
|
||||
&context,
|
||||
&artifact,
|
||||
&DownloadPolicy {
|
||||
max_bytes: max_download_bytes,
|
||||
},
|
||||
),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(panel)
|
||||
}
|
||||
|
||||
fn ensure_operator_panel_scope(
|
||||
&self,
|
||||
tenant: &TenantId,
|
||||
project: &ProjectId,
|
||||
process: &ProcessId,
|
||||
) -> Result<(), CoordinatorServiceError> {
|
||||
let active = self
|
||||
.coordinator
|
||||
.active_process(tenant, project, process)
|
||||
.ok_or_else(|| {
|
||||
CoordinatorError::Unauthorized(
|
||||
"operator panel requires an active virtual process".to_owned(),
|
||||
)
|
||||
})?;
|
||||
debug_assert_eq!(active.tenant, *tenant);
|
||||
debug_assert_eq!(active.project, *project);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState {
|
||||
panel.freeze_program_ui_events();
|
||||
if let Some(status) = panel.widgets.get_mut("process-status") {
|
||||
status.kind = PanelWidgetKind::Text {
|
||||
value: "stopped".to_owned(),
|
||||
};
|
||||
}
|
||||
panel
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue