#!/usr/bin/env node const assert = require("assert"); const cp = require("child_process"); const { nodeIdentity } = require("./node-signing"); const { ensureRootlessPodman, repo, runFlagshipWorker, send, startFlagship, waitForTaskEvent, waitForJsonLine, } = require("./real-flagship-harness"); const panelNode = "panel-node"; const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode); function widget(panel, id) { const item = panel.widgets[id]; assert(item, `missing panel widget ${id}`); return item; } const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); async function waitForBreakpointHit(addr, process) { for (let attempt = 0; attempt < 2400; attempt += 1) { const status = await send(addr, { type: "inspect_debug_breakpoints", tenant: "tenant", project: "project", actor_user: "user", process, }); assert.strictEqual(status.type, "debug_breakpoints", JSON.stringify(status)); if (status.hit_epoch != null) return status; await delay(25); } throw new Error(`timed out waiting for package breakpoint in ${process}`); } async function waitForDebugEpochFrozen(addr, process, epoch) { for (let attempt = 0; attempt < 2400; attempt += 1) { const status = await send(addr, { type: "inspect_debug_epoch", tenant: "tenant", project: "project", actor_user: "user", process, epoch, }); assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status)); if (status.failed) { throw new Error(status.failure_messages.join("; ")); } if (status.fully_frozen) return status; await delay(25); } throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`); } (async () => { ensureRootlessPodman(); const coordinator = cp.spawn( "cargo", [ "run", "-q", "-p", "clusterflux-coordinator", "--bin", "clusterflux-coordinator", "--", "--listen", "127.0.0.1:0", "--allow-local-trusted-loopback" ], { cwd: repo } ); let coordinatorStderr = ""; let worker; coordinator.stderr.on("data", (chunk) => { coordinatorStderr += chunk.toString(); }); try { const ready = await waitForJsonLine(coordinator); const [host, portText] = ready.listen.split(":"); const addr = { host, port: Number(portText) }; assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); const projectCreated = await send(addr, { type: "create_project", tenant: "tenant", actor_user: "user", project: "project", name: "Operator panel smoke", }); assert.strictEqual(projectCreated.type, "project_created"); worker = await runFlagshipWorker(addr, panelNode, panelNodeIdentity); const workerReady = await worker.ready; assert.strictEqual(workerReady.node_status, "ready"); const workerCompletion = waitForJsonLine(worker.child); const flagship = startFlagship(addr); await waitForTaskEvent( addr, flagship.process, (event) => event.task_definition === "prepare_source", "prepare_source before breakpoint configuration" ); const configuredBreakpoints = await send(addr, { type: "set_debug_breakpoints", tenant: "tenant", project: "project", actor_user: "user", process: flagship.process, probe_symbols: ["clusterflux.probe.package_release"], }); assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints"); const breakpointHit = await waitForBreakpointHit(addr, flagship.process); assert.strictEqual( breakpointHit.hit_probe_symbol, "clusterflux.probe.package_release" ); const frozenEpoch = await waitForDebugEpochFrozen( addr, flagship.process, breakpointHit.hit_epoch ); assert(frozenEpoch.acknowledgements.length >= 2); const compileEvent = await waitForTaskEvent( addr, flagship.process, (event) => event.task_definition === "compile_linux", "compile_linux before the package breakpoint" ); const report = await workerCompletion; assert.strictEqual(report.node_status, "completed"); assert.strictEqual(report.coordinator_response.type, "task_recorded"); const process = flagship.process; const rendered = await send(addr, { type: "render_operator_panel", tenant: "tenant", project: "project", process, actor_user: "user", max_download_bytes: 1024 * 1024, stopped: false }); assert.strictEqual(rendered.type, "operator_panel"); const panel = rendered.panel; assert.strictEqual(panel.tenant, "tenant"); assert.strictEqual(panel.project, "project"); assert.strictEqual(panel.process, process); assert.strictEqual(panel.program_ui_events_enabled, true); assert.deepStrictEqual(widget(panel, "process-status").kind, { Text: { value: "running" } }); const taskProgress = widget(panel, "task-progress").kind.Progress; assert(taskProgress.current >= 2); assert.strictEqual(taskProgress.current, taskProgress.total); const taskSummary = widget(panel, "task-summary").kind.Text.value; assert.match( taskSummary, new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`) ); assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/); const downloadWidget = widget(panel, "download-artifact").kind; const artifact = downloadWidget.ArtifactDownload.artifact; assert( artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length), "panel download must point at a real flagship artifact" ); assert(!JSON.stringify(downloadWidget).includes("url_path")); assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest")); assert.deepStrictEqual(widget(panel, "debug-process").kind, { Button: { action: "debug-process" } }); assert.deepStrictEqual(widget(panel, "cancel-process").kind, { Button: { action: "cancel-process" } }); assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, { Button: { action: "restart-task" } }); assert(panel.control_plane_actions.includes("DebugProcess")); assert(panel.control_plane_actions.includes("CancelProcess")); const restartTarget = panel.control_plane_actions.find( (action) => action.RestartTask )?.RestartTask; assert( restartTarget && taskSummary.includes(`[${restartTarget}]`), "panel restart must target the real flagship task instance" ); assert( panel.control_plane_actions.some( (action) => action.DownloadArtifact === artifact ) ); assert(!JSON.stringify(panel).includes(" action.DownloadArtifact === artifact ) ); const frozenEvent = await send(addr, { type: "submit_panel_event", tenant: "tenant", project: "project", process, widget_id: "debug-process", kind: "ButtonClicked", max_events: 10 }); assert.strictEqual(frozenEvent.type, "error"); assert.match(frozenEvent.message, /program UI events are disabled/i); const crossTenant = await send(addr, { type: "render_operator_panel", tenant: "other", project: "project", process, actor_user: "user", max_download_bytes: 1024 * 1024, stopped: false }); assert.strictEqual(crossTenant.type, "error"); assert.match( crossTenant.message, /scope|tenant|project|requires an active virtual process/i ); assert(!crossTenant.message.includes(process)); const resumed = await send(addr, { type: "resume_debug_epoch", tenant: "tenant", project: "project", actor_user: "user", process, epoch: breakpointHit.hit_epoch, }); assert.strictEqual(resumed.type, "debug_epoch"); const cleanup = await send(addr, { type: "abort_process", tenant: "tenant", project: "project", actor_user: "user", process, }); assert.strictEqual(cleanup.type, "process_aborted"); } catch (error) { if (coordinatorStderr) { error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; } throw error; } finally { worker?.child.kill("SIGTERM"); coordinator.kill("SIGTERM"); } console.log("Operator panel smoke passed"); })().catch((error) => { console.error(error.stack || error.message); process.exit(1); });