c1769967c1
Source commit: 01875e88a3e25379c309489f9f057dd97c0d37de Public tree identity: sha256:8f37a1aa0cc8f408975daf9cabbe193f8f4c169c6d25e8795e67a3ed5083c76b
367 lines
12 KiB
JavaScript
Executable file
367 lines
12 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
|
|
|
const assert = require("assert");
|
|
const cp = require("child_process");
|
|
const { nodeIdentity } = require("./node-signing");
|
|
const {
|
|
ensureRootlessPodman,
|
|
repo,
|
|
runFlagshipWorker,
|
|
send,
|
|
startFlagship,
|
|
waitForTaskEvent,
|
|
waitForJsonLine,
|
|
waitForNodeStatus,
|
|
} = require("./real-flagship-harness");
|
|
|
|
const panelNode = "panel-node";
|
|
const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode);
|
|
|
|
function widget(panel, id) {
|
|
const item = panel.widgets[id];
|
|
assert(item, `missing panel widget ${id}`);
|
|
return item;
|
|
}
|
|
|
|
const delay = (milliseconds) =>
|
|
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
|
|
async function waitForBreakpointHit(addr, process) {
|
|
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
|
const status = await send(addr, {
|
|
type: "inspect_debug_breakpoints",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
process,
|
|
});
|
|
assert.strictEqual(status.type, "debug_breakpoints", JSON.stringify(status));
|
|
if (status.hit_epoch != null) return status;
|
|
await delay(25);
|
|
}
|
|
throw new Error(`timed out waiting for package breakpoint in ${process}`);
|
|
}
|
|
|
|
async function waitForDebugEpochFrozen(addr, process, epoch) {
|
|
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
|
const status = await send(addr, {
|
|
type: "inspect_debug_epoch",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
process,
|
|
epoch,
|
|
});
|
|
assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status));
|
|
if (status.failed) {
|
|
throw new Error(status.failure_messages.join("; "));
|
|
}
|
|
if (status.fully_frozen) return status;
|
|
await delay(25);
|
|
}
|
|
throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`);
|
|
}
|
|
|
|
(async () => {
|
|
ensureRootlessPodman();
|
|
const coordinator = cp.spawn(
|
|
"cargo",
|
|
[
|
|
"run",
|
|
"-q",
|
|
"-p",
|
|
"clusterflux-coordinator",
|
|
"--bin",
|
|
"clusterflux-coordinator",
|
|
"--",
|
|
"--listen",
|
|
"127.0.0.1:0",
|
|
"--allow-local-trusted-loopback"
|
|
],
|
|
{ cwd: repo }
|
|
);
|
|
let coordinatorStderr = "";
|
|
let worker;
|
|
coordinator.stderr.on("data", (chunk) => {
|
|
coordinatorStderr += chunk.toString();
|
|
});
|
|
|
|
try {
|
|
const ready = await waitForJsonLine(coordinator);
|
|
const [host, portText] = ready.listen.split(":");
|
|
const addr = { host, port: Number(portText) };
|
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
|
const projectCreated = await send(addr, {
|
|
type: "create_project",
|
|
tenant: "tenant",
|
|
actor_user: "user",
|
|
project: "project",
|
|
name: "Operator panel smoke",
|
|
});
|
|
assert.strictEqual(projectCreated.type, "project_created");
|
|
|
|
worker = await runFlagshipWorker(addr, panelNode, panelNodeIdentity);
|
|
const workerReady = await worker.ready;
|
|
assert.strictEqual(workerReady.node_status, "ready");
|
|
const workerCompletion = waitForNodeStatus(worker.child, "completed");
|
|
const flagship = startFlagship(addr);
|
|
await waitForTaskEvent(
|
|
addr,
|
|
flagship.process,
|
|
(event) => event.task_definition === "prepare_source",
|
|
"prepare_source before breakpoint configuration"
|
|
);
|
|
const configuredBreakpoints = await send(addr, {
|
|
type: "set_debug_breakpoints",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
process: flagship.process,
|
|
probe_symbols: ["clusterflux.probe.package_release"],
|
|
});
|
|
assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints");
|
|
const breakpointHit = await waitForBreakpointHit(addr, flagship.process);
|
|
assert.strictEqual(
|
|
breakpointHit.hit_probe_symbol,
|
|
"clusterflux.probe.package_release"
|
|
);
|
|
const frozenEpoch = await waitForDebugEpochFrozen(
|
|
addr,
|
|
flagship.process,
|
|
breakpointHit.hit_epoch
|
|
);
|
|
assert(frozenEpoch.acknowledgements.length >= 2);
|
|
const compileEvent = await waitForTaskEvent(
|
|
addr,
|
|
flagship.process,
|
|
(event) => event.task_definition === "compile_linux",
|
|
"compile_linux before the package breakpoint"
|
|
);
|
|
const report = await workerCompletion;
|
|
assert.strictEqual(report.node_status, "completed");
|
|
assert.strictEqual(report.coordinator_response.type, "task_recorded");
|
|
const process = flagship.process;
|
|
|
|
const rendered = await send(addr, {
|
|
type: "render_operator_panel",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
actor_user: "user",
|
|
max_download_bytes: 1024 * 1024,
|
|
stopped: false
|
|
});
|
|
assert.strictEqual(rendered.type, "operator_panel");
|
|
const panel = rendered.panel;
|
|
assert.strictEqual(panel.tenant, "tenant");
|
|
assert.strictEqual(panel.project, "project");
|
|
assert.strictEqual(panel.process, process);
|
|
assert.strictEqual(panel.program_ui_events_enabled, true);
|
|
|
|
assert.deepStrictEqual(widget(panel, "process-status").kind, {
|
|
Text: { value: "running" }
|
|
});
|
|
const taskProgress = widget(panel, "task-progress").kind.Progress;
|
|
assert(taskProgress.current >= 2);
|
|
assert.strictEqual(taskProgress.current, taskProgress.total);
|
|
const taskSummary = widget(panel, "task-summary").kind.Text.value;
|
|
assert.match(
|
|
taskSummary,
|
|
new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`)
|
|
);
|
|
assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/);
|
|
const downloadWidget = widget(panel, "download-artifact").kind;
|
|
const artifact = downloadWidget.ArtifactDownload.artifact;
|
|
assert(
|
|
artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length),
|
|
"panel download must point at a real flagship artifact"
|
|
);
|
|
assert(!JSON.stringify(downloadWidget).includes("url_path"));
|
|
assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest"));
|
|
assert.deepStrictEqual(widget(panel, "debug-process").kind, {
|
|
Button: { action: "debug-process" }
|
|
});
|
|
assert.deepStrictEqual(widget(panel, "cancel-process").kind, {
|
|
Button: { action: "cancel-process" }
|
|
});
|
|
assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, {
|
|
Button: { action: "restart-task" }
|
|
});
|
|
assert(panel.control_plane_actions.includes("DebugProcess"));
|
|
assert(panel.control_plane_actions.includes("CancelProcess"));
|
|
const restartTarget = panel.control_plane_actions.find(
|
|
(action) => action.RestartTask
|
|
)?.RestartTask;
|
|
assert(
|
|
restartTarget && taskSummary.includes(`[${restartTarget}]`),
|
|
"panel restart must target the real flagship task instance"
|
|
);
|
|
assert(
|
|
panel.control_plane_actions.some(
|
|
(action) => action.DownloadArtifact === artifact
|
|
)
|
|
);
|
|
assert(!JSON.stringify(panel).includes("<script"));
|
|
assert(!JSON.stringify(panel).toLowerCase().includes("oauth"));
|
|
|
|
const panelLink = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact: downloadWidget.ArtifactDownload.artifact,
|
|
max_bytes: 1024 * 1024,
|
|
ttl_seconds: 60
|
|
});
|
|
assert.strictEqual(panelLink.type, "artifact_download_link");
|
|
assert.strictEqual(panelLink.link.artifact, downloadWidget.ArtifactDownload.artifact);
|
|
assert.match(panelLink.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
|
|
assert.deepStrictEqual(panelLink.link.source, { RetainedNode: "panel-node" });
|
|
assert(panelLink.link.url_path.endsWith(`/artifacts/tenant/project/${process}/${artifact}`));
|
|
|
|
const apiTooLarge = await send(addr, {
|
|
type: "create_artifact_download_link",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
artifact: downloadWidget.ArtifactDownload.artifact,
|
|
max_bytes: 1,
|
|
ttl_seconds: 60
|
|
});
|
|
assert.strictEqual(apiTooLarge.type, "error");
|
|
assert.match(apiTooLarge.message, /exceeds download limit/);
|
|
|
|
const panelTooLarge = await send(addr, {
|
|
type: "render_operator_panel",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
actor_user: "user",
|
|
max_download_bytes: 1,
|
|
stopped: false
|
|
});
|
|
assert.strictEqual(panelTooLarge.type, "error");
|
|
assert.match(panelTooLarge.message, /exceeds download limit/);
|
|
|
|
const accepted = await send(addr, {
|
|
type: "submit_panel_event",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
widget_id: "debug-process",
|
|
kind: "ButtonClicked",
|
|
max_events: 1
|
|
});
|
|
assert.strictEqual(accepted.type, "panel_event_accepted");
|
|
assert.strictEqual(accepted.used_events, 1);
|
|
assert.strictEqual(accepted.max_events, 1);
|
|
|
|
const rateLimited = await send(addr, {
|
|
type: "submit_panel_event",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
widget_id: "debug-process",
|
|
kind: "ButtonClicked",
|
|
max_events: 1
|
|
});
|
|
assert.strictEqual(rateLimited.type, "error");
|
|
assert.match(rateLimited.message, /rate limit/i);
|
|
|
|
const stopped = await send(addr, {
|
|
type: "render_operator_panel",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
actor_user: "user",
|
|
max_download_bytes: 1024 * 1024,
|
|
stopped: true
|
|
});
|
|
assert.strictEqual(stopped.type, "operator_panel");
|
|
assert.strictEqual(stopped.panel.program_ui_events_enabled, false);
|
|
assert.deepStrictEqual(widget(stopped.panel, "process-status").kind, {
|
|
Text: { value: "stopped" }
|
|
});
|
|
assert.deepStrictEqual(
|
|
widget(stopped.panel, "task-progress").kind,
|
|
widget(panel, "task-progress").kind
|
|
);
|
|
assert.deepStrictEqual(
|
|
widget(stopped.panel, "task-summary").kind,
|
|
widget(panel, "task-summary").kind
|
|
);
|
|
assert.deepStrictEqual(
|
|
widget(stopped.panel, "recent-logs").kind,
|
|
widget(panel, "recent-logs").kind
|
|
);
|
|
assert.deepStrictEqual(
|
|
widget(stopped.panel, "download-artifact").kind,
|
|
widget(panel, "download-artifact").kind
|
|
);
|
|
assert(stopped.panel.control_plane_actions.includes("DebugProcess"));
|
|
assert(
|
|
stopped.panel.control_plane_actions.some(
|
|
(action) => action.DownloadArtifact === artifact
|
|
)
|
|
);
|
|
|
|
const frozenEvent = await send(addr, {
|
|
type: "submit_panel_event",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
process,
|
|
widget_id: "debug-process",
|
|
kind: "ButtonClicked",
|
|
max_events: 10
|
|
});
|
|
assert.strictEqual(frozenEvent.type, "error");
|
|
assert.match(frozenEvent.message, /program UI events are disabled/i);
|
|
|
|
const crossTenant = await send(addr, {
|
|
type: "render_operator_panel",
|
|
tenant: "other",
|
|
project: "project",
|
|
process,
|
|
actor_user: "user",
|
|
max_download_bytes: 1024 * 1024,
|
|
stopped: false
|
|
});
|
|
assert.strictEqual(crossTenant.type, "error");
|
|
assert.match(
|
|
crossTenant.message,
|
|
/scope|tenant|project|requires an active virtual process/i
|
|
);
|
|
assert(!crossTenant.message.includes(process));
|
|
|
|
const resumed = await send(addr, {
|
|
type: "resume_debug_epoch",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
process,
|
|
epoch: breakpointHit.hit_epoch,
|
|
});
|
|
assert.strictEqual(resumed.type, "debug_epoch");
|
|
const cleanup = await send(addr, {
|
|
type: "abort_process",
|
|
tenant: "tenant",
|
|
project: "project",
|
|
actor_user: "user",
|
|
process,
|
|
});
|
|
assert.strictEqual(cleanup.type, "process_aborted");
|
|
} catch (error) {
|
|
if (coordinatorStderr) {
|
|
error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`;
|
|
}
|
|
throw error;
|
|
} finally {
|
|
worker?.child.kill("SIGTERM");
|
|
coordinator.kill("SIGTERM");
|
|
}
|
|
|
|
console.log("Operator panel smoke passed");
|
|
})().catch((error) => {
|
|
console.error(error.stack || error.message);
|
|
process.exit(1);
|
|
});
|