Public release release-b8454b34d38c
Source commit: b8454b34d38cc2ba0bd278e842060db45d091e1c Public tree identity: sha256:69d6c8143bf6227e1bb07852aa94e8af9c26793eca252bb46788894f728cb6d2
This commit is contained in:
Clusterflux release
commit
d2e84229c5
221 changed files with 80960 additions and 0 deletions
367
scripts/operator-panel-smoke.js
Executable file
367
scripts/operator-panel-smoke.js
Executable file
|
|
@ -0,0 +1,367 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const assert = require("assert");
|
||||
const cp = require("child_process");
|
||||
const { nodeIdentity } = require("./node-signing");
|
||||
const {
|
||||
ensureRootlessPodman,
|
||||
repo,
|
||||
runFlagshipWorker,
|
||||
send,
|
||||
startFlagship,
|
||||
waitForTaskEvent,
|
||||
waitForJsonLine,
|
||||
waitForNodeStatus,
|
||||
} = require("./real-flagship-harness");
|
||||
|
||||
const panelNode = "panel-node";
|
||||
const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode);
|
||||
|
||||
function widget(panel, id) {
|
||||
const item = panel.widgets[id];
|
||||
assert(item, `missing panel widget ${id}`);
|
||||
return item;
|
||||
}
|
||||
|
||||
const delay = (milliseconds) =>
|
||||
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
async function waitForBreakpointHit(addr, process) {
|
||||
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
||||
const status = await send(addr, {
|
||||
type: "inspect_debug_breakpoints",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
process,
|
||||
});
|
||||
assert.strictEqual(status.type, "debug_breakpoints", JSON.stringify(status));
|
||||
if (status.hit_epoch != null) return status;
|
||||
await delay(25);
|
||||
}
|
||||
throw new Error(`timed out waiting for package breakpoint in ${process}`);
|
||||
}
|
||||
|
||||
async function waitForDebugEpochFrozen(addr, process, epoch) {
|
||||
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
||||
const status = await send(addr, {
|
||||
type: "inspect_debug_epoch",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
process,
|
||||
epoch,
|
||||
});
|
||||
assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status));
|
||||
if (status.failed) {
|
||||
throw new Error(status.failure_messages.join("; "));
|
||||
}
|
||||
if (status.fully_frozen) return status;
|
||||
await delay(25);
|
||||
}
|
||||
throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
ensureRootlessPodman();
|
||||
const coordinator = cp.spawn(
|
||||
"cargo",
|
||||
[
|
||||
"run",
|
||||
"-q",
|
||||
"-p",
|
||||
"clusterflux-coordinator",
|
||||
"--bin",
|
||||
"clusterflux-coordinator",
|
||||
"--",
|
||||
"--listen",
|
||||
"127.0.0.1:0",
|
||||
"--allow-local-trusted-loopback"
|
||||
],
|
||||
{ cwd: repo }
|
||||
);
|
||||
let coordinatorStderr = "";
|
||||
let worker;
|
||||
coordinator.stderr.on("data", (chunk) => {
|
||||
coordinatorStderr += chunk.toString();
|
||||
});
|
||||
|
||||
try {
|
||||
const ready = await waitForJsonLine(coordinator);
|
||||
const [host, portText] = ready.listen.split(":");
|
||||
const addr = { host, port: Number(portText) };
|
||||
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||
const projectCreated = await send(addr, {
|
||||
type: "create_project",
|
||||
tenant: "tenant",
|
||||
actor_user: "user",
|
||||
project: "project",
|
||||
name: "Operator panel smoke",
|
||||
});
|
||||
assert.strictEqual(projectCreated.type, "project_created");
|
||||
|
||||
worker = await runFlagshipWorker(addr, panelNode, panelNodeIdentity);
|
||||
const workerReady = await worker.ready;
|
||||
assert.strictEqual(workerReady.node_status, "ready");
|
||||
const workerCompletion = waitForNodeStatus(worker.child, "completed");
|
||||
const flagship = startFlagship(addr);
|
||||
await waitForTaskEvent(
|
||||
addr,
|
||||
flagship.process,
|
||||
(event) => event.task_definition === "snapshot_current_project",
|
||||
"snapshot_current_project before breakpoint configuration"
|
||||
);
|
||||
const configuredBreakpoints = await send(addr, {
|
||||
type: "set_debug_breakpoints",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
process: flagship.process,
|
||||
probe_symbols: ["clusterflux.probe.compile"],
|
||||
});
|
||||
assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints");
|
||||
const breakpointHit = await waitForBreakpointHit(addr, flagship.process);
|
||||
assert.strictEqual(
|
||||
breakpointHit.hit_probe_symbol,
|
||||
"clusterflux.probe.compile"
|
||||
);
|
||||
const frozenEpoch = await waitForDebugEpochFrozen(
|
||||
addr,
|
||||
flagship.process,
|
||||
breakpointHit.hit_epoch
|
||||
);
|
||||
assert(frozenEpoch.acknowledgements.length >= 2);
|
||||
const compileEvent = await waitForTaskEvent(
|
||||
addr,
|
||||
flagship.process,
|
||||
(event) => event.task_definition === "compile",
|
||||
"compile before the package breakpoint"
|
||||
);
|
||||
const report = await workerCompletion;
|
||||
assert.strictEqual(report.node_status, "completed");
|
||||
assert.strictEqual(report.coordinator_response.type, "task_recorded");
|
||||
const process = flagship.process;
|
||||
|
||||
const rendered = await send(addr, {
|
||||
type: "render_operator_panel",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
actor_user: "user",
|
||||
max_download_bytes: 1024 * 1024,
|
||||
stopped: false
|
||||
});
|
||||
assert.strictEqual(rendered.type, "operator_panel");
|
||||
const panel = rendered.panel;
|
||||
assert.strictEqual(panel.tenant, "tenant");
|
||||
assert.strictEqual(panel.project, "project");
|
||||
assert.strictEqual(panel.process, process);
|
||||
assert.strictEqual(panel.program_ui_events_enabled, true);
|
||||
|
||||
assert.deepStrictEqual(widget(panel, "process-status").kind, {
|
||||
Text: { value: "running" }
|
||||
});
|
||||
const taskProgress = widget(panel, "task-progress").kind.Progress;
|
||||
assert(taskProgress.current >= 2);
|
||||
assert.strictEqual(taskProgress.current, taskProgress.total);
|
||||
const taskSummary = widget(panel, "task-summary").kind.Text.value;
|
||||
assert.match(
|
||||
taskSummary,
|
||||
new RegExp(`compile \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`)
|
||||
);
|
||||
assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/);
|
||||
const downloadWidget = widget(panel, "download-artifact").kind;
|
||||
const artifact = downloadWidget.ArtifactDownload.artifact;
|
||||
assert(
|
||||
artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length),
|
||||
"panel download must point at a real flagship artifact"
|
||||
);
|
||||
assert(!JSON.stringify(downloadWidget).includes("url_path"));
|
||||
assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest"));
|
||||
assert.deepStrictEqual(widget(panel, "debug-process").kind, {
|
||||
Button: { action: "debug-process" }
|
||||
});
|
||||
assert.deepStrictEqual(widget(panel, "cancel-process").kind, {
|
||||
Button: { action: "cancel-process" }
|
||||
});
|
||||
assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, {
|
||||
Button: { action: "restart-task" }
|
||||
});
|
||||
assert(panel.control_plane_actions.includes("DebugProcess"));
|
||||
assert(panel.control_plane_actions.includes("CancelProcess"));
|
||||
const restartTarget = panel.control_plane_actions.find(
|
||||
(action) => action.RestartTask
|
||||
)?.RestartTask;
|
||||
assert(
|
||||
restartTarget && taskSummary.includes(`[${restartTarget}]`),
|
||||
"panel restart must target the real flagship task instance"
|
||||
);
|
||||
assert(
|
||||
panel.control_plane_actions.some(
|
||||
(action) => action.DownloadArtifact === artifact
|
||||
)
|
||||
);
|
||||
assert(!JSON.stringify(panel).includes("<script"));
|
||||
assert(!JSON.stringify(panel).toLowerCase().includes("oauth"));
|
||||
|
||||
const panelLink = await send(addr, {
|
||||
type: "create_artifact_download_link",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
artifact: downloadWidget.ArtifactDownload.artifact,
|
||||
max_bytes: 1024 * 1024,
|
||||
ttl_seconds: 60
|
||||
});
|
||||
assert.strictEqual(panelLink.type, "artifact_download_link");
|
||||
assert.strictEqual(panelLink.link.artifact, downloadWidget.ArtifactDownload.artifact);
|
||||
assert.match(panelLink.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
|
||||
assert.deepStrictEqual(panelLink.link.source, { RetainedNode: "panel-node" });
|
||||
assert(panelLink.link.url_path.endsWith(`/artifacts/tenant/project/${process}/${artifact}`));
|
||||
|
||||
const apiTooLarge = await send(addr, {
|
||||
type: "create_artifact_download_link",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
artifact: downloadWidget.ArtifactDownload.artifact,
|
||||
max_bytes: 1,
|
||||
ttl_seconds: 60
|
||||
});
|
||||
assert.strictEqual(apiTooLarge.type, "error");
|
||||
assert.match(apiTooLarge.message, /exceeds download limit/);
|
||||
|
||||
const panelTooLarge = await send(addr, {
|
||||
type: "render_operator_panel",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
actor_user: "user",
|
||||
max_download_bytes: 1,
|
||||
stopped: false
|
||||
});
|
||||
assert.strictEqual(panelTooLarge.type, "error");
|
||||
assert.match(panelTooLarge.message, /exceeds download limit/);
|
||||
|
||||
const accepted = await send(addr, {
|
||||
type: "submit_panel_event",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
widget_id: "debug-process",
|
||||
kind: "ButtonClicked",
|
||||
max_events: 1
|
||||
});
|
||||
assert.strictEqual(accepted.type, "panel_event_accepted");
|
||||
assert.strictEqual(accepted.used_events, 1);
|
||||
assert.strictEqual(accepted.max_events, 1);
|
||||
|
||||
const rateLimited = await send(addr, {
|
||||
type: "submit_panel_event",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
widget_id: "debug-process",
|
||||
kind: "ButtonClicked",
|
||||
max_events: 1
|
||||
});
|
||||
assert.strictEqual(rateLimited.type, "error");
|
||||
assert.match(rateLimited.message, /rate limit/i);
|
||||
|
||||
const stopped = await send(addr, {
|
||||
type: "render_operator_panel",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
actor_user: "user",
|
||||
max_download_bytes: 1024 * 1024,
|
||||
stopped: true
|
||||
});
|
||||
assert.strictEqual(stopped.type, "operator_panel");
|
||||
assert.strictEqual(stopped.panel.program_ui_events_enabled, false);
|
||||
assert.deepStrictEqual(widget(stopped.panel, "process-status").kind, {
|
||||
Text: { value: "stopped" }
|
||||
});
|
||||
assert.deepStrictEqual(
|
||||
widget(stopped.panel, "task-progress").kind,
|
||||
widget(panel, "task-progress").kind
|
||||
);
|
||||
assert.deepStrictEqual(
|
||||
widget(stopped.panel, "task-summary").kind,
|
||||
widget(panel, "task-summary").kind
|
||||
);
|
||||
assert.deepStrictEqual(
|
||||
widget(stopped.panel, "recent-logs").kind,
|
||||
widget(panel, "recent-logs").kind
|
||||
);
|
||||
assert.deepStrictEqual(
|
||||
widget(stopped.panel, "download-artifact").kind,
|
||||
widget(panel, "download-artifact").kind
|
||||
);
|
||||
assert(stopped.panel.control_plane_actions.includes("DebugProcess"));
|
||||
assert(
|
||||
stopped.panel.control_plane_actions.some(
|
||||
(action) => action.DownloadArtifact === artifact
|
||||
)
|
||||
);
|
||||
|
||||
const frozenEvent = await send(addr, {
|
||||
type: "submit_panel_event",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
process,
|
||||
widget_id: "debug-process",
|
||||
kind: "ButtonClicked",
|
||||
max_events: 10
|
||||
});
|
||||
assert.strictEqual(frozenEvent.type, "error");
|
||||
assert.match(frozenEvent.message, /program UI events are disabled/i);
|
||||
|
||||
const crossTenant = await send(addr, {
|
||||
type: "render_operator_panel",
|
||||
tenant: "other",
|
||||
project: "project",
|
||||
process,
|
||||
actor_user: "user",
|
||||
max_download_bytes: 1024 * 1024,
|
||||
stopped: false
|
||||
});
|
||||
assert.strictEqual(crossTenant.type, "error");
|
||||
assert.match(
|
||||
crossTenant.message,
|
||||
/scope|tenant|project|requires an active virtual process/i
|
||||
);
|
||||
assert(!crossTenant.message.includes(process));
|
||||
|
||||
const resumed = await send(addr, {
|
||||
type: "resume_debug_epoch",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
process,
|
||||
epoch: breakpointHit.hit_epoch,
|
||||
});
|
||||
assert.strictEqual(resumed.type, "debug_epoch");
|
||||
const cleanup = await send(addr, {
|
||||
type: "abort_process",
|
||||
tenant: "tenant",
|
||||
project: "project",
|
||||
actor_user: "user",
|
||||
process,
|
||||
});
|
||||
assert.strictEqual(cleanup.type, "process_aborted");
|
||||
} catch (error) {
|
||||
if (coordinatorStderr) {
|
||||
error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`;
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
worker?.child.kill("SIGTERM");
|
||||
coordinator.kill("SIGTERM");
|
||||
}
|
||||
|
||||
console.log("Operator panel smoke passed");
|
||||
})().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue