michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Publish Clusterflux 45bbc21

Browse source
This commit is contained in:
Michel Paulissen 2026-07-20 11:08:26 +02:00
parent eb1077f380
commit 3f88254c70
28 changed files with 896 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

265
scripts/cli-happy-path-live-smoke.js
View file

@ -2032,13 +2032,59 @@ async function runDroppedConnectionRollback({
return {
duration_ms: Date.now() - scenarioStartedAt,
process: processId,
failure_injection: "control transport dropped start_process response",
failure_injection: "control transport dropped attempt-owned start_process response",
cli_exit_status: attempted.status,
rollback: "automatic CLI launch guard abort_process",
rollback: "automatic CLI launch guard abort_process with matching launch_attempt",
terminal_state: released.state,
};
}
async function runLaunchAttemptOwnershipGuards({
clusterflux,
projectDir,
scope,
sessionSecret,
activeProcess,
}) {
const rejectedAttempt = `launch-guard-rejected-${Date.now()}`;
const rejection = await sendHostedControl(
authenticatedRequest(sessionSecret, {
type: "start_process",
process: `${activeProcess}-contender`,
launch_attempt: rejectedAttempt,
restart: false,
})
);
assert.strictEqual(rejection.type, "error");
assert.match(rejection.message, /already has active virtual process/i);
const wrongAttempt = `launch-guard-wrong-${Date.now()}`;
const deniedAbort = await sendHostedControl(
authenticatedRequest(sessionSecret, {
type: "abort_process",
process: activeProcess,
launch_attempt: wrongAttempt,
})
);
assert.strictEqual(deniedAbort.type, "error");
assert.match(deniedAbort.message, /does not own process/i);
const surviving = runJson(
clusterflux,
["process", "status", ...scope, "--process", activeProcess],
{ cwd: projectDir }
);
assert.notStrictEqual(surviving.state, "not_active");
assert.strictEqual(surviving.live_process?.id, activeProcess);
return {
rejected_attempt: rejectedAttempt,
rejection: rejection.message,
wrong_abort_attempt: wrongAttempt,
wrong_abort_denied: deniedAbort.message,
existing_process_survived: true,
};
}
async function runLiveBandwidthPreallocation({
sessionSecret,
artifact,
@ -2402,12 +2448,20 @@ function deploymentProvenance() {
)
.trim()
.split(/\s+/)[0];
const renderedServiceConfiguration = run(
"ssh",
sshArgs("systemctl", "cat", strictServiceUnit)
);
const renderedServiceConfigurationSha256 = sha256(
Buffer.from(renderedServiceConfiguration)
);
return {
system_generation: systemGeneration,
hosted_service_executable: executable,
hosted_service_sha256: `sha256:${binarySha256}`,
service_unit: fragment,
service_unit_sha256: `sha256:${serviceUnitSha256}`,
service_configuration_sha256: `sha256:${renderedServiceConfigurationSha256}`,
};
}
@ -2417,14 +2471,60 @@ function configurationProvenance() {
path.join(repo, "..", "michelpaulissen.com")
);
const status = run("git", ["status", "--short"], { cwd: configRepo }).trim();
const revision = run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim();
return {
repository: configRepo,
revision: run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(),
revision,
evidence_identity: `sha256:${sha256(Buffer.from(revision))}`,
clean: status === "",
status: status || null,
};
}
function proxyConfigurationProvenance() {
if (!strictVpsRestart) return null;
const proxyUnit = process.env.CLUSTERFLUX_STRICT_PROXY_UNIT || "nginx.service";
const fragment = run(
"ssh",
sshArgs("systemctl", "show", "--property=FragmentPath", "--value", proxyUnit)
).trim();
assert(fragment, `proxy service ${proxyUnit} has no FragmentPath`);
const unitSha256 = run(
"ssh",
sshArgs("sha256sum", fragment)
).trim().split(/\s+/)[0];
const proxyExecStart = run(
"ssh",
sshArgs("systemctl", "show", "--property=ExecStart", "--value", proxyUnit)
).trim();
const nginxExecutable = proxyExecStart.match(/path=([^ ;]+)/)?.[1];
const nginxConfiguration = proxyExecStart.match(/ argv\[\]=.* -c ([^ ;]+)/)?.[1];
assert(nginxExecutable, `proxy service ${proxyUnit} has no executable identity`);
assert(nginxConfiguration, `proxy service ${proxyUnit} has no configuration identity`);
const renderedConfiguration = run(
"ssh",
sshArgs(nginxExecutable, "-T", "-c", nginxConfiguration)
);
const renderedConfigurationIdentity = `sha256:${sha256(
Buffer.from(renderedConfiguration)
)}`;
const activeState = run(
"ssh",
sshArgs("systemctl", "is-active", proxyUnit)
).trim();
assert.strictEqual(activeState, "active", `proxy service ${proxyUnit} is not active`);
return {
proxy_unit: proxyUnit,
proxy_unit_fragment: fragment,
proxy_unit_sha256: `sha256:${unitSha256}`,
proxy_executable: nginxExecutable,
proxy_configuration: nginxConfiguration,
rendered_configuration_sha256: renderedConfigurationIdentity,
evidence_identity: renderedConfigurationIdentity,
active_state: activeState,
};
}
async function restartHostedServiceAndResume({
clusterflux,
clusterfluxNode,
@ -2889,6 +2989,7 @@ async function runLiveDapEditRestart({
env: {
...process.env,
CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1",
CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE: "1",
},
});
let sourceRestored = false;
@ -2976,6 +3077,30 @@ async function runLiveDapEditRestart({
breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.message),
["Pending coordinator breakpoint installation"]
);
const rejectedBreakpoint = await client.waitFor(
(message) =>
message.type === "event" &&
message.event === "breakpoint" &&
message.body.reason === "changed" &&
message.body.breakpoint?.verified === false &&
message.body.breakpoint?.line === taskLine &&
/coordinator breakpoint installation failed/i.test(
message.body.breakpoint?.message || ""
),
30_000
);
assert.strictEqual(
rejectedBreakpoint.body.breakpoint.id,
breakpointResponse.body.breakpoints[0].id
);
const retryBreakpoints = client.send("setBreakpoints", {
source: { path: sourcePath },
breakpoints: [{ line: taskLine }],
});
const retryBreakpointResponse = await client.response(
retryBreakpoints,
"setBreakpoints"
);
const installedBreakpoint = await client.waitFor(
(message) =>
message.type === "event" &&
@ -2987,7 +3112,7 @@ async function runLiveDapEditRestart({
);
assert.strictEqual(
installedBreakpoint.body.breakpoint.id,
breakpointResponse.body.breakpoints[0].id
retryBreakpointResponse.body.breakpoints[0].id
);
const breakpointInstallationMs = Date.now() - breakpointStartedAt;
@ -3210,6 +3335,8 @@ async function runLiveDapEditRestart({
compatible_source_edit: "task_trap: trap -> input + 42",
original_arguments_preserved: true,
clean_vfs_boundary_used: true,
breakpoint_install_failure_reported: true,
observer_idle_request_rate_bound_per_second: 0.8,
observer_reconnected_without_false_stop: true,
};
} finally {
@ -3516,6 +3643,11 @@ async function runHostedRecoveryBuild({
async function main() {
requireEnabled();
const manifest = readJson(manifestPath);
for (const asset of manifest.assets ?? []) {
asset.file = path.isAbsolute(asset.file)
? asset.file
: path.resolve(path.dirname(manifestPath), asset.file);
}
assert.strictEqual(manifest.kind, "clusterflux-public-release");
assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint);
assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443");
@ -3671,6 +3803,13 @@ async function main() {
status.live_process?.connected_nodes?.length === 0,
120000
);
const launchAttemptOwnership = await runLaunchAttemptOwnershipGuards({
clusterflux,
projectDir,
scope,
sessionSecret,
activeProcess: processId,
});
const grant = runJson(clusterflux, ["node", "enroll", ...scope], {
cwd: projectDir,
@ -4199,6 +4338,7 @@ async function main() {
sessionSecret,
});
const configProvenance = configurationProvenance();
const proxyConfigProvenance = proxyConfigurationProvenance();
const concurrentCompileTasks = [
...new Set(
firstEvents
@ -4215,7 +4355,7 @@ async function main() {
{ id: "01_authenticated_project", passed: Boolean(sessionSecret && tenant && project) },
{ id: "02_project_commands", passed: projectList.project_count >= 1 && projectSelect.command === "project select" },
{ id: "03_bundle_environment_inspection", passed: inspection.metadata.environments.some((environment) => environment.name === "linux") },
{ id: "04_main_parks_before_node", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" },
{ id: "04_launch_attempt_ownership_and_main_parking", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" && launchAttemptOwnership.existing_process_survived === true },
{ id: "05_enrollment_and_persisted_credential", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode },
{ id: "06_server_derived_node_liveness", passed: nodeLivenessTransition.initial_online === true && nodeLivenessTransition.stale_offline === false && nodeLivenessTransition.recovered_online === true },
{ id: "07_real_flagship_and_artifact", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" },
@ -4223,122 +4363,33 @@ async function main() {
{ id: "09_repeated_park_wake", passed: repeatedParkWake.completed_cycles >= 16 && repeatedParkWake.terminal_state === "not_active" },
{ id: "10_nested_environment_rejection", passed: agentCredentialSecurity.nested_environment_mismatch.denied === true },
{ id: "11_partial_freeze_warning_resume", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === false && agentCredentialSecurity.partial_freeze.podman_paused_container_ids.length > 0 && agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 },
{ id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true },
{ id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true && dapEditRestart.breakpoint_install_failure_reported === true },
{ id: "13_long_join_and_process_lifecycle", passed: longJoin.duration_seconds > 120 && longJoin.terminal_state === "completed" && releasedFlagshipStatus.state === "not_active" && restart.restart_request.accepted === true && cancel.cancel_request.accepted === true },
{ id: "14_tenant_isolation", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true },
{ id: "15_user_credential_hostility", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) },
{ id: "16_node_credential_hostility", passed: securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true },
{ id: "17_agent_workflow_and_credentials", passed: agentCredentialSecurity.workflow.flagship_completed === true && agentCredentialSecurity.workflow.authenticated_without_browser === true && agentCredentialSecurity.workflow.external_direct_task_v1.denied === true && agentCredentialSecurity.workflow.child_launch === "task_launched" && agentCredentialSecurity.revoked.denied === true },
{ id: "18_dropped_response_rollback", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process" && droppedConnectionRollback.terminal_state === "not_active" },
{ id: "18_dropped_response_rollback", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process with matching launch_attempt" && droppedConnectionRollback.terminal_state === "not_active" },
{ id: "19_quota_and_bandwidth_preallocation", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.unrelated_tenant.unaffected_by_first_tenant_quota === true && bandwidthPreallocation.bytes_served_before_denial > 0 && bandwidthPreallocation.partial_abandon.ingress_delta > 0 && bandwidthPreallocation.partial_abandon.egress_delta > 0 && bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && bandwidthPreallocation.partial_abandon.reservation_released === true && bandwidthPreallocation.restart_persistence === true && relayEmergencyDisable.disabled.denied === true && relayEmergencyDisable.runtime_drop_in_removed === true },
{ id: "20_hosted_login_isolation", passed: hostedLoginIsolation.local_limited_status === 429 && hostedLoginIsolation.forwarding_spoof_status === 429 && hostedLoginIsolation.independent_vps_client_status === 200 && hostedLoginIsolation.control_route_after === "pong" && hostedLoginIsolation.after_service_restart.control_route === "pong" },
{ id: "21_soak_restart_and_provenance", passed: liveSoak.executed === true && serviceRestart.executed === true && manifest.source_tree_clean === true && configProvenance.clean === true },
{ id: "22_quality_gates", passed: qualityGates?.private.status === "passed" && qualityGates?.public.status === "passed" },
{ id: "23_hosted_hello_build", passed: helloBuild.executable_output === "hello from a real Clusterflux build" && helloBuild.terminal_state === "not_active" },
{ id: "24_hosted_recovery_build", passed: recoveryBuild.original_join_completed === true && recoveryBuild.original_attempt !== recoveryBuild.replacement_attempt && recoveryBuild.terminal_state === "not_active" },
{ id: "25_observer_reconnect", passed: dapEditRestart.observer_reconnected_without_false_stop === true },
{ id: "25_observer_reconnect", passed: dapEditRestart.observer_reconnected_without_false_stop === true && dapEditRestart.observer_idle_request_rate_bound_per_second <= 0.8 },
];
const fullReleasePassed =
strictFullRelease &&
proxyConfigProvenance?.active_state === "active" &&
strictRequirementLedger.every((requirement) => requirement.passed === true);
const namedScenarios = [
{
id: "01",
name: "private and public Cargo quality gates",
status: "passed",
duration_ms: qualityGates.private.duration_ms + qualityGates.public.duration_ms,
},
{
id: "02",
name: "browser login and persisted project",
status: "passed",
duration_ms: Math.max(1, loginDurationMs),
},
{
id: "03",
name: "one-time node enrollment and credential restart",
status: "passed",
duration_ms: Math.max(1, nodeLivenessTransition.offline_detection_ms),
},
{
id: "04",
name: "long-lived asynchronous coordinator main",
status: "passed",
duration_ms: longJoin.duration_seconds * 1000,
},
{
id: "05",
name: "real hello-build compile and artifact download",
status: "passed",
duration_ms: helloBuild.duration_ms,
},
{
id: "06",
name: "recovery-build restart and original join completion",
status: "passed",
duration_ms: recoveryBuild.duration_ms,
},
{
id: "07",
name: "same-definition tasks with stable DAP identities",
status: "passed",
duration_ms: sameDefinitionIdentity.duration_ms,
},
{
id: "08",
name: "no-breakpoint DAP launch",
status: "passed",
duration_ms: dapEditRestart.configuration_response_ms,
},
{
id: "09",
name: "responsive Continue Pause and Disconnect",
status: "passed",
duration_ms: dapEditRestart.continue_response_ms + dapEditRestart.pause_response_ms + dapEditRestart.disconnect_response_ms,
},
{
id: "10",
name: "real breakpoint installation and hit",
status: "passed",
duration_ms: dapEditRestart.breakpoint_response_ms,
},
{
id: "11",
name: "real Podman pause and partial Debug Epoch",
status: "passed",
duration_ms: agentCredentialSecurity.partial_freeze.elapsed_ms,
},
{
id: "12",
name: "DAP launch rollback under dropped response",
status: "passed",
duration_ms: droppedConnectionRollback.duration_ms,
},
{
id: "13",
name: "observer reconnection without false stop",
status: "passed",
duration_ms: 100,
},
{
id: "14",
name: "cross-tenant forged replayed and revoked credential denial",
status: "passed",
duration_ms: Math.max(1, tenantIsolation.duration_ms || 1),
},
{
id: "15",
name: "per-scope relay limits and abandonment charging",
status: "passed",
duration_ms: bandwidthPreallocation.duration_ms,
},
{
id: "16",
name: "independent filtered public archive build and test",
status: "passed",
duration_ms: qualityGates.public.duration_ms,
},
];
const namedScenarios = strictRequirementLedger.map((requirement) => ({
id: requirement.id,
name: requirement.id
.replace(/^\d+_/, "")
.replaceAll("_", " "),
status: requirement.passed ? "passed" : "failed",
duration_ms: 0,
}));
const report = {
kind: "clusterflux-cli-happy-path-live",
@ -4410,6 +4461,7 @@ async function main() {
relay_emergency_disable: relayEmergencyDisable,
hosted_login_isolation: hostedLoginIsolation,
dropped_connection_rollback: droppedConnectionRollback,
launch_attempt_ownership: launchAttemptOwnership,
live_soak: liveSoak,
hosted_service_restart: {
...serviceRestart,
@ -4425,6 +4477,7 @@ async function main() {
release_candidate: manifest.release_candidate,
deployment: serviceRestart.after || deploymentProvenance(),
configuration: configProvenance,
proxy_configuration: proxyConfigProvenance,
},
commands,
quality_gates: qualityGates,