Public release release-ea887c8f56cd
Source commit: ea887c8f56cd53985a1179b13e5f1b85c485f584 Public tree identity: sha256:b96a97aacdbc8fd4fc2ea20d0e1450280d46db3f24aabe1475e5fbe20e05f255
This commit is contained in:
Clusterflux release
parent
9223c54939
commit
2a0f7ded04
37 changed files with 3145 additions and 628 deletions
|
|
@ -81,17 +81,12 @@ function requireEnabled() {
|
|||
const hasSecondTenantSession = Boolean(
|
||||
process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE
|
||||
);
|
||||
const hasSingleAccountIsolationTarget = Boolean(
|
||||
process.env.CLUSTERFLUX_ISOLATION_TARGET_TENANT &&
|
||||
process.env.CLUSTERFLUX_ISOLATION_TARGET_PROJECT
|
||||
);
|
||||
if (
|
||||
strictFullRelease &&
|
||||
!hasSecondTenantSession &&
|
||||
!hasSingleAccountIsolationTarget
|
||||
!hasSecondTenantSession
|
||||
) {
|
||||
throw new Error(
|
||||
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires either CLUSTERFLUX_SECOND_TENANT_SESSION_FILE or both CLUSTERFLUX_ISOLATION_TARGET_TENANT and CLUSTERFLUX_ISOLATION_TARGET_PROJECT"
|
||||
"CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE so the compiled two-tenant Node and artifact collision can run"
|
||||
);
|
||||
}
|
||||
if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) {
|
||||
|
|
@ -500,8 +495,12 @@ async function runMalformedIdentifierSuite({
|
|||
send: async (value) =>
|
||||
sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
node: value,
|
||||
node_signature: signedNodeHeartbeat(
|
||||
tenant,
|
||||
project,
|
||||
value,
|
||||
securityNode.identity,
|
||||
{
|
||||
|
|
@ -803,8 +802,12 @@ async function runMalformedIdentifierSuite({
|
|||
send: async (value) => {
|
||||
const request = {
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
node: securityNode.node,
|
||||
node_signature: signedNodeHeartbeat(
|
||||
tenant,
|
||||
project,
|
||||
securityNode.node,
|
||||
securityNode.identity,
|
||||
{ nonce: value }
|
||||
|
|
@ -1616,8 +1619,10 @@ async function prepareLiveNodeCredentialSecurity({
|
|||
|
||||
const heartbeat = {
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
node,
|
||||
node_signature: signedNodeHeartbeat(node, identity, {
|
||||
node_signature: signedNodeHeartbeat(tenant, project, node, identity, {
|
||||
nonce: `strict-valid-${suffix}`,
|
||||
}),
|
||||
};
|
||||
|
|
@ -1632,8 +1637,10 @@ async function prepareLiveNodeCredentialSecurity({
|
|||
const expired = assertDenied(
|
||||
await sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
node,
|
||||
node_signature: signedNodeHeartbeat(node, identity, {
|
||||
node_signature: signedNodeHeartbeat(tenant, project, node, identity, {
|
||||
nonce: `strict-expired-${suffix}`,
|
||||
issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600,
|
||||
}),
|
||||
|
|
@ -1646,8 +1653,10 @@ async function prepareLiveNodeCredentialSecurity({
|
|||
const forged = assertDenied(
|
||||
await sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
node,
|
||||
node_signature: signedNodeHeartbeat(node, forgedIdentity, {
|
||||
node_signature: signedNodeHeartbeat(tenant, project, node, forgedIdentity, {
|
||||
nonce: `strict-forged-${suffix}`,
|
||||
}),
|
||||
}),
|
||||
|
|
@ -2556,6 +2565,12 @@ async function runLiveAgentCredentialSecurity({
|
|||
}
|
||||
|
||||
async function runSecondTenantIsolation({
|
||||
clusterflux,
|
||||
clusterfluxNode,
|
||||
firstScope,
|
||||
firstHelloBuild,
|
||||
firstHelloProjectDir,
|
||||
workRoot,
|
||||
firstSessionSecret,
|
||||
firstTenant,
|
||||
firstProject,
|
||||
|
|
@ -2597,14 +2612,17 @@ async function runSecondTenantIsolation({
|
|||
"reused isolation evidence must target the exact public binaries"
|
||||
);
|
||||
return {
|
||||
...isolation,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
reused_from_immutable_evidence: {
|
||||
report_sha256: sha256(evidenceBytes),
|
||||
source_commit: previous.source_commit,
|
||||
hosted_service_sha256: currentDeployment.hosted_service_sha256,
|
||||
public_binary_digests: manifest.binary_digests,
|
||||
evidence: {
|
||||
...isolation,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
reused_from_immutable_evidence: {
|
||||
report_sha256: sha256(evidenceBytes),
|
||||
source_commit: previous.source_commit,
|
||||
hosted_service_sha256: currentDeployment.hosted_service_sha256,
|
||||
public_binary_digests: manifest.binary_digests,
|
||||
},
|
||||
},
|
||||
runtime: null,
|
||||
};
|
||||
}
|
||||
if (!file && targetTenant && targetProject) {
|
||||
|
|
@ -2702,36 +2720,56 @@ async function runSecondTenantIsolation({
|
|||
"single-account foreign process control"
|
||||
);
|
||||
return {
|
||||
executed: true,
|
||||
mode: "single_authenticated_account_foreign_project",
|
||||
second_tenant: targetTenant,
|
||||
target_project: targetProject,
|
||||
project,
|
||||
process_hidden: true,
|
||||
node_hidden: true,
|
||||
tasks_and_logs: tasksAndLogs,
|
||||
debug: {
|
||||
denied: true,
|
||||
reason: debugResponse.authorization.reason,
|
||||
audited: true,
|
||||
charged_debug_read_bytes: debugResponse.charged_debug_read_bytes,
|
||||
evidence: {
|
||||
executed: true,
|
||||
mode: "single_authenticated_account_foreign_project",
|
||||
second_tenant: targetTenant,
|
||||
target_project: targetProject,
|
||||
project,
|
||||
process_hidden: true,
|
||||
node_hidden: true,
|
||||
tasks_and_logs: tasksAndLogs,
|
||||
debug: {
|
||||
denied: true,
|
||||
reason: debugResponse.authorization.reason,
|
||||
audited: true,
|
||||
charged_debug_read_bytes: debugResponse.charged_debug_read_bytes,
|
||||
},
|
||||
artifact_and_download: artifact,
|
||||
process_control: control,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
},
|
||||
artifact_and_download: artifact,
|
||||
process_control: control,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
runtime: null,
|
||||
};
|
||||
}
|
||||
if (!file) return { executed: false, reason: "second tenant session not supplied" };
|
||||
const second = readJson(path.resolve(file));
|
||||
if (!file) {
|
||||
return {
|
||||
evidence: {
|
||||
executed: false,
|
||||
reason: "second tenant session not supplied",
|
||||
duration_ms: Math.max(1, Date.now() - startedAt),
|
||||
},
|
||||
runtime: null,
|
||||
};
|
||||
}
|
||||
const secondSessionPath = path.resolve(file);
|
||||
const second = readJson(secondSessionPath);
|
||||
assert.strictEqual(second.coordinator, serviceEndpoint);
|
||||
assert(second.session_secret, "second tenant session omitted its session secret");
|
||||
const secondSessionSecret =
|
||||
second.cli_session_secret || second.session_secret;
|
||||
assert(secondSessionSecret, "second tenant session omitted its session secret");
|
||||
assert.notStrictEqual(
|
||||
second.tenant,
|
||||
firstTenant,
|
||||
"isolation proof requires a genuinely distinct hosted tenant"
|
||||
);
|
||||
assert.notStrictEqual(
|
||||
second.project,
|
||||
firstProject,
|
||||
"isolation proof requires a genuinely distinct hosted project"
|
||||
);
|
||||
const call = (request) =>
|
||||
sendHostedControl(authenticatedRequest(second.session_secret, request));
|
||||
sendHostedControl(authenticatedRequest(secondSessionSecret, request));
|
||||
|
||||
const project = assertDenied(
|
||||
await call({ type: "select_project", project: firstProject }),
|
||||
|
|
@ -2778,7 +2816,7 @@ async function runSecondTenantIsolation({
|
|||
max_bytes: 1024,
|
||||
ttl_seconds: 60,
|
||||
}),
|
||||
/outside|scope|tenant|project|not found|unavailable/i,
|
||||
/outside|scope|tenant|project|not found|does not exist|unavailable/i,
|
||||
"cross-tenant artifact download"
|
||||
);
|
||||
const control = assertDenied(
|
||||
|
|
@ -2786,17 +2824,437 @@ async function runSecondTenantIsolation({
|
|||
/outside|scope|tenant|project|not active|requires an active|unknown/i,
|
||||
"cross-tenant process control"
|
||||
);
|
||||
|
||||
const secondCheckout = path.join(workRoot, "second-tenant-public-repo");
|
||||
stagePublicCheckout(manifest, secondCheckout);
|
||||
const secondProjectDir = path.join(
|
||||
secondCheckout,
|
||||
"examples",
|
||||
"hello-build"
|
||||
);
|
||||
const secondControlDir = path.join(secondProjectDir, ".clusterflux");
|
||||
ensureDir(secondControlDir);
|
||||
const secondProjectPath = path.join(
|
||||
path.dirname(secondSessionPath),
|
||||
"project.json"
|
||||
);
|
||||
assert(
|
||||
fs.existsSync(secondProjectPath),
|
||||
`second tenant session is missing adjacent project.json: ${secondProjectPath}`
|
||||
);
|
||||
fs.copyFileSync(
|
||||
secondSessionPath,
|
||||
path.join(secondControlDir, "session.json")
|
||||
);
|
||||
fs.copyFileSync(
|
||||
secondProjectPath,
|
||||
path.join(secondControlDir, "project.json")
|
||||
);
|
||||
fs.chmodSync(path.join(secondControlDir, "session.json"), 0o600);
|
||||
fs.chmodSync(path.join(secondControlDir, "project.json"), 0o644);
|
||||
const secondScope = [
|
||||
"--coordinator",
|
||||
serviceEndpoint,
|
||||
"--tenant",
|
||||
second.tenant,
|
||||
"--project-id",
|
||||
second.project,
|
||||
"--user",
|
||||
second.user,
|
||||
"--json",
|
||||
];
|
||||
const secondProjectList = runJson(
|
||||
clusterflux,
|
||||
["project", "list", ...secondScope],
|
||||
{ cwd: secondProjectDir }
|
||||
);
|
||||
assert(secondProjectList.project_count >= 1);
|
||||
const secondProjectSelect = runJson(
|
||||
clusterflux,
|
||||
["project", "select", ...secondScope, second.project],
|
||||
{ cwd: secondProjectDir }
|
||||
);
|
||||
assert.strictEqual(secondProjectSelect.command, "project select");
|
||||
|
||||
const collisionStartedAt = Date.now();
|
||||
const secondGrant = runJson(
|
||||
clusterflux,
|
||||
["node", "enroll", ...secondScope],
|
||||
{ cwd: secondProjectDir }
|
||||
);
|
||||
const secondAttach = runJson(
|
||||
clusterflux,
|
||||
[
|
||||
"node",
|
||||
"attach",
|
||||
"--coordinator",
|
||||
serviceEndpoint,
|
||||
"--tenant",
|
||||
second.tenant,
|
||||
"--project-id",
|
||||
second.project,
|
||||
"--node",
|
||||
firstNode,
|
||||
"--enrollment-grant",
|
||||
secondGrant.enrollment_grant.grant,
|
||||
"--json",
|
||||
],
|
||||
{ cwd: secondProjectDir }
|
||||
);
|
||||
assert.strictEqual(secondAttach.boundary.used_enrollment_exchange, true);
|
||||
const secondStored = readNodeCredential(secondProjectDir, firstNode);
|
||||
const secondCredentialDigest = sha256(
|
||||
fs.readFileSync(secondStored.absolute)
|
||||
);
|
||||
const secondIdentity = nodeIdentityFromPrivateKey(
|
||||
secondStored.credential.private_key
|
||||
);
|
||||
const secondWorkerArgs = [
|
||||
"--coordinator",
|
||||
serviceEndpoint,
|
||||
"--tenant",
|
||||
second.tenant,
|
||||
"--project-id",
|
||||
second.project,
|
||||
"--node",
|
||||
firstNode,
|
||||
"--worker",
|
||||
"--project-root",
|
||||
secondProjectDir,
|
||||
"--assignment-poll-ms",
|
||||
"500",
|
||||
"--emit-ready",
|
||||
];
|
||||
const firstHelloWorker = spawnJsonLines(
|
||||
clusterfluxNode,
|
||||
[
|
||||
"--coordinator",
|
||||
serviceEndpoint,
|
||||
"--tenant",
|
||||
firstTenant,
|
||||
"--project-id",
|
||||
firstProject,
|
||||
"--node",
|
||||
firstNode,
|
||||
"--worker",
|
||||
"--project-root",
|
||||
firstHelloProjectDir,
|
||||
"--assignment-poll-ms",
|
||||
"500",
|
||||
"--emit-ready",
|
||||
],
|
||||
{
|
||||
cwd: firstHelloProjectDir,
|
||||
env: { ...process.env },
|
||||
}
|
||||
);
|
||||
const firstHelloReady = await firstHelloWorker.waitFor(
|
||||
(value) => value.node_status === "ready",
|
||||
"first-tenant hello-build artifact worker ready for collision proof"
|
||||
);
|
||||
assert.strictEqual(firstHelloReady.node, firstNode);
|
||||
const firstCollisionHelloBuild = await runHostedHelloBuild({
|
||||
clusterflux,
|
||||
projectDir: firstHelloProjectDir,
|
||||
scope: firstScope,
|
||||
outputFile: path.join(workRoot, "hello-clusterflux-first-tenant-collision"),
|
||||
});
|
||||
assert.strictEqual(
|
||||
firstCollisionHelloBuild.artifact,
|
||||
firstHelloBuild.artifact,
|
||||
"repeated deterministic first-tenant hello-build changed its ArtifactId"
|
||||
);
|
||||
assert.strictEqual(firstCollisionHelloBuild.digest, firstHelloBuild.digest);
|
||||
assert.strictEqual(
|
||||
firstCollisionHelloBuild.executable_output,
|
||||
firstHelloBuild.executable_output
|
||||
);
|
||||
const secondWorker = spawnJsonLines(clusterfluxNode, secondWorkerArgs, {
|
||||
cwd: secondProjectDir,
|
||||
env: { ...process.env },
|
||||
});
|
||||
let secondHelloBuild;
|
||||
try {
|
||||
const secondReady = await secondWorker.waitFor(
|
||||
(value) => value.node_status === "ready",
|
||||
"same-name second-tenant worker ready"
|
||||
);
|
||||
assert.strictEqual(secondReady.node, firstNode);
|
||||
secondHelloBuild = await runHostedHelloBuild({
|
||||
clusterflux,
|
||||
projectDir: secondProjectDir,
|
||||
scope: secondScope,
|
||||
outputFile: path.join(workRoot, "hello-clusterflux-second-tenant"),
|
||||
});
|
||||
} finally {
|
||||
await stopChild(secondWorker.child);
|
||||
}
|
||||
assert.strictEqual(
|
||||
secondHelloBuild.artifact,
|
||||
firstCollisionHelloBuild.artifact,
|
||||
"deterministic two-tenant hello-build did not collide on ArtifactId"
|
||||
);
|
||||
assert.strictEqual(
|
||||
secondHelloBuild.digest,
|
||||
firstCollisionHelloBuild.digest,
|
||||
"deterministic two-tenant hello-build did not produce identical bytes"
|
||||
);
|
||||
assert.strictEqual(
|
||||
secondHelloBuild.executable_output,
|
||||
firstCollisionHelloBuild.executable_output
|
||||
);
|
||||
|
||||
const firstArtifactsAfterCollision = runJson(
|
||||
clusterflux,
|
||||
[
|
||||
"artifact",
|
||||
"list",
|
||||
...firstScope,
|
||||
"--process",
|
||||
firstCollisionHelloBuild.process,
|
||||
],
|
||||
{ cwd: firstHelloProjectDir }
|
||||
);
|
||||
assert(
|
||||
firstArtifactsAfterCollision.artifacts.some(
|
||||
(candidate) =>
|
||||
candidate.artifact === firstCollisionHelloBuild.artifact &&
|
||||
candidate.digest === firstCollisionHelloBuild.digest
|
||||
),
|
||||
"the second tenant's same-ID artifact masked the first tenant's metadata"
|
||||
);
|
||||
const secondCollisionLink = await call({
|
||||
type: "create_artifact_download_link",
|
||||
artifact: secondHelloBuild.artifact,
|
||||
max_bytes: secondHelloBuild.downloaded_bytes,
|
||||
ttl_seconds: 60,
|
||||
});
|
||||
assert.strictEqual(
|
||||
secondCollisionLink.type,
|
||||
"artifact_download_link",
|
||||
JSON.stringify(secondCollisionLink)
|
||||
);
|
||||
const secondCollisionLinkRevoked = await call({
|
||||
type: "revoke_artifact_download_link",
|
||||
artifact: secondHelloBuild.artifact,
|
||||
token_digest: secondCollisionLink.link.scoped_token_digest,
|
||||
});
|
||||
assert.strictEqual(
|
||||
secondCollisionLinkRevoked.type,
|
||||
"artifact_download_link_revoked",
|
||||
JSON.stringify(secondCollisionLinkRevoked)
|
||||
);
|
||||
const firstDownloadAfterSecondLinkRevocation = runJson(
|
||||
clusterflux,
|
||||
[
|
||||
"artifact",
|
||||
"download",
|
||||
...firstScope,
|
||||
firstCollisionHelloBuild.artifact,
|
||||
"--to",
|
||||
path.join(workRoot, "hello-clusterflux-first-after-second-link-revoke"),
|
||||
],
|
||||
{ cwd: firstHelloProjectDir }
|
||||
);
|
||||
assert.strictEqual(
|
||||
firstDownloadAfterSecondLinkRevocation.local_download.verified_digest,
|
||||
firstCollisionHelloBuild.digest
|
||||
);
|
||||
await stopChild(firstHelloWorker.child);
|
||||
|
||||
return {
|
||||
executed: true,
|
||||
second_tenant: second.tenant,
|
||||
evidence: {
|
||||
executed: true,
|
||||
second_tenant: second.tenant,
|
||||
second_project: second.project,
|
||||
project,
|
||||
process_hidden: true,
|
||||
node_hidden: true,
|
||||
tasks_and_logs: tasksAndLogs,
|
||||
debug,
|
||||
artifact_and_download: artifact,
|
||||
process_control: control,
|
||||
scoped_node_and_artifact_collision: {
|
||||
request: {
|
||||
node: firstNode,
|
||||
first_scope: {
|
||||
tenant: firstTenant,
|
||||
project: firstProject,
|
||||
},
|
||||
second_scope: {
|
||||
tenant: second.tenant,
|
||||
project: second.project,
|
||||
},
|
||||
example: "hello-build",
|
||||
},
|
||||
expected:
|
||||
"same-name Nodes coexist and deterministic hello-build artifacts share an ArtifactId without metadata, link, or download interference",
|
||||
observed: {
|
||||
first_process: firstCollisionHelloBuild.process,
|
||||
second_process: secondHelloBuild.process,
|
||||
same_artifact_id: secondHelloBuild.artifact,
|
||||
first_digest: firstCollisionHelloBuild.digest,
|
||||
second_digest: secondHelloBuild.digest,
|
||||
first_downloaded_bytes: firstCollisionHelloBuild.downloaded_bytes,
|
||||
second_downloaded_bytes: secondHelloBuild.downloaded_bytes,
|
||||
exact_executable_output: secondHelloBuild.executable_output,
|
||||
both_metadata_records_visible_to_owner: true,
|
||||
cross_tenant_unique_artifact_denied: artifact.denied,
|
||||
second_link_revoked: true,
|
||||
first_download_survived_second_link_revocation: true,
|
||||
},
|
||||
duration_ms: Math.max(1, Date.now() - collisionStartedAt),
|
||||
},
|
||||
duration_ms: Math.max(1, Date.now() - startedAt),
|
||||
},
|
||||
runtime: {
|
||||
node: firstNode,
|
||||
tenant: second.tenant,
|
||||
project: second.project,
|
||||
projectDir: secondProjectDir,
|
||||
scope: secondScope,
|
||||
workerArgs: secondWorkerArgs,
|
||||
credentialPath: secondStored.absolute,
|
||||
credentialDigest: secondCredentialDigest,
|
||||
identity: secondIdentity,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runLiveSignedHostileArtifactPath({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
scope,
|
||||
tenant,
|
||||
project,
|
||||
suffix,
|
||||
securityNode,
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
const runReport = runJson(
|
||||
clusterflux,
|
||||
["run", "park-wake", "--project", ".", "--json"],
|
||||
{ cwd: projectDir }
|
||||
);
|
||||
assert.strictEqual(runReport.status, "main_launched");
|
||||
const process = runReport.process;
|
||||
await waitForCli(
|
||||
"hostile-path probe process to become active",
|
||||
() =>
|
||||
runJson(
|
||||
clusterflux,
|
||||
["process", "status", ...scope, "--process", process],
|
||||
{ cwd: projectDir }
|
||||
),
|
||||
(status) => status.state !== "not_active",
|
||||
120_000
|
||||
);
|
||||
|
||||
const invalidStartedAt = Date.now();
|
||||
const invalid = assertDenied(
|
||||
await sendHostedControl(
|
||||
signedNodeRequest(
|
||||
securityNode.node,
|
||||
securityNode.identity,
|
||||
"report_vfs_metadata",
|
||||
{
|
||||
type: "report_vfs_metadata",
|
||||
tenant,
|
||||
project,
|
||||
process,
|
||||
node: securityNode.node,
|
||||
task: `hostile-path-${suffix}`,
|
||||
artifact_path: "/vfs/artifacts/bad artifact!",
|
||||
artifact_digest: sha256(Buffer.from("hostile-path-invalid")),
|
||||
artifact_size_bytes: 20,
|
||||
large_bytes_uploaded: false,
|
||||
},
|
||||
{ nonce: `strict-hostile-path-invalid-${suffix}-${Date.now()}` }
|
||||
)
|
||||
),
|
||||
/invalid VFS artifact path|invalid artifact path|ArtifactId is invalid/i,
|
||||
"correctly signed hostile artifact path"
|
||||
);
|
||||
const invalidDurationMs = Math.max(1, Date.now() - invalidStartedAt);
|
||||
|
||||
const validStartedAt = Date.now();
|
||||
const validArtifact = `strict-hostile-followup-${suffix}`;
|
||||
const valid = await sendHostedControl(
|
||||
signedNodeRequest(
|
||||
securityNode.node,
|
||||
securityNode.identity,
|
||||
"report_vfs_metadata",
|
||||
{
|
||||
type: "report_vfs_metadata",
|
||||
tenant,
|
||||
project,
|
||||
process,
|
||||
node: securityNode.node,
|
||||
task: `hostile-path-${suffix}`,
|
||||
artifact_path: `/vfs/artifacts/${validArtifact}`,
|
||||
artifact_digest: sha256(Buffer.from("hostile-path-valid")),
|
||||
artifact_size_bytes: 18,
|
||||
large_bytes_uploaded: false,
|
||||
},
|
||||
{ nonce: `strict-hostile-path-valid-${suffix}-${Date.now()}` }
|
||||
)
|
||||
);
|
||||
assert.strictEqual(valid.type, "vfs_metadata_recorded", JSON.stringify(valid));
|
||||
const healthyHeartbeat = await sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant,
|
||||
project,
|
||||
process_hidden: true,
|
||||
node_hidden: true,
|
||||
tasks_and_logs: tasksAndLogs,
|
||||
debug,
|
||||
artifact_and_download: artifact,
|
||||
process_control: control,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
node: securityNode.node,
|
||||
node_signature: signedNodeHeartbeat(
|
||||
tenant,
|
||||
project,
|
||||
securityNode.node,
|
||||
securityNode.identity,
|
||||
{ nonce: `strict-hostile-path-health-${suffix}-${Date.now()}` }
|
||||
),
|
||||
});
|
||||
assert.strictEqual(
|
||||
healthyHeartbeat.type,
|
||||
"node_heartbeat",
|
||||
JSON.stringify(healthyHeartbeat)
|
||||
);
|
||||
const validDurationMs = Math.max(1, Date.now() - validStartedAt);
|
||||
const aborted = runJson(
|
||||
clusterflux,
|
||||
["process", "abort", ...scope, "--process", process, "--yes"],
|
||||
{ cwd: projectDir }
|
||||
);
|
||||
assert.strictEqual(aborted.abort_request.accepted, true);
|
||||
await waitForCli(
|
||||
"hostile-path probe process cleanup",
|
||||
() =>
|
||||
runJson(
|
||||
clusterflux,
|
||||
["process", "status", ...scope, "--process", process],
|
||||
{ cwd: projectDir }
|
||||
),
|
||||
(status) => status.state === "not_active",
|
||||
120_000
|
||||
);
|
||||
return {
|
||||
request: {
|
||||
principal: "enrolled signed Node",
|
||||
variant: "report_vfs_metadata",
|
||||
process,
|
||||
malformed_path: "/vfs/artifacts/bad artifact!",
|
||||
},
|
||||
expected:
|
||||
"structured InvalidArtifactPath rejection followed immediately by valid signed metadata and heartbeat on the same service instance",
|
||||
observed: {
|
||||
rejection: invalid,
|
||||
valid_metadata_response: valid.type,
|
||||
valid_artifact: validArtifact,
|
||||
health_response: healthyHeartbeat.type,
|
||||
process_cleanup: "not_active",
|
||||
},
|
||||
invalid_duration_ms: invalidDurationMs,
|
||||
valid_followup_duration_ms: validDurationMs,
|
||||
duration_ms: Math.max(1, Date.now() - startedAt),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -2960,8 +3418,12 @@ async function revokeSecurityNode({
|
|||
const denied = assertDenied(
|
||||
await sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant: securityNode.capability_body.tenant,
|
||||
project: securityNode.capability_body.project,
|
||||
node: securityNode.node,
|
||||
node_signature: signedNodeHeartbeat(
|
||||
securityNode.capability_body.tenant,
|
||||
securityNode.capability_body.project,
|
||||
securityNode.node,
|
||||
securityNode.identity,
|
||||
{ nonce: `strict-node-revoked-${Date.now()}` }
|
||||
|
|
@ -3742,6 +4204,7 @@ async function restartHostedServiceAndResume({
|
|||
workerNode,
|
||||
credentialPath,
|
||||
credentialDigest,
|
||||
scopedCollisionRuntime,
|
||||
}) {
|
||||
if (!strictVpsRestart) {
|
||||
return {
|
||||
|
|
@ -3841,6 +4304,79 @@ async function restartHostedServiceAndResume({
|
|||
assert.strictEqual(ready.node, workerNode);
|
||||
assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest);
|
||||
|
||||
let scopedNodeCollisionAfterRestart = null;
|
||||
if (scopedCollisionRuntime) {
|
||||
assert.strictEqual(scopedCollisionRuntime.node, workerNode);
|
||||
assert.notStrictEqual(
|
||||
scopedCollisionRuntime.tenant,
|
||||
readJson(path.join(projectDir, ".clusterflux", "session.json")).tenant
|
||||
);
|
||||
assert.strictEqual(
|
||||
sha256(fs.readFileSync(scopedCollisionRuntime.credentialPath)),
|
||||
scopedCollisionRuntime.credentialDigest
|
||||
);
|
||||
const secondWorker = spawnJsonLines(
|
||||
clusterfluxNode,
|
||||
scopedCollisionRuntime.workerArgs,
|
||||
{
|
||||
cwd: scopedCollisionRuntime.projectDir,
|
||||
env: { ...process.env },
|
||||
}
|
||||
);
|
||||
try {
|
||||
const secondReady = await secondWorker.waitFor(
|
||||
(value) => value.node_status === "ready",
|
||||
"same-name second-tenant worker ready after hosted service restart"
|
||||
);
|
||||
assert.strictEqual(secondReady.node, workerNode);
|
||||
const firstStatus = await waitForCli(
|
||||
"first scoped same-name Node online after restart",
|
||||
() =>
|
||||
runJson(
|
||||
clusterflux,
|
||||
["node", "status", ...scope, "--node", workerNode],
|
||||
{ cwd: projectDir }
|
||||
),
|
||||
(status) => nodeDescriptor(status, workerNode)?.online === true,
|
||||
30_000
|
||||
);
|
||||
const secondStatus = await waitForCli(
|
||||
"second scoped same-name Node online after restart",
|
||||
() =>
|
||||
runJson(
|
||||
clusterflux,
|
||||
[
|
||||
"node",
|
||||
"status",
|
||||
...scopedCollisionRuntime.scope,
|
||||
"--node",
|
||||
workerNode,
|
||||
],
|
||||
{ cwd: scopedCollisionRuntime.projectDir }
|
||||
),
|
||||
(status) => nodeDescriptor(status, workerNode)?.online === true,
|
||||
30_000
|
||||
);
|
||||
scopedNodeCollisionAfterRestart = {
|
||||
node: workerNode,
|
||||
first_scope_online:
|
||||
nodeDescriptor(firstStatus, workerNode)?.online === true,
|
||||
second_scope_online:
|
||||
nodeDescriptor(secondStatus, workerNode)?.online === true,
|
||||
both_credentials_reused_without_grants: true,
|
||||
distinct_credential_digests:
|
||||
credentialDigest !== scopedCollisionRuntime.credentialDigest,
|
||||
};
|
||||
assert.strictEqual(
|
||||
scopedNodeCollisionAfterRestart.distinct_credential_digests,
|
||||
true,
|
||||
"same-name scoped Nodes unexpectedly reused one credential"
|
||||
);
|
||||
} finally {
|
||||
await stopChild(secondWorker.child);
|
||||
}
|
||||
}
|
||||
|
||||
const restartedRun = runJson(
|
||||
clusterflux,
|
||||
["run", "build", "--project", ".", "--json"],
|
||||
|
|
@ -3873,6 +4409,7 @@ async function restartHostedServiceAndResume({
|
|||
terminated_ephemeral_process: ephemeralProcess,
|
||||
new_process: restartedProcess,
|
||||
new_process_result: completedEvent.result,
|
||||
scoped_node_collision_after_restart: scopedNodeCollisionAfterRestart,
|
||||
before,
|
||||
after_first_restart: afterFirstRestart,
|
||||
after,
|
||||
|
|
@ -3880,6 +4417,79 @@ async function restartHostedServiceAndResume({
|
|||
};
|
||||
}
|
||||
|
||||
async function finishScopedNodeCollisionAfterRestart({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
scope,
|
||||
workerNode,
|
||||
scopedCollisionRuntime,
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
assert(scopedCollisionRuntime, "scoped collision runtime evidence is required");
|
||||
const revoked = runJson(
|
||||
clusterflux,
|
||||
[
|
||||
"node",
|
||||
"revoke",
|
||||
...scopedCollisionRuntime.scope,
|
||||
"--node",
|
||||
scopedCollisionRuntime.node,
|
||||
"--yes",
|
||||
],
|
||||
{ cwd: scopedCollisionRuntime.projectDir }
|
||||
);
|
||||
assert.strictEqual(revoked.command, "node revoke");
|
||||
const revokedHeartbeat = assertDenied(
|
||||
await sendHostedControl({
|
||||
type: "node_heartbeat",
|
||||
tenant: scopedCollisionRuntime.tenant,
|
||||
project: scopedCollisionRuntime.project,
|
||||
node: scopedCollisionRuntime.node,
|
||||
node_signature: signedNodeHeartbeat(
|
||||
scopedCollisionRuntime.tenant,
|
||||
scopedCollisionRuntime.project,
|
||||
scopedCollisionRuntime.node,
|
||||
scopedCollisionRuntime.identity,
|
||||
{
|
||||
nonce: `strict-scoped-node-revoked-${Date.now()}`,
|
||||
}
|
||||
),
|
||||
}),
|
||||
/not enrolled|unknown node|revoked|credential/i,
|
||||
"revoked second scoped Node"
|
||||
);
|
||||
const firstStatus = await waitForCli(
|
||||
"first same-name Node remains live after scoped revocation",
|
||||
() =>
|
||||
runJson(
|
||||
clusterflux,
|
||||
["node", "status", ...scope, "--node", workerNode],
|
||||
{ cwd: projectDir }
|
||||
),
|
||||
(status) => nodeDescriptor(status, workerNode)?.online === true,
|
||||
30_000
|
||||
);
|
||||
return {
|
||||
request: {
|
||||
revoked_scope: {
|
||||
tenant: scopedCollisionRuntime.tenant,
|
||||
project: scopedCollisionRuntime.project,
|
||||
node: scopedCollisionRuntime.node,
|
||||
},
|
||||
retained_node: workerNode,
|
||||
},
|
||||
expected:
|
||||
"revocation affects only the selected scoped Node after both same-name credentials survive restart",
|
||||
observed: {
|
||||
revoked_command: revoked.command,
|
||||
revoked_credential_denied: revokedHeartbeat.denied,
|
||||
first_scope_node_online:
|
||||
nodeDescriptor(firstStatus, workerNode)?.online === true,
|
||||
},
|
||||
duration_ms: Math.max(1, Date.now() - startedAt),
|
||||
};
|
||||
}
|
||||
|
||||
async function finishUserCredentialSecurity({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
|
|
@ -5713,7 +6323,14 @@ async function main() {
|
|||
scope,
|
||||
});
|
||||
|
||||
const tenantIsolation = await runSecondTenantIsolation({
|
||||
await stopChild(worker.child);
|
||||
const tenantIsolationResult = await runSecondTenantIsolation({
|
||||
clusterflux,
|
||||
clusterfluxNode,
|
||||
firstScope: scope,
|
||||
firstHelloBuild: helloBuild,
|
||||
firstHelloProjectDir: helloProjectDir,
|
||||
workRoot,
|
||||
firstSessionSecret: sessionSecret,
|
||||
firstTenant: tenant,
|
||||
firstProject: project,
|
||||
|
|
@ -5722,6 +6339,14 @@ async function main() {
|
|||
firstArtifact: releaseArtifact.artifact,
|
||||
manifest,
|
||||
});
|
||||
const tenantIsolation = tenantIsolationResult.evidence;
|
||||
const scopedCollisionRuntime = tenantIsolationResult.runtime;
|
||||
worker = spawnWorker();
|
||||
const collisionRecoveryReady = await worker.waitFor(
|
||||
(value) => value.node_status === "ready",
|
||||
"primary worker restored after two-tenant collision proof"
|
||||
);
|
||||
assert.strictEqual(collisionRecoveryReady.node, workerNode);
|
||||
|
||||
const securityNode = await prepareLiveNodeCredentialSecurity({
|
||||
clusterflux,
|
||||
|
|
@ -5824,6 +6449,15 @@ async function main() {
|
|||
},
|
||||
releaseArtifact,
|
||||
});
|
||||
const signedHostileArtifactPath = await runLiveSignedHostileArtifactPath({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
scope,
|
||||
tenant,
|
||||
project,
|
||||
suffix,
|
||||
securityNode,
|
||||
});
|
||||
const revokedNodeCredential = await revokeSecurityNode({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
|
|
@ -5844,8 +6478,22 @@ async function main() {
|
|||
workerNode,
|
||||
credentialPath,
|
||||
credentialDigest,
|
||||
scopedCollisionRuntime,
|
||||
});
|
||||
if (serviceRestart.worker) worker = serviceRestart.worker;
|
||||
const scopedNodeRevocationAfterRestart = scopedCollisionRuntime
|
||||
? await finishScopedNodeCollisionAfterRestart({
|
||||
clusterflux,
|
||||
projectDir,
|
||||
scope,
|
||||
workerNode,
|
||||
scopedCollisionRuntime,
|
||||
})
|
||||
: {
|
||||
executed: false,
|
||||
reason: "second tenant runtime session was not supplied",
|
||||
duration_ms: 1,
|
||||
};
|
||||
const relayAfterRestart = relayDurableState();
|
||||
assert(
|
||||
relayAfterRestart.ingress_used >=
|
||||
|
|
@ -6298,6 +6946,49 @@ async function main() {
|
|||
evidence: processCancellationLifecycle,
|
||||
durationMs: processCancellationLifecycle.duration_ms,
|
||||
}),
|
||||
requirement({
|
||||
id: "30_scoped_node_and_artifact_collision",
|
||||
passed:
|
||||
tenantIsolation.scoped_node_and_artifact_collision?.observed
|
||||
?.both_metadata_records_visible_to_owner === true &&
|
||||
tenantIsolation.scoped_node_and_artifact_collision?.observed
|
||||
?.first_download_survived_second_link_revocation === true &&
|
||||
tenantIsolation.scoped_node_and_artifact_collision?.observed
|
||||
?.same_artifact_id === helloBuild.artifact &&
|
||||
serviceRestart.scoped_node_collision_after_restart
|
||||
?.first_scope_online === true &&
|
||||
serviceRestart.scoped_node_collision_after_restart
|
||||
?.second_scope_online === true &&
|
||||
serviceRestart.scoped_node_collision_after_restart
|
||||
?.both_credentials_reused_without_grants === true &&
|
||||
scopedNodeRevocationAfterRestart.observed
|
||||
?.revoked_credential_denied === true &&
|
||||
scopedNodeRevocationAfterRestart.observed
|
||||
?.first_scope_node_online === true,
|
||||
evidence: {
|
||||
live_collision:
|
||||
tenantIsolation.scoped_node_and_artifact_collision,
|
||||
restart:
|
||||
serviceRestart.scoped_node_collision_after_restart,
|
||||
scoped_revocation: scopedNodeRevocationAfterRestart,
|
||||
},
|
||||
durationMs:
|
||||
tenantIsolation.scoped_node_and_artifact_collision?.duration_ms +
|
||||
scopedNodeRevocationAfterRestart.duration_ms,
|
||||
}),
|
||||
requirement({
|
||||
id: "31_signed_hostile_artifact_path_preserves_service",
|
||||
passed:
|
||||
signedHostileArtifactPath.observed.rejection.denied === true &&
|
||||
signedHostileArtifactPath.observed.valid_metadata_response ===
|
||||
"vfs_metadata_recorded" &&
|
||||
signedHostileArtifactPath.observed.health_response ===
|
||||
"node_heartbeat" &&
|
||||
signedHostileArtifactPath.observed.process_cleanup ===
|
||||
"not_active",
|
||||
evidence: signedHostileArtifactPath,
|
||||
durationMs: signedHostileArtifactPath.duration_ms,
|
||||
}),
|
||||
];
|
||||
const fullReleasePassed =
|
||||
strictFullRelease &&
|
||||
|
|
@ -6371,6 +7062,8 @@ async function main() {
|
|||
dap_process_aborted: true,
|
||||
process_cancellation_lifecycle: processCancellationLifecycle,
|
||||
tenant_isolation: tenantIsolation,
|
||||
scoped_node_revocation_after_restart:
|
||||
scopedNodeRevocationAfterRestart,
|
||||
credential_security: {
|
||||
user: userCredentialSecurity,
|
||||
node: {
|
||||
|
|
@ -6409,6 +7102,7 @@ async function main() {
|
|||
hello_build: helloBuild,
|
||||
recovery_build: recoveryBuild,
|
||||
malformed_identifiers: malformedIdentifiers,
|
||||
signed_hostile_artifact_path: signedHostileArtifactPath,
|
||||
named_scenarios: namedScenarios,
|
||||
scenario_skips: [],
|
||||
strict_requirement_ledger: strictRequirementLedger,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue