clusterflux-public/scripts/prepare-public-release.js
Clusterflux release c1769967c1 Public release release-01875e88a3e2
Source commit: 01875e88a3e25379c309489f9f057dd97c0d37de

Public tree identity: sha256:8f37a1aa0cc8f408975daf9cabbe193f8f4c169c6d25e8795e67a3ed5083c76b
2026-07-17 06:09:42 +02:00

741 lines
22 KiB
JavaScript
Executable file

#!/usr/bin/env node
const crypto = require("crypto");
const cp = require("child_process");
const fs = require("fs");
const os = require("os");
const path = require("path");
const repo = path.resolve(__dirname, "..");
const outputRoot = path.resolve(
process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR ||
path.join(repo, "target/public-release")
);
const publicTree = path.join(outputRoot, "public-tree");
const assetsDir = path.join(outputRoot, "assets");
const stagingDir = path.join(outputRoot, "staging");
const publicBuildTarget = path.resolve(
process.env.CLUSTERFLUX_PUBLIC_BUILD_TARGET_DIR ||
path.join(outputRoot, "cargo-target")
);
const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com";
const forgejoHost = "git.michelpaulissen.com";
const args = new Set(process.argv.slice(2));
const includeForgejoWorkflows =
args.has("--include-forgejo-workflows") ||
/^(1|true|yes)$/i.test(process.env.CLUSTERFLUX_INCLUDE_FORGEJO_WORKFLOWS || "");
const filteredTopLevel = ["private", "internal", "experiments", ".git", "target"];
const filteredDirectoryNames = [".clusterflux"];
const archiveIgnoredPathFallbacks = [
"target",
".clusterflux",
"vscode-extension/node_modules",
"scripts/containers-home",
];
const publicRepoBranch = process.env.CLUSTERFLUX_PUBLIC_REPO_BRANCH || "main";
const publicBinaries = [
"clusterflux",
"clusterflux-coordinator",
"clusterflux-node",
"clusterflux-debug-dap",
];
function commandOutput(command, args, options = {}) {
try {
const execOptions = {
cwd: repo,
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
...options,
};
execOptions.env = commandEnv(command, options.env);
return cp
.execFileSync(command, args, execOptions)
.trim();
} catch (_) {
return null;
}
}
function run(command, args, options = {}) {
const execOptions = {
cwd: repo,
stdio: "inherit",
...options,
};
execOptions.env = commandEnv(command, options.env);
cp.execFileSync(command, args, execOptions);
}
function nonInteractiveGitEnv(extra = {}) {
return {
...process.env,
GIT_TERMINAL_PROMPT: "0",
GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false",
SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false",
GIT_SSH_COMMAND:
process.env.GIT_SSH_COMMAND ||
"ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0",
...extra,
};
}
function commandEnv(command, env) {
if (command !== "git") {
return env;
}
return nonInteractiveGitEnv(env);
}
function ensureDir(dir) {
fs.mkdirSync(dir, { recursive: true });
}
function filteredOutPatterns() {
return [
"private/**",
"internal/**",
"experiments/**",
".git",
"target",
"git-ignored source paths",
"root/*.md except README.md",
"**/.clusterflux/**",
...(includeForgejoWorkflows ? [] : [".forgejo/**"]),
];
}
function gitIgnoredSourcePaths() {
const output = commandOutput("git", [
"ls-files",
"--others",
"--ignored",
"--exclude-standard",
"--directory",
"-z",
]);
if (output === null) {
return archiveIgnoredPathFallbacks;
}
return [
...new Set([
...archiveIgnoredPathFallbacks,
...output
.split("\0")
.filter(Boolean)
.map((relativePath) =>
relativePath.replaceAll("\\", "/").replace(/\/$/, "")
),
]),
];
}
function isGitIgnored(relativePath, ignoredSourcePaths) {
const normalized = relativePath.replaceAll(path.sep, "/");
return ignoredSourcePaths.some(
(ignored) => normalized === ignored || normalized.startsWith(`${ignored}/`)
);
}
function isFilteredRootMarkdown(relativePath, entry) {
return (
!relativePath.includes(path.sep) &&
(entry.isFile() || entry.isSymbolicLink()) &&
path.extname(entry.name).toLowerCase() === ".md" &&
!["README.md", "SECURITY.md"].includes(entry.name)
);
}
function shouldFilter(entry, relativePath) {
const parts = relativePath.split(path.sep).filter(Boolean);
const topLevel = parts[0];
if (filteredTopLevel.includes(topLevel)) return true;
if (parts.some((part) => filteredDirectoryNames.includes(part))) return true;
if (topLevel === ".forgejo" && !includeForgejoWorkflows) return true;
if (isFilteredRootMarkdown(relativePath, entry)) return true;
return false;
}
function copyFilteredTree(src, dest, ignoredSourcePaths, relative = "") {
ensureDir(dest);
const entries = fs
.readdirSync(src, { withFileTypes: true })
.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const childRelative = relative ? path.join(relative, entry.name) : entry.name;
if (
shouldFilter(entry, childRelative) ||
isGitIgnored(childRelative, ignoredSourcePaths)
) {
continue;
}
const from = path.join(src, entry.name);
const to = path.join(dest, entry.name);
if (entry.isDirectory()) {
copyFilteredTree(from, to, ignoredSourcePaths, childRelative);
} else if (entry.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(from), to);
} else if (entry.isFile()) {
fs.copyFileSync(from, to);
fs.chmodSync(to, fs.statSync(from).mode & 0o777);
}
}
}
function assertFilteredTree(ignoredSourcePaths) {
for (const excluded of ["private", "internal", "experiments"]) {
if (fs.existsSync(path.join(publicTree, excluded))) {
throw new Error(`${excluded}/ leaked into the public tree`);
}
}
for (const file of walkFiles(publicTree)) {
if (file.split(path.sep).includes(".clusterflux")) {
throw new Error(`generated .clusterflux view state leaked into the public tree: ${file}`);
}
}
if (!includeForgejoWorkflows && fs.existsSync(path.join(publicTree, ".forgejo"))) {
throw new Error(".forgejo/ leaked into the host-neutral public tree");
}
if (!fs.existsSync(path.join(publicTree, "README.md"))) {
throw new Error("product README.md is missing from the public tree");
}
for (const ignored of ignoredSourcePaths) {
if (fs.existsSync(path.join(publicTree, ...ignored.split("/")))) {
throw new Error(`Git-ignored source path leaked into the public tree: ${ignored}`);
}
}
for (const entry of fs.readdirSync(publicTree, { withFileTypes: true })) {
if (isFilteredRootMarkdown(entry.name, entry)) {
throw new Error(`internal root Markdown file leaked into the public tree: ${entry.name}`);
}
}
}
function walkFiles(root, relative = "") {
const dir = path.join(root, relative);
const entries = fs
.readdirSync(dir, { withFileTypes: true })
.sort((left, right) => left.name.localeCompare(right.name));
const files = [];
for (const entry of entries) {
const childRelative = relative ? path.join(relative, entry.name) : entry.name;
if (entry.isDirectory()) {
files.push(...walkFiles(root, childRelative));
} else if (entry.isFile()) {
files.push(childRelative);
} else if (entry.isSymbolicLink()) {
files.push(childRelative);
}
}
return files;
}
function hashTree(root) {
const hash = crypto.createHash("sha256");
for (const file of walkFiles(root)) {
const absolute = path.join(root, file);
const stat = fs.lstatSync(absolute);
hash.update(file.replaceAll(path.sep, "/"));
hash.update("\0");
hash.update(String(stat.mode & 0o777));
hash.update("\0");
if (stat.isSymbolicLink()) {
hash.update("symlink");
hash.update("\0");
hash.update(fs.readlinkSync(absolute));
} else {
hash.update(fs.readFileSync(absolute));
}
hash.update("\0");
}
return `sha256:${hash.digest("hex")}`;
}
function sha256File(file) {
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
}
function tarGz(output, cwd, inputs) {
run("tar", ["-czf", output, "-C", cwd, ...inputs]);
}
function platformName() {
return `${os.platform()}-${os.arch()}`;
}
function binaryName(name) {
return process.platform === "win32" ? `${name}.exe` : name;
}
function buildPublicBinaries() {
run("cargo", ["build", "--locked", "--workspace", "--bins", "--release", "--jobs", "2"], {
cwd: publicTree,
env: { ...process.env, CARGO_TARGET_DIR: publicBuildTarget },
});
}
function stageBinaryAssets(releaseName) {
const stageRoot = path.join(stagingDir, "binaries");
const binDir = path.join(stageRoot, "bin");
fs.rmSync(stageRoot, { recursive: true, force: true });
ensureDir(binDir);
for (const binary of publicBinaries) {
const fileName = binaryName(binary);
const built = path.join(publicBuildTarget, "release", fileName);
if (!fs.existsSync(built)) {
throw new Error(`expected release binary ${built}`);
}
const staged = path.join(binDir, fileName);
fs.copyFileSync(built, staged);
fs.chmodSync(staged, 0o755);
}
const archive = path.join(
assetsDir,
`clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz`
);
tarGz(archive, stageRoot, ["."]);
return archive;
}
function publicBinaryDigests() {
return Object.fromEntries(
publicBinaries.map((binary) => {
const fileName = binaryName(binary);
const file = path.join(publicBuildTarget, "release", fileName);
if (!fs.existsSync(file)) throw new Error(`missing built release binary ${file}`);
return [fileName, `sha256:${sha256File(file)}`];
})
);
}
function stageEvidenceAsset(
releaseName,
sourceCommit,
publicTreeIdentity,
binaryDigests
) {
const stage = path.join(stagingDir, "evidence");
fs.rmSync(stage, { recursive: true, force: true });
ensureDir(stage);
const evidenceRoot = path.join(repo, "target/acceptance");
const included = [];
for (const name of ["public-environment.json", "wasmtime-assignment.json"]) {
const source = path.join(evidenceRoot, name);
if (!fs.existsSync(source)) continue;
fs.copyFileSync(source, path.join(stage, name));
included.push(name);
}
const binding = {
kind: "clusterflux-release-evidence-binding",
source_commit: sourceCommit,
public_tree_identity: publicTreeIdentity,
binary_digests: binaryDigests,
configuration_generation:
process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null,
included_evidence: included,
};
fs.writeFileSync(
path.join(stage, "EVIDENCE_BINDING.json"),
`${JSON.stringify(binding, null, 2)}\n`
);
const archive = path.join(
assetsDir,
`clusterflux-public-evidence-${releaseName}.tar.gz`
);
tarGz(archive, stage, ["."]);
return archive;
}
function stageSourceAsset(releaseName) {
const archive = path.join(assetsDir, `clusterflux-public-source-${releaseName}.tar.gz`);
tarGz(archive, publicTree, ["."]);
return archive;
}
function stageExtensionAsset() {
const packageJson = JSON.parse(
fs.readFileSync(path.join(publicTree, "vscode-extension/package.json"), "utf8")
);
const archive = path.join(
assetsDir,
`${packageJson.name}-${packageJson.version}.vsix`
);
run(
"npx",
[
"--yes",
"@vscode/vsce",
"package",
"--allow-missing-repository",
"--skip-license",
"--out",
archive,
],
{ cwd: path.join(publicTree, "vscode-extension") }
);
return archive;
}
function resolverInstructions() {
if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS) {
return process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS.trim();
}
if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY) {
return [
"Add the controlled hosts entry supplied for this release:",
"",
"```",
process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY.trim(),
"```",
].join("\n");
}
if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP) {
return [
"Add this controlled hosts entry for the release:",
"",
"```",
`${process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP} clusterflux.michelpaulissen.com`,
"```",
].join("\n");
}
return [
"`clusterflux.michelpaulissen.com` should resolve through public DNS. If it",
"does not resolve yet, wait for DNS propagation or use the fallback hosts",
"entry supplied with the invitation.",
].join("\n");
}
function writeGettingStartedAsset(releaseName, publicTreeIdentity, resolution) {
const file = path.join(assetsDir, `CLUSTERFLUX_GETTING_STARTED-${releaseName}.md`);
fs.writeFileSync(
file,
`# Clusterflux Getting Started
Use the public repository and release downloads with the hosted coordinator at
\`https://clusterflux.michelpaulissen.com\`.
## DNS
${resolution}
## Install
1. Download the binary archive for your platform from the Forgejo release.
2. Check the archive against \`SHA256SUMS\`.
3. Extract it and put \`bin/\` on your \`PATH\`.
4. Install \`clusterflux-vscode-*.vsix\` when you want the VS Code debugger:
\`\`\`bash
code --install-extension clusterflux-vscode-*.vsix
\`\`\`
## Sign in and run
\`\`\`bash
clusterflux login --browser
clusterflux auth status
clusterflux project list
clusterflux bundle inspect --project examples/launch-build-demo
\`\`\`
The CLI opens the server-provided Authentik authorization URL. State, nonce,
PKCE, provider code exchange, and userinfo remain on the hosted service. The CLI
stores only the resulting scoped Clusterflux session.
Create a node enrollment grant, attach the node, and start the worker:
\`\`\`bash
clusterflux node enroll --project-id <project-id> --json
clusterflux node attach --project-id <project-id> --node workstation \\
--enrollment-grant "$ENROLLMENT_GRANT"
clusterflux-node --coordinator https://clusterflux.michelpaulissen.com \\
--tenant <tenant> --project-id <project-id> --node workstation \\
--project-root "$PWD" --worker --emit-ready
\`\`\`
Then run the workflow:
\`\`\`bash
clusterflux run --project examples/launch-build-demo build
\`\`\`
Public tree identity: \`${publicTreeIdentity}\`
Release name: \`${releaseName}\`
`,
"utf8"
);
return file;
}
function writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution) {
const file = path.join(assetsDir, `CLUSTERFLUX_RELEASE_NOTES-${releaseName}.md`);
const publicRepo =
process.env.CLUSTERFLUX_PUBLIC_REPO_URL ||
"https://git.michelpaulissen.com/michel/clusterflux-public";
const releaseUrl =
process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL ||
"the Forgejo release attached to the public repository";
fs.writeFileSync(
file,
`# Clusterflux Release Notes
Use this release with the public Clusterflux repository and hosted coordinator.
## Links
- Public repository: ${publicRepo}
- Release downloads: ${releaseUrl}
- Hosted coordinator: ${defaultHostedCoordinatorEndpoint}
- Public tree identity: ${publicTreeIdentity}
- Release name: ${releaseName}
## DNS
${resolution}
## First run
1. Download the archive for your platform and \`SHA256SUMS\`.
2. Extract the archive and put \`bin/\` on your \`PATH\`.
3. Optionally install the VS Code extension archive.
4. Run \`clusterflux login --browser\`.
5. Run \`clusterflux node enroll\`, attach your node, and start the worker.
6. Run \`clusterflux run --project examples/launch-build-demo build\`.
`,
"utf8"
);
return file;
}
function writeSha256Sums(assets) {
const sumsPath = path.join(assetsDir, "SHA256SUMS");
const lines = assets
.map((asset) => `${sha256File(asset)} ${path.basename(asset)}`)
.sort()
.join("\n");
fs.writeFileSync(sumsPath, `${lines}\n`);
return sumsPath;
}
function boolEnv(name) {
return /^(1|true|yes)$/i.test(process.env[name] || "");
}
function writePublicTreeProvenance(sourceCommit, releaseName) {
const provenance = {
kind: "clusterflux-filtered-public-tree",
source_commit: sourceCommit,
release_name: releaseName,
filtered_out: filteredOutPatterns(),
public_export: {
host_neutral: !includeForgejoWorkflows,
include_forgejo_workflows: includeForgejoWorkflows,
},
forgejo_host: forgejoHost,
default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint,
};
fs.writeFileSync(
path.join(publicTree, "CLUSTERFLUX_PUBLIC_TREE.json"),
`${JSON.stringify(provenance, null, 2)}\n`
);
}
function publishPublicTree(releaseName, sourceCommit, publicTreeIdentity) {
const remote =
process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE ||
process.env.CLUSTERFLUX_PUBLIC_REPO_URL ||
null;
const enabled = boolEnv("CLUSTERFLUX_PUBLISH_PUBLIC_TREE");
const result = {
enabled,
remote,
branch: publicRepoBranch,
commit: null,
pushed: false,
};
if (!enabled) {
return result;
}
if (!remote) {
throw new Error(
"CLUSTERFLUX_PUBLISH_PUBLIC_TREE requires CLUSTERFLUX_PUBLIC_REPO_REMOTE or CLUSTERFLUX_PUBLIC_REPO_URL"
);
}
if (!remote.includes(forgejoHost)) {
throw new Error(`public repo remote must point at ${forgejoHost}: ${remote}`);
}
run("git", ["init"], { cwd: publicTree });
run("git", ["checkout", "-B", publicRepoBranch], { cwd: publicTree });
run("git", ["config", "user.name", "Clusterflux release"], {
cwd: publicTree,
});
run("git", ["config", "user.email", "release-dryrun@clusterflux.invalid"], {
cwd: publicTree,
});
run("git", ["add", "."], { cwd: publicTree });
run(
"git",
[
"commit",
"-m",
`Public release ${releaseName}`,
"-m",
`Source commit: ${sourceCommit}`,
"-m",
`Public tree identity: ${publicTreeIdentity}`,
],
{ cwd: publicTree }
);
result.commit = commandOutput("git", ["rev-parse", "HEAD"], { cwd: publicTree });
run("git", ["remote", "add", "public", remote], { cwd: publicTree });
const pushArgs = ["push", "public", `HEAD:${publicRepoBranch}`];
if (boolEnv("CLUSTERFLUX_PUBLIC_REPO_PUSH_FORCE_WITH_LEASE")) {
run(
"git",
[
"fetch",
"public",
`refs/heads/${publicRepoBranch}:refs/remotes/public/${publicRepoBranch}`,
],
{ cwd: publicTree }
);
pushArgs.splice(1, 0, "--force-with-lease");
}
run("git", pushArgs, { cwd: publicTree });
result.pushed = true;
return result;
}
function main() {
const sourceCommit = commandOutput("git", ["rev-parse", "HEAD"]);
if (!sourceCommit || !/^[0-9a-f]{40}$/u.test(sourceCommit)) {
throw new Error("the public release requires an exact Git HEAD object id");
}
const acceptanceCommit = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT?.trim();
if (acceptanceCommit && acceptanceCommit !== sourceCommit) {
throw new Error(
`CLUSTERFLUX_ACCEPTANCE_COMMIT ${acceptanceCommit} does not match source HEAD ${sourceCommit}`
);
}
const shortCommit = sourceCommit.slice(0, 12);
const releaseName = process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || `release-${shortCommit}`;
const sourceStatus = commandOutput("git", ["status", "--short"]);
if (sourceStatus === null) {
throw new Error("the public release must be prepared from a Git checkout");
}
if (sourceStatus !== "") {
throw new Error("the public release must be prepared from a clean source tree");
}
const sourceTreeClean = true;
fs.rmSync(outputRoot, { recursive: true, force: true });
ensureDir(publicTree);
ensureDir(assetsDir);
ensureDir(stagingDir);
const ignoredSourcePaths = gitIgnoredSourcePaths();
copyFilteredTree(repo, publicTree, ignoredSourcePaths);
assertFilteredTree(ignoredSourcePaths);
writePublicTreeProvenance(sourceCommit, releaseName);
const publicTreeIdentity = hashTree(publicTree);
const sourceArchive = stageSourceAsset(releaseName);
run("scripts/check-old-name.sh", [], { cwd: publicTree });
run("node", ["scripts/check-docs.js"], { cwd: publicTree });
run("scripts/check-code-size.sh", [], { cwd: publicTree });
const publicTreePublish = publishPublicTree(
releaseName,
sourceCommit,
publicTreeIdentity
);
buildPublicBinaries();
const binaryArchive = stageBinaryAssets(releaseName);
const binaryDigests = publicBinaryDigests();
const evidenceArchive = stageEvidenceAsset(
releaseName,
sourceCommit,
publicTreeIdentity,
binaryDigests
);
const extensionArchive = stageExtensionAsset();
const resolution = resolverInstructions();
const gettingStarted = writeGettingStartedAsset(
releaseName,
publicTreeIdentity,
resolution
);
const invite = writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution);
const assets = [
sourceArchive,
binaryArchive,
evidenceArchive,
extensionArchive,
gettingStarted,
invite,
];
const sha256Sums = writeSha256Sums(assets);
const manifest = {
kind: "clusterflux-public-release",
release_name: releaseName,
source_commit: sourceCommit,
source_tree_clean: sourceTreeClean,
public_tree_identity: publicTreeIdentity,
public_tree: publicTree,
filtered_out: filteredOutPatterns(),
public_export: {
host_neutral: !includeForgejoWorkflows,
include_forgejo_workflows: includeForgejoWorkflows,
},
forgejo_host: forgejoHost,
public_repo_url: process.env.CLUSTERFLUX_PUBLIC_REPO_URL || publicTreePublish.remote,
public_repo_remote: process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || null,
public_tree_publish: publicTreePublish,
forgejo_release_url: process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || null,
default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint,
dns_publication_state:
process.env.CLUSTERFLUX_DNS_PUBLICATION_STATE || "published",
resolver_override:
process.env.CLUSTERFLUX_RESOLVER_OVERRIDE || "none-required-public-dns",
platform: platformName(),
binary_digests: binaryDigests,
configuration_generation:
process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null,
tool_versions: {
node: process.version,
rustc: commandOutput("rustc", ["--version"]) || null,
cargo: commandOutput("cargo", ["--version"]) || null,
tar: commandOutput("tar", ["--version"]) || null,
},
commands: [
"scripts/check-old-name.sh",
"node scripts/check-docs.js",
"scripts/check-code-size.sh",
...(publicTreePublish.enabled
? [`git push public HEAD:${publicRepoBranch}`]
: []),
"cargo build --locked --workspace --bins --release --jobs 2",
],
assets: [...assets, sha256Sums].map((asset) => ({
file: asset,
name: path.basename(asset),
sha256: sha256File(asset),
})),
notes: [
"Upload the assets to the Forgejo Release for the filtered public repository.",
"The real service deployment and full e2e release remain separate acceptance evidence.",
],
};
const manifestPath = path.join(outputRoot, "public-release-manifest.json");
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
console.log(JSON.stringify({ manifest: manifestPath, assets: assetsDir }, null, 2));
}
main();