diff --git a/.gitignore b/.gitignore index 4bdddd1..28c74c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ /target/ +/web/target/ /.clusterflux/ **/.clusterflux/ /vscode-extension/node_modules/ -/private/*/Cargo.lock -!/private/hosted-policy/Cargo.lock -/private/*/target/ -/scripts/containers-home/ diff --git a/CLUSTERFLUX_PUBLIC_TREE.json b/CLUSTERFLUX_PUBLIC_TREE.json deleted file mode 100644 index f968739..0000000 --- a/CLUSTERFLUX_PUBLIC_TREE.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "kind": "clusterflux-filtered-public-tree", - "source_commit": "ea887c8f56cd53985a1179b13e5f1b85c485f584", - "release_name": "release-ea887c8f56cd", - "filtered_out": [ - "private/**", - "internal/**", - "experiments/**", - ".git", - "target", - "git-ignored source paths", - "root/*.md except README.md", - "**/.clusterflux/**", - ".forgejo/**" - ], - "public_export": { - "host_neutral": true, - "include_forgejo_workflows": false - }, - "forgejo_host": "git.michelpaulissen.com", - "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com" -} diff --git a/Cargo.lock b/Cargo.lock index c4ac325..26517cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,20 @@ dependencies = [ "wasmparser 0.245.1", ] +[[package]] +name = "clusterflux-client" +version = "0.1.0" +dependencies = [ + "base64", + "clusterflux-control", + "clusterflux-coordinator", + "clusterflux-core", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "clusterflux-control" version = "0.1.0" @@ -1712,16 +1726,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "runtime-conformance" -version = "0.1.0" -dependencies = [ - "clusterflux-sdk", - "futures-executor", - "serde", - "serde_json", -] - [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index 3c10aa0..b4ff4d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,9 @@ [workspace] resolver = "2" +exclude = ["web"] members = [ "crates/clusterflux-cli", + "crates/clusterflux-client", "crates/clusterflux-control", "crates/clusterflux-coordinator", "crates/clusterflux-core", @@ -12,13 +14,12 @@ members = [ "crates/clusterflux-wasm-runtime", "examples/hello-build", "examples/recovery-build", - "tests/fixtures/runtime-conformance", ] [workspace.package] edition = "2021" license = "Apache-2.0 OR MIT" -repository = "https://git.michelpaulissen.com/michel/clusterflux-public" +repository = "https://clusterflux.lesstuff.com" [workspace.dependencies] anyhow = "1.0" diff --git a/README.md b/README.md index de9d56c..e2c11ac 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,57 @@ # Clusterflux -Clusterflux runs a Rust-defined workflow as one distributed virtual process. A -coordinator hosts the async main, while attached nodes execute Wasm tasks, -rootless containers, and native commands. Tasks exchange canonical values and -portable typed handles instead of sharing host memory. +Clusterflux runs a Rust-defined workflow as one distributed virtual process. The +async main runs serverless, provisioning nodes to run tasks through rootless +containers. Tasks on nodes exchange data simply and efficiently. + +The user experience is built to be as much as possible like building a regular +program. The processes are debuggable as normal through a debugger adapter. + +The primary use case is to consolidate build processes into a single streamlined +developer experience. An example program can be seen below: + +~~~rust +use clusterflux::prelude::*; + +#[clusterflux::task(capabilities = "command")] +pub async fn compile(source: SourceSnapshot) -> Result { + let executable = fs::output("hello-clusterflux")?; + Command::new("cc") + .args([ + "-Os", + "-static", + "-s", + "fixture/hello-clusterflux.c", + "-o", + executable.as_str(), + ]) + .cwd(source.mount()?) + .env("SOURCE_DATE_EPOCH", "0") + .network_disabled() + .run() + .await?; + fs::publish(&executable).await +} + +#[clusterflux::main] +pub async fn build() -> Result { + let source = source::current_project().snapshot().await?; + let compile = clusterflux::spawn!(compile(source)) + .on(clusterflux::env!("linux")) + .await?; + compile.join().await +} +~~~ + +After setup, this build pipeline could be deployed as easily as launching it +through your IDE. This repository includes a VS Code extension to make +development as straightforward as possible. A full collection of CLI tools is +included for advanced usage. + +Clusterflux is explicitly local-first. It is trivial to provision existing +hardware as resources. Bulk data will typically not leave the local network, +allowing maximum throughput. The same capability, however, also makes it +possible to leverage cloud resources easily. Start with [Getting started](docs/getting-started.md). It takes you through authentication, project setup, node enrollment, a run, debugging, task restart, @@ -49,6 +97,9 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ +On NixOS or another system with Nix, the equivalent package is available with +`nix profile install .#clusterflux-tools`. + Rootless Podman is required on Linux nodes that build or run a declared Containerfile environment. Install VS Code when you want the graphical debug workflow. diff --git a/SECURITY.md b/SECURITY.md index a050c6d..2c1cd10 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained. ## Report a vulnerability -Email security@michelpaulissen.com with a concise description, affected version +Email security@clusterflux.lesstuff.com with a concise description, affected version or source revision, reproduction steps, and impact. Do not open a public issue for an unpatched vulnerability or include credentials, session tokens, private keys, provider tokens, customer data, or operator secrets in a public report. diff --git a/crates/clusterflux-cli/src/admin.rs b/crates/clusterflux-cli/src/admin.rs index 83f19a6..ce17d88 100644 --- a/crates/clusterflux-cli/src/admin.rs +++ b/crates/clusterflux-cli/src/admin.rs @@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { .unwrap_or(json!(false)), "response": response, "safe_default": "read_only", - "private_website_required": false, + "external_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { "command": "admin status", "mode": "self_hosted_local", "safe_default": "read_only", - "private_website_required": false, + "external_website_required": false, })) } @@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "project": project.clone(), "user": user, "coordinator": scope.coordinator, - "private_website_required": false, + "external_website_required": false, "self_hosted_cli_only": true, "project_config_written": project_init .get("project_config_written") @@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> { "step": "start_self_hosted_coordinator", "command": "clusterflux-coordinator --listen 127.0.0.1:0", - "private_website_required": false, + "external_website_required": false, }, { "step": "create_or_link_project", "command": "clusterflux project init --yes", "completed": true, - "private_website_required": false, + "external_website_required": false, }, { "step": "create_node_enrollment_grant", @@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "private_website_required": false, + "external_website_required": false, }, { "step": "attach_worker_node", @@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker", tenant, project ), - "private_website_required": false, + "external_website_required": false, }, { "step": "run_process", @@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "private_website_required": false, + "external_website_required": false, }, { "step": "inspect_status_logs_artifacts", @@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux artifact list", "clusterflux quota status", ], - "private_website_required": false, + "external_website_required": false, }, { "step": "revoke_access", "command": "clusterflux admin revoke-node --node --yes", - "private_website_required": false, + "external_website_required": false, } ], })) @@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "actor_tenant": actor_tenant, "actor_user": actor_user, "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"), - "private_website_required": false, + "external_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "status": "requires_coordinator", "requires_confirmation": !args.yes, "tenant": tenant, - "private_website_required": false, + "external_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/artifact.rs b/crates/clusterflux-cli/src/artifact.rs index a0440be..a2c0620 100644 --- a/crates/clusterflux-cli/src/artifact.rs +++ b/crates/clusterflux-cli/src/artifact.rs @@ -12,6 +12,7 @@ use crate::client::{ }; use crate::config::StoredCliSession; use crate::errors::cli_error_summary_for_category; +use crate::process::hydrate_process_scope; use crate::process_events::{ artifact_download_grant_disclosures, artifact_download_session_summary, artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries, @@ -27,9 +28,10 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result { } pub(crate) fn artifact_list_report_with_session( - args: ArtifactListArgs, + mut args: ArtifactListArgs, stored_session: Option<&StoredCliSession>, ) -> Result { + hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, @@ -53,9 +55,10 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result, ) -> Result { + hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( @@ -143,9 +146,10 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result } pub(crate) fn artifact_export_report_with_session( - args: ArtifactExportArgs, + mut args: ArtifactExportArgs, stored_session: Option<&StoredCliSession>, ) -> Result { + hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( diff --git a/crates/clusterflux-cli/src/auth.rs b/crates/clusterflux-cli/src/auth.rs index 608c9ee..d820cbc 100644 --- a/crates/clusterflux-cli/src/auth.rs +++ b/crates/clusterflux-cli/src/auth.rs @@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result Value { "account_status": "unknown", "suspension_known": false, "account_state_known": false, - "private_moderation_details_exposed": false, + "sensitive_moderation_details_exposed": false, "signup_failure_details_exposed": false, "next_actions": ["clusterflux login --browser"], }) diff --git a/crates/clusterflux-cli/src/config.rs b/crates/clusterflux-cli/src/config.rs index db22a3f..ab744ca 100644 --- a/crates/clusterflux-cli/src/config.rs +++ b/crates/clusterflux-cli/src/config.rs @@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize}; use crate::CliScopeArgs; -pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = - "https://clusterflux.michelpaulissen.com"; +pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct ProjectConfig { diff --git a/crates/clusterflux-cli/src/debug.rs b/crates/clusterflux-cli/src/debug.rs index d39c780..356ce8b 100644 --- a/crates/clusterflux-cli/src/debug.rs +++ b/crates/clusterflux-cli/src/debug.rs @@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result { "command": "dap", "adapter": dap_binary_path()?.display().to_string(), "args": args.args, - "private_website_required": false, + "external_website_required": false, })) } @@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session( .cloned() .unwrap_or_else(|| json!(0)), "debug_reads_quota_limited": true, - "private_website_required": false, + "external_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session( "dap": dap, "authorized": "unknown_without_coordinator", "debug_reads_quota_limited": "unknown_without_coordinator", - "private_website_required": false, + "external_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/errors.rs b/crates/clusterflux-cli/src/errors.rs index 0dce2be..282a91d 100644 --- a/crates/clusterflux-cli/src/errors.rs +++ b/crates/clusterflux-cli/src/errors.rs @@ -38,7 +38,10 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s ); object.insert("community_tier_language".to_owned(), json!(true)); object.insert("community_tier_label".to_owned(), json!("community tier")); - object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false)); + object.insert( + "sensitive_abuse_heuristics_exposed".to_owned(), + json!(false), + ); } } summary diff --git a/crates/clusterflux-cli/src/logs.rs b/crates/clusterflux-cli/src/logs.rs index a12815a..dcad5f7 100644 --- a/crates/clusterflux-cli/src/logs.rs +++ b/crates/clusterflux-cli/src/logs.rs @@ -3,6 +3,7 @@ use serde_json::{json, Value}; use crate::client::list_task_events_if_available_with_session; use crate::config::StoredCliSession; +use crate::process::hydrate_process_scope; use crate::process_events::log_entries; use crate::LogsArgs; @@ -12,9 +13,10 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result { } pub(crate) fn logs_report_with_session( - args: LogsArgs, + mut args: LogsArgs, stored_session: Option<&StoredCliSession>, ) -> Result { + hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, diff --git a/crates/clusterflux-cli/src/node.rs b/crates/clusterflux-cli/src/node.rs index 902fa14..3f04516 100644 --- a/crates/clusterflux-cli/src/node.rs +++ b/crates/clusterflux-cli/src/node.rs @@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result Result String { } push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason"); if let Some(exposed) = account - .get("private_moderation_details_exposed") + .get("sensitive_moderation_details_exposed") .and_then(Value::as_bool) { - lines.push(format!("private moderation details exposed: {exposed}")); + lines.push(format!("sensitive moderation details exposed: {exposed}")); } } if let Some(coordinator_selection) = value.get("coordinator") { @@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String { )); } if let Some(flag) = value - .get("private_website_required") + .get("external_website_required") .and_then(Value::as_bool) { - lines.push(format!("private website required: {flag}")); + lines.push(format!("external website required: {flag}")); } if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) { let actions = next_actions diff --git a/crates/clusterflux-cli/src/process.rs b/crates/clusterflux-cli/src/process.rs index e7696b4..5e6adde 100644 --- a/crates/clusterflux-cli/src/process.rs +++ b/crates/clusterflux-cli/src/process.rs @@ -15,7 +15,10 @@ use crate::{ ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs, }; -fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) { +pub(crate) fn hydrate_process_scope( + scope: &mut CliScopeArgs, + stored_session: Option<&StoredCliSession>, +) { if scope.coordinator.is_none() { scope.coordinator = stored_session .filter(|session| session.session_secret.is_some()) diff --git a/crates/clusterflux-cli/src/process_events.rs b/crates/clusterflux-cli/src/process_events.rs index fa577c7..50adc60 100644 --- a/crates/clusterflux-cli/src/process_events.rs +++ b/crates/clusterflux-cli/src/process_events.rs @@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value { "cross_tenant_reuse_allowed": false, "unauthorized_project_reuse_allowed": false, "default_durable_store_assumed": false, - "private_website_required": false, + "external_website_required": false, }]) } @@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option< "current_usage": current_usage, "limits": quota_limits_value(), "next_blocked_action": next_blocked_action, - "private_abuse_heuristics_exposed": false, + "sensitive_abuse_heuristics_exposed": false, }) } diff --git a/crates/clusterflux-cli/src/project.rs b/crates/clusterflux-cli/src/project.rs index 47140db..1573880 100644 --- a/crates/clusterflux-cli/src/project.rs +++ b/crates/clusterflux-cli/src/project.rs @@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result Ok(json!({ "command": "project init", "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, - "private_website_required": false, + "external_website_required": false, "project_config_written": true, "project_config_write_after_coordinator_acceptance": coordinator.is_some(), "coordinator_create_before_local_write": coordinator.is_some(), @@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "config_format": "clusterflux_project_config_v1", "links_current_directory": true, "writes_current_directory_only": true, - "private_website_required": false, + "external_website_required": false, }, "safe_defaults": { "tenant": config.tenant.clone(), @@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "default_project_id_used": args.new_project == "project", "default_project_name_used": args.name == "Clusterflux Project", "browser_interaction_required": false, - "private_website_required": false, + "external_website_required": false, }, "project_config": config, "config_file": project_config_file(&cwd), @@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "user": user, "projects": projects, "project_count": project_count, - "private_website_required": false, + "external_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "source": "local_project_config", "projects": projects, "project_count": project_count, - "private_website_required": false, + "external_website_required": false, })) } @@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, "selected_project": selected_project, "project_config_written": true, - "private_website_required": false, + "external_website_required": false, "project_config": config, "coordinator_response": coordinator_response, })) diff --git a/crates/clusterflux-cli/src/quota.rs b/crates/clusterflux-cli/src/quota.rs index a37977f..9c30880 100644 --- a/crates/clusterflux-cli/src/quota.rs +++ b/crates/clusterflux-cli/src/quota.rs @@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "user": effective_scope.user, "coordinator": coordinator, "project_config": config, - "policy_surface": "generic public quota categories; hosted tuning remains private policy", + "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined", "limits": limits, "window_seconds": window_seconds, "current_usage": current_usage, @@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "next_blocked_action": quota_next_blocked_action(¤t_usage), "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" }, "quota_tier": quota_tier, - "private_abuse_heuristics_exposed": false, + "sensitive_abuse_heuristics_exposed": false, "quota_response": quota_status, })) } diff --git a/crates/clusterflux-cli/src/run.rs b/crates/clusterflux-cli/src/run.rs index b12d5a5..7dfc187 100644 --- a/crates/clusterflux-cli/src/run.rs +++ b/crates/clusterflux-cli/src/run.rs @@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu "safe_failure": true, "message": message, "next_actions": next_actions, - "private_website_required": false, + "external_website_required": false, "machine_error": machine_error, }) } @@ -329,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result { "task_launch": launch_task_response, "coordinator_response": response, "coordinator_session_requests": session.requests(), - "private_website_required": false, + "external_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/tests.rs b/crates/clusterflux-cli/src/tests.rs index c122a86..57b4afe 100644 --- a/crates/clusterflux-cli/src/tests.rs +++ b/crates/clusterflux-cli/src/tests.rs @@ -124,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() { assert_eq!(summary["resource_category"], "api_calls"); assert_eq!(summary["community_tier_language"], true); assert_eq!(summary["community_tier_label"], "community tier"); - assert_eq!(summary["private_abuse_heuristics_exposed"], false); + assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false); let rendered = human_report(&json!({ "command": "run", "machine_error": summary, @@ -207,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() { assert_eq!(report["status"], "authentication_required"); assert_eq!(report["non_interactive"], true); assert_eq!(report["browser_opened"], false); - assert_eq!(report["private_website_required"], false); + assert_eq!(report["external_website_required"], false); assert_eq!(report["machine_error"]["category"], "authentication"); assert_eq!(report["machine_error"]["stable_exit_code"], 20); assert_eq!(report["machine_error"]["browser_opened"], false); @@ -827,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() { assert_eq!(rejected["machine_error"]["resource_category"], "api_calls"); assert_eq!(rejected["machine_error"]["community_tier_language"], true); assert_eq!( - rejected["machine_error"]["private_abuse_heuristics_exposed"], + rejected["machine_error"]["sensitive_abuse_heuristics_exposed"], false ); assert!(rejected["machine_error"]["next_actions"] @@ -1665,12 +1665,11 @@ fn node_attach_refuses_a_symlink_credential_target() { fn hosted_coordinator_remains_a_real_https_control_endpoint() { assert_eq!( control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(), - "https://clusterflux.michelpaulissen.com/api/v1/control" + "https://clusterflux.lesstuff.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") - .unwrap(), - "https://clusterflux.michelpaulissen.com/api/v1/control" + control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), + "https://clusterflux.lesstuff.com/api/v1/control" ); assert!(control_endpoint_identity("http://operator.example.test").is_err()); assert_eq!( @@ -1788,7 +1787,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { assert!(!line.contains(r#""actor_user":"user-session""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, + br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1851,7 +1850,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { "active" ); assert_eq!( - report["coordinator_account_status"]["private_moderation_details_exposed"], + report["coordinator_account_status"]["sensitive_moderation_details_exposed"], false ); } @@ -1933,7 +1932,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() { } #[test] -fn auth_status_queries_coordinator_account_state_without_private_moderation_details() { +fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap().to_string(); let server = std::thread::spawn(move || { @@ -1947,7 +1946,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta assert!(line.contains(r#""actor_user":"user-live""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#, + br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1987,7 +1986,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta "account or tenant is suspended by hosted policy" ); assert_eq!( - report["coordinator_account_status"]["private_moderation_details_exposed"], + report["coordinator_account_status"]["sensitive_moderation_details_exposed"], false ); assert_eq!( @@ -2005,13 +2004,13 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta let serialized = serde_json::to_string(&report).unwrap(); assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); - assert!(!serialized.contains("private moderation note")); + assert!(!serialized.contains("sensitive moderation note")); let rendered = human_report(&report); assert!(rendered.contains("account status: suspended")); assert!(rendered.contains("account suspended: true")); - assert!(rendered.contains("private moderation details exposed: false")); - assert!(!rendered.contains("private moderation note")); + assert!(rendered.contains("sensitive moderation details exposed: false")); + assert!(!rendered.contains("sensitive moderation note")); } #[test] @@ -2062,11 +2061,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { "manual_review": manual_review, "sanitized_reason": reason, "next_actions": ["contact the hosted operator"], - "private_moderation_details_exposed": false, + "sensitive_moderation_details_exposed": false, "signup_failure_details_exposed": false, "abuse_score": 99, - "moderation_notes": "private moderation note", - "signup_policy_trace": "private signup trace", + "moderation_notes": "sensitive moderation note", + "signup_policy_trace": "sensitive signup trace", }) ) .unwrap(); @@ -2105,7 +2104,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { true ); assert_eq!( - report["coordinator_account_status"]["private_moderation_details_exposed"], + report["coordinator_account_status"]["sensitive_moderation_details_exposed"], false ); assert_eq!( @@ -2116,11 +2115,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); assert!(!serialized.contains("signup_policy_trace")); - assert!(!serialized.contains("private moderation note")); + assert!(!serialized.contains("sensitive moderation note")); let rendered = human_report(&report); assert!(rendered.contains(&format!("account status: {status}"))); assert!(rendered.contains(rendered_marker)); - assert!(!rendered.contains("private moderation note")); + assert!(!rendered.contains("sensitive moderation note")); } server.join().unwrap(); } @@ -2235,11 +2234,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { assert_eq!(report["command"], "admin bootstrap"); assert_eq!(report["mode"], "self_hosted_local"); - assert_eq!(report["private_website_required"], false); + assert_eq!(report["external_website_required"], false); assert_eq!(report["self_hosted_cli_only"], true); assert_eq!(report["project_config_written"], true); assert_eq!(report["project_init"]["command"], "project init"); - assert_eq!(report["project_init"]["private_website_required"], false); + assert_eq!(report["project_init"]["external_website_required"], false); assert_eq!( report["project_init"]["project_config"]["project"], "self-hosted" @@ -2265,7 +2264,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { } assert!(steps.iter().all(|step| { !step - .get("private_website_required") + .get("external_website_required") .and_then(Value::as_bool) .unwrap_or(false) })); @@ -2893,13 +2892,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() { assert_eq!(status["command"], "admin status"); assert_eq!(status["safe_default"], "read_only"); - assert_eq!(status["private_website_required"], false); + assert_eq!(status["external_website_required"], false); assert_eq!(status["suspended"], false); assert_eq!(suspended["command"], "admin suspend-tenant"); assert_eq!(suspended["tenant"], "tenant"); assert_eq!(suspended["actor_tenant"], "admin-tenant"); assert_eq!(suspended["suspended"], true); - assert_eq!(suspended["private_website_required"], false); + assert_eq!(suspended["external_website_required"], false); } #[test] @@ -2968,7 +2967,7 @@ fn debug_attach_reports_public_authorization() { assert_eq!(report["charged_debug_read_bytes"], 1024); assert_eq!(report["used_debug_read_bytes"], 1024); assert_eq!(report["debug_reads_quota_limited"], true); - assert_eq!(report["private_website_required"], false); + assert_eq!(report["external_website_required"], false); } #[test] @@ -3005,6 +3004,14 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { "restart_task", br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(), ), + ( + "list_task_events", + br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(), + ), + ( + "list_task_events", + br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(), + ), ( "create_artifact_download_link", br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(), @@ -3109,9 +3116,26 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); - artifact_download_report_with_session( + let logs = logs_report_with_session( + LogsArgs { + scope: scope.clone(), + process: Some("vp".to_owned()), + task: Some("task-a".to_owned()), + }, + Some(&session), + ) + .unwrap(); + let artifacts = artifact_list_report_with_session( + ArtifactListArgs { + scope: scope.clone(), + process: Some("vp".to_owned()), + }, + Some(&session), + ) + .unwrap(); + let download = artifact_download_report_with_session( ArtifactDownloadArgs { - scope: coordinator_scope.clone(), + scope, artifact: "app.txt".to_owned(), to: None, max_bytes: 2048, @@ -3119,6 +3143,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); + assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n"); + assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt"); + assert_eq!(download["coordinator"], session.coordinator); debug_attach_report_with_dap_and_session( DebugAttachArgs { scope: coordinator_scope, @@ -3311,7 +3338,7 @@ fn project_init_uses_public_create_before_writing_local_config() { created["safe_defaults"]["browser_interaction_required"], false ); - assert_eq!(created["private_website_required"], false); + assert_eq!(created["external_website_required"], false); assert_eq!( read_project_config(temp_success.path()) .unwrap() @@ -3411,7 +3438,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(list["source"], "public_coordinator_api"); assert_eq!(list["project_count"], 1); assert_eq!(list["projects"][0]["id"], "project-a"); - assert_eq!(list["private_website_required"], false); + assert_eq!(list["external_website_required"], false); assert_eq!(list["coordinator_session_requests"], 1); let selected = project_select_report( @@ -3426,7 +3453,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(selected["source"], "public_coordinator_api"); assert_eq!(selected["selected_project"]["id"], "project-a"); assert_eq!(selected["project_config_written"], true); - assert_eq!(selected["private_website_required"], false); + assert_eq!(selected["external_website_required"], false); assert_eq!( read_project_config(temp.path()).unwrap().unwrap().project, "project-a" @@ -4455,7 +4482,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { let temp = tempfile::tempdir().unwrap(); let project = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../..") - .join("tests/fixtures/runtime-conformance"); + .join("examples/hello-build"); let output = temp.path().join("bundle"); let report = build_report( @@ -4473,8 +4500,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { assert_eq!(report["command"], "build"); assert_eq!(report["content_addressed"], true); assert_eq!(report["contains_full_repository_upload"], false); - assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 12); - assert_eq!(report["bundle_artifact"]["entrypoint_count"], 6); + assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2); + assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1); assert!(output.join("module.wasm").is_file()); assert!(output.join("manifest.json").is_file()); assert!(output.join("task-descriptors.json").is_file()); @@ -4485,9 +4512,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { .unwrap() .iter() .any(|descriptor| { - descriptor["name"] == "task_add_one" - && descriptor["argument_schema"] == "input : i32" - && descriptor["result_schema"] == "i32" + descriptor["name"] == "compile" + && descriptor["argument_schema"] == "source : SourceSnapshot" + && descriptor["result_schema"] == "Result < Artifact >" && descriptor["restart_compatibility_hash"] .as_str() .unwrap() @@ -4648,7 +4675,7 @@ fn node_enroll_reports_short_lived_public_api_grant() { assert_eq!(report["command"], "node enroll"); assert_eq!(report["status"], "created"); - assert_eq!(report["private_website_required"], false); + assert_eq!(report["external_website_required"], false); assert_eq!(report["tenant"], "tenant"); assert_eq!(report["project"], "project"); assert_eq!(report["user"], "user"); @@ -4791,7 +4818,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() { ) .unwrap(); assert_eq!(enroll["status"], "requires_coordinator"); - assert_eq!(enroll["private_website_required"], false); + assert_eq!(enroll["external_website_required"], false); assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null); assert_eq!(enroll["requested_ttl_seconds"], 60); diff --git a/crates/clusterflux-client/Cargo.toml b/crates/clusterflux-client/Cargo.toml new file mode 100644 index 0000000..fa2b5cc --- /dev/null +++ b/crates/clusterflux-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clusterflux-client" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +base64.workspace = true +clusterflux-control = { path = "../clusterflux-control" } +clusterflux-core = { path = "../clusterflux-core" } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["sync", "time"] } + +[dev-dependencies] +clusterflux-coordinator = { path = "../clusterflux-coordinator" } diff --git a/crates/clusterflux-client/README.md b/crates/clusterflux-client/README.md new file mode 100644 index 0000000..89951c1 --- /dev/null +++ b/crates/clusterflux-client/README.md @@ -0,0 +1,68 @@ +# clusterflux-client + +`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web +backend. It talks to the same versioned control and login endpoints as the CLI. +Callers do not construct JSON envelopes or place session secrets into request +objects. + +The crate intentionally contains no coordinator implementation, persistence +types, scheduler policy, or website-specific business rules. A web backend only +needs this crate to authenticate and work with account status, projects, Agent +keys, node enrollment and liveness, current/recent processes, task attempts, +recent logs, artifacts, quota state, process control, task recovery, and Debug +Epoch state. It does not expose workflow launch or whole-process replay. + +```rust,no_run +use clusterflux_client::{ClusterfluxClient, SessionCredential}; + +# async fn example() -> Result<(), clusterflux_client::ClientError> { +let credential = SessionCredential::from_secret( + std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"), +); +let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")? + .with_session_credential(&credential); + +let account = client.account_status().await?; +let nodes = client.list_nodes().await?; +let processes = client.list_processes(None, 20).await?; +# let _ = (account, nodes, processes); +# Ok(()) +# } +``` + +## Login boundary + +`begin_browser_login` starts the hosted Authentik flow. The identity callback +returns only to the fixed configured website URL with a short-lived handoff. +The website backend calls `exchange_browser_login_handoff` once and stores the +returned `SessionCredential` only in encrypted server-side session storage. +The credential has redacted `Debug` output and deliberately does not implement +Serde serialization. Logout calls the existing session-revocation operation. + +The browser must not receive or persist the Clusterflux session credential, +provider tokens, authorization code, PKCE verifier, or CLI polling secret. + +## Errors, bounds, and transport + +API failures are returned as `ClientError::Api(ApiError)`. Decisions should use +the stable `code`, `category`, `retryable`, and `request_id` fields, while +`message` is for people. The client rejects an error whose request ID does not +match the originating request. + +Paginated list methods require an explicit bounded page size. Process pages are +limited to 100 entries; node, artifact, and recent-log pages to 200. +`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes +its cursor. Recent logs are ephemeral and can report truncation or sequence +loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates +offsets and decodes bounded chunks without materializing the complete artifact. + +`ControlTransport` has bounded connect and I/O timeouts and performs blocking +network work outside the async executor. Dropping a request future cancels the +caller’s wait; any already-started blocking I/O remains bounded by its timeout. +`MockTransport` supplies deterministic responses and records exact envelopes +for application tests. + +`CLIENT_API_VERSION` is the one supported control protocol version. The +checked-in `web_operations.json` contract fixture records every website +operation and its stable error shape, and the crate’s contract suite checks the +fixture. diff --git a/crates/clusterflux-client/src/lib.rs b/crates/clusterflux-client/src/lib.rs new file mode 100644 index 0000000..93c6feb --- /dev/null +++ b/crates/clusterflux-client/src/lib.rs @@ -0,0 +1,924 @@ +mod protocol; +mod transport; +mod types; + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; +use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; +use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION}; +use protocol::{AuthenticatedRequest, LoginRequest, WireResponse}; +use serde_json::json; +use thiserror::Error; +use transport::{ClientTransport, TransportRequest}; + +pub use clusterflux_core::{ + AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind, + Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId, + ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId, + UserId, VfsPath, +}; +pub use transport::{ + ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse, +}; +pub use types::*; + +pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION; + +#[derive(Debug, Error)] +pub enum ClientError { + #[error("Clusterflux API error: {0}")] + Api(ApiError), + #[error(transparent)] + Transport(#[from] ClientTransportError), + #[error("Clusterflux client protocol error: {0}")] + Protocol(String), +} + +#[derive(Clone)] +pub struct ClusterfluxClient { + transport: Arc, + session_secret: Arc>>, + next_request: Arc, +} + +impl ClusterfluxClient { + pub fn connect(endpoint: impl Into) -> Result { + Ok(Self::with_transport(ControlTransport::new(endpoint)?)) + } + + pub fn with_transport(transport: impl ClientTransport) -> Self { + Self { + transport: Arc::new(transport), + session_secret: Arc::new(Mutex::new(None)), + next_request: Arc::new(AtomicU64::new(1)), + } + } + + pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self { + self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone()))); + self + } + + pub fn is_session_configured(&self) -> bool { + self.session_secret + .lock() + .map(|secret| secret.is_some()) + .unwrap_or(false) + } + + pub async fn begin_browser_login(&self) -> Result { + match self + .send_login(LoginRequest::BeginWebBrowserLogin {}) + .await? + { + WireResponse::WebBrowserLoginStarted { + transaction_id, + authorization_url, + expires_at_epoch_seconds, + } => Ok(BrowserLoginStart { + transaction_id, + authorization_url, + expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("web_browser_login_started")), + } + } + + pub async fn exchange_browser_login_handoff( + &self, + transaction_id: impl Into, + handoff_code: impl Into, + ) -> Result { + match self + .send_login(LoginRequest::ExchangeWebLoginHandoff { + transaction_id: transaction_id.into(), + handoff_code: handoff_code.into(), + }) + .await? + { + WireResponse::WebBrowserSession { session } => Ok(BrowserSession { + tenant: session.tenant, + project: session.project, + user: session.user, + credential: SessionCredential(session.session_secret), + expires_at_epoch_seconds: session.expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("web_browser_session")), + } + } + + pub async fn cancel_browser_login( + &self, + transaction_id: impl Into, + ) -> Result<(), ClientError> { + match self + .send_login(LoginRequest::CancelWebBrowserLogin { + transaction_id: transaction_id.into(), + }) + .await? + { + WireResponse::WebBrowserLoginCancelled {} => Ok(()), + _ => Err(unexpected_response("web_browser_login_cancelled")), + } + } + + pub async fn account_status(&self) -> Result { + match self + .send_authenticated(AuthenticatedRequest::AuthStatus) + .await? + { + WireResponse::AuthStatus { + tenant, + project, + actor, + authenticated, + account_status, + suspended, + disabled, + deleted, + manual_review, + sanitized_reason, + next_actions, + } => Ok(AccountStatus { + tenant, + project, + actor, + authenticated, + account_status, + suspended, + disabled, + deleted, + manual_review, + sanitized_reason, + next_actions, + }), + _ => Err(unexpected_response("auth_status")), + } + } + + pub async fn logout(&self) -> Result<(), ClientError> { + match self + .send_authenticated(AuthenticatedRequest::RevokeCliSession) + .await? + { + WireResponse::CliSessionRevoked {} => { + *self.session_secret.lock().map_err(|_| { + ClientError::Protocol("session credential lock was poisoned".to_owned()) + })? = None; + Ok(()) + } + _ => Err(unexpected_response("cli_session_revoked")), + } + } + + pub async fn list_projects(&self) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListProjects) + .await? + { + WireResponse::Projects { projects } => Ok(projects), + _ => Err(unexpected_response("projects")), + } + } + + pub async fn create_project( + &self, + project: ProjectId, + name: impl Into, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateProject { + project, + name: name.into(), + }) + .await? + { + WireResponse::ProjectCreated { project } => Ok(project), + _ => Err(unexpected_response("project_created")), + } + } + + pub async fn select_project(&self, project: ProjectId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::SelectProject { project }) + .await? + { + WireResponse::ProjectSelected { project } => Ok(project), + _ => Err(unexpected_response("project_selected")), + } + } + + pub async fn register_agent_public_key( + &self, + agent: AgentId, + public_key: impl Into, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey { + agent, + public_key: public_key.into(), + }) + .await + } + + pub async fn list_agent_public_keys(&self) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys) + .await? + { + WireResponse::AgentPublicKeys { records } => Ok(records), + _ => Err(unexpected_response("agent_public_keys")), + } + } + + pub async fn rotate_agent_public_key( + &self, + agent: AgentId, + public_key: impl Into, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey { + agent, + public_key: public_key.into(), + }) + .await + } + + pub async fn revoke_agent_public_key( + &self, + agent: AgentId, + ) -> Result { + self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent }) + .await + } + + async fn agent_key_mutation( + &self, + request: AuthenticatedRequest, + ) -> Result { + match self.send_authenticated(request).await? { + WireResponse::AgentPublicKey { record } => Ok(record), + _ => Err(unexpected_response("agent_public_key")), + } + } + + pub async fn create_node_enrollment_grant( + &self, + ttl_seconds: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds }) + .await? + { + WireResponse::NodeEnrollmentGrantCreated { + tenant, + project, + grant, + scope, + expires_at_epoch_seconds, + } => Ok(NodeEnrollmentGrant { + tenant, + project, + grant, + scope, + expires_at_epoch_seconds, + }), + _ => Err(unexpected_response("node_enrollment_grant_created")), + } + } + + pub async fn list_nodes(&self) -> Result, ClientError> { + Ok(self.list_nodes_page(None, 200).await?.nodes) + } + + pub async fn list_nodes_page( + &self, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit }) + .await? + { + WireResponse::NodeSummaries { nodes, next_cursor } => { + Ok(NodePage { nodes, next_cursor }) + } + _ => Err(unexpected_response("node_summaries")), + } + } + + pub async fn revoke_node(&self, node: NodeId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node }) + .await? + { + WireResponse::NodeCredentialRevoked { + node, + tenant, + project, + actor, + descriptor_removed, + queued_assignments_removed, + } => Ok(NodeRevocation { + node, + tenant, + project, + actor, + descriptor_removed, + queued_assignments_removed, + }), + _ => Err(unexpected_response("node_credential_revoked")), + } + } + + pub async fn list_processes( + &self, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit }) + .await? + { + WireResponse::ProcessSummaries { + processes, + next_cursor, + } => Ok(ProcessPage { + processes, + next_cursor, + }), + _ => Err(unexpected_response("process_summaries")), + } + } + + pub async fn cancel_process( + &self, + process: ProcessId, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CancelProcess { process }) + .await? + { + WireResponse::ProcessCancellationRequested { + process, + cancelled_tasks, + affected_nodes, + } => Ok(ProcessCancellation { + process, + affected_tasks: cancelled_tasks, + affected_nodes, + aborted: false, + }), + _ => Err(unexpected_response("process_cancellation_requested")), + } + } + + pub async fn abort_process( + &self, + process: ProcessId, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::AbortProcess { + process, + launch_attempt: None, + }) + .await? + { + WireResponse::ProcessAborted { + process, + aborted_tasks, + affected_nodes, + } => Ok(ProcessCancellation { + process, + affected_tasks: aborted_tasks, + affected_nodes, + aborted: true, + }), + _ => Err(unexpected_response("process_aborted")), + } + } + + pub async fn quota_status(&self) -> Result { + match self + .send_authenticated(AuthenticatedRequest::QuotaStatus) + .await? + { + WireResponse::QuotaStatus { + tenant, + project, + actor, + policy_label, + limits, + window_seconds, + usage, + window_started_epoch_seconds, + } => Ok(QuotaStatus { + tenant, + project, + actor, + policy_label, + limits, + window_seconds, + usage, + window_started_epoch_seconds, + }), + _ => Err(unexpected_response("quota_status")), + } + } + + pub async fn list_task_events( + &self, + process: Option, + ) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListTaskEvents { process }) + .await? + { + WireResponse::TaskEvents { events } => Ok(events), + _ => Err(unexpected_response("task_events")), + } + } + + pub async fn list_task_snapshots( + &self, + process: ProcessId, + ) -> Result, ClientError> { + match self + .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process }) + .await? + { + WireResponse::TaskSnapshots { snapshots } => Ok(snapshots), + _ => Err(unexpected_response("task_snapshots")), + } + } + + pub async fn list_recent_logs( + &self, + process: ProcessId, + task: Option, + after_sequence: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListRecentLogs { + process, + task, + after_sequence, + limit, + }) + .await? + { + WireResponse::RecentLogs { + entries, + next_sequence, + history_truncated, + } => Ok(RecentLogPage { + entries, + next_sequence, + history_truncated, + }), + _ => Err(unexpected_response("recent_logs")), + } + } + + pub async fn restart_task( + &self, + process: ProcessId, + task: TaskInstanceId, + replacement_bundle: Option, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::RestartTask { + process, + task, + replacement_bundle, + }) + .await? + { + WireResponse::TaskRestart { + process, + task, + restarted_task_instance, + restarted_attempt_id, + actor, + accepted, + clean_boundary_available, + active_task, + completed_event_observed, + requires_whole_process_restart, + message, + audit_event, + } => Ok(TaskRestart { + process, + task, + restarted_task_instance, + restarted_attempt_id, + actor, + accepted, + clean_boundary_available, + active_task, + completed_event_observed, + requires_whole_process_restart, + message, + audit_event, + }), + _ => Err(unexpected_response("task_restart")), + } + } + + pub async fn resolve_task_failure( + &self, + process: ProcessId, + task: TaskInstanceId, + resolution: TaskFailureResolution, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ResolveTaskFailure { + process, + task, + resolution, + }) + .await? + { + WireResponse::TaskFailureResolved { + process, + task, + attempt_id, + resolution, + } => Ok(TaskFailureResolutionResult { + process, + task, + attempt_id, + resolution, + }), + _ => Err(unexpected_response("task_failure_resolved")), + } + } + + pub async fn debug_attach(&self, process: ProcessId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::DebugAttach { process }) + .await? + { + WireResponse::DebugAttach { + process, + actor, + authorization, + audit_event, + } => Ok(DebugAttach { + process, + actor, + authorization, + audit_event, + }), + _ => Err(unexpected_response("debug_attach")), + } + } + + pub async fn create_debug_epoch( + &self, + process: ProcessId, + stopped_task: TaskInstanceId, + reason: impl Into, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateDebugEpoch { + process, + stopped_task, + reason: reason.into(), + }) + .await? + { + WireResponse::DebugEpoch { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + } => Ok(DebugEpochControl { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch")), + } + } + + pub async fn resume_debug_epoch( + &self, + process: ProcessId, + epoch: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch }) + .await? + { + WireResponse::DebugEpoch { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + } => Ok(DebugEpochControl { + process, + actor, + epoch, + command, + affected_tasks, + all_stop_requested, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch")), + } + } + + pub async fn inspect_debug_epoch( + &self, + process: ProcessId, + epoch: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch }) + .await? + { + WireResponse::DebugEpochStatus { + process, + actor, + epoch, + command, + expected_tasks, + acknowledgements, + fully_frozen, + partially_frozen, + fully_resumed, + failed, + failure_messages, + audit_event, + } => Ok(DebugEpochStatus { + process, + actor, + epoch, + command, + expected_tasks, + acknowledgements, + fully_frozen, + partially_frozen, + fully_resumed, + failed, + failure_messages, + audit_event, + }), + _ => Err(unexpected_response("debug_epoch_status")), + } + } + + pub async fn list_artifacts( + &self, + process: Option, + cursor: Option, + limit: u32, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::ListArtifacts { + process, + cursor, + limit, + }) + .await? + { + WireResponse::Artifacts { + artifacts, + next_cursor, + } => Ok(ArtifactPage { + artifacts, + next_cursor, + }), + _ => Err(unexpected_response("artifacts")), + } + } + + pub async fn get_artifact(&self, artifact: ArtifactId) -> Result { + match self + .send_authenticated(AuthenticatedRequest::GetArtifact { artifact }) + .await? + { + WireResponse::Artifact { artifact } => Ok(artifact), + _ => Err(unexpected_response("artifact")), + } + } + + pub async fn begin_artifact_download( + &self, + artifact: ArtifactId, + max_bytes: u64, + ttl_seconds: u64, + chunk_bytes: u64, + ) -> Result { + match self + .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink { + artifact: artifact.clone(), + max_bytes, + ttl_seconds, + }) + .await? + { + WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload { + client: self.clone(), + artifact, + max_bytes, + chunk_bytes, + token_digest: link.scoped_token_digest.clone(), + link, + expected_offset: 0, + finished: false, + }), + _ => Err(unexpected_response("artifact_download_link")), + } + } + + async fn send_authenticated( + &self, + request: AuthenticatedRequest, + ) -> Result { + let session_secret = self + .session_secret + .lock() + .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))? + .clone() + .ok_or_else(|| { + ClientError::Api(ApiError::from_message( + "client", + "no authenticated Clusterflux session is configured", + )) + })?; + self.send( + CONTROL_API_PATH, + json!({ + "type": "authenticated", + "session_secret": session_secret, + "request": request, + }), + ) + .await + } + + async fn send_login(&self, request: LoginRequest) -> Result { + let payload = serde_json::to_value(request) + .map_err(|error| ClientError::Protocol(error.to_string()))?; + self.send(LOGIN_API_PATH, payload).await + } + + async fn send( + &self, + api_path: &str, + payload: serde_json::Value, + ) -> Result { + let request_number = self.next_request.fetch_add(1, Ordering::Relaxed); + let request_id = format!("client-{request_number}"); + let envelope = coordinator_wire_request(&request_id, payload); + let body = serde_json::to_vec(&envelope) + .map_err(|error| ClientError::Protocol(error.to_string()))?; + let response = self + .transport + .send(TransportRequest { + api_path: api_path.to_owned(), + body, + }) + .await?; + let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| { + ClientError::Protocol(format!("decode typed API response: {error}")) + })?; + match response { + WireResponse::Error { + code, + category, + message, + retryable, + request_id: response_request_id, + } => { + if response_request_id != request_id { + return Err(ClientError::Protocol(format!( + "error response request_id {response_request_id} does not match {request_id}" + ))); + } + Err(ClientError::Api(ApiError::new( + code, + category, + message, + retryable, + response_request_id, + ))) + } + response => Ok(response), + } + } +} + +pub struct ArtifactDownload { + client: ClusterfluxClient, + artifact: ArtifactId, + max_bytes: u64, + chunk_bytes: u64, + token_digest: Digest, + link: clusterflux_core::DownloadLink, + expected_offset: u64, + finished: bool, +} + +impl ArtifactDownload { + pub fn link(&self) -> &clusterflux_core::DownloadLink { + &self.link + } + + pub async fn next_chunk(&mut self) -> Result { + if self.finished { + return Ok(ArtifactDownloadPoll::Chunk { + offset: self.expected_offset, + bytes: Vec::new(), + eof: true, + }); + } + match self + .client + .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream { + artifact: self.artifact.clone(), + max_bytes: self.max_bytes, + token_digest: self.token_digest.clone(), + chunk_bytes: self.chunk_bytes, + }) + .await? + { + WireResponse::ArtifactDownloadStream { + content_bytes_available, + content_offset, + content_eof, + content_base64, + .. + } => { + if !content_bytes_available { + return Ok(ArtifactDownloadPoll::Pending); + } + let offset = content_offset.ok_or_else(|| { + ClientError::Protocol( + "artifact response contained bytes without an offset".to_owned(), + ) + })?; + if offset != self.expected_offset { + return Err(ClientError::Protocol(format!( + "artifact chunk offset {offset} does not match expected {}", + self.expected_offset + ))); + } + let bytes = BASE64_STANDARD + .decode(content_base64.unwrap_or_default()) + .map_err(|error| { + ClientError::Protocol(format!( + "artifact response contains invalid base64: {error}" + )) + })?; + self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64); + self.finished = content_eof; + Ok(ArtifactDownloadPoll::Chunk { + offset, + bytes, + eof: content_eof, + }) + } + _ => Err(unexpected_response("artifact_download_stream")), + } + } + + pub async fn cancel(mut self) -> Result<(), ClientError> { + if self.finished { + return Ok(()); + } + match self + .client + .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink { + artifact: self.artifact.clone(), + token_digest: self.token_digest.clone(), + }) + .await? + { + WireResponse::ArtifactDownloadLinkRevoked {} => { + self.finished = true; + Ok(()) + } + _ => Err(unexpected_response("artifact_download_link_revoked")), + } + } +} + +fn unexpected_response(expected: &str) -> ClientError { + ClientError::Protocol(format!( + "expected typed response {expected}, received another response variant" + )) +} diff --git a/crates/clusterflux-client/src/protocol.rs b/crates/clusterflux-client/src/protocol.rs new file mode 100644 index 0000000..a71ecff --- /dev/null +++ b/crates/clusterflux-client/src/protocol.rs @@ -0,0 +1,403 @@ +use std::collections::BTreeMap; + +use clusterflux_core::{ + AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink, + LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId, +}; +use serde::{Deserialize, Serialize}; + +use crate::types::*; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum AuthenticatedRequest { + AuthStatus, + RevokeCliSession, + CreateProject { + project: ProjectId, + name: String, + }, + SelectProject { + project: ProjectId, + }, + ListProjects, + RegisterAgentPublicKey { + agent: AgentId, + public_key: String, + }, + ListAgentPublicKeys, + RotateAgentPublicKey { + agent: AgentId, + public_key: String, + }, + RevokeAgentPublicKey { + agent: AgentId, + }, + CreateNodeEnrollmentGrant { + ttl_seconds: u64, + }, + ListNodeSummaries { + cursor: Option, + limit: u32, + }, + RevokeNodeCredential { + node: NodeId, + }, + ListProcessSummaries { + cursor: Option, + limit: u32, + }, + CancelProcess { + process: ProcessId, + }, + AbortProcess { + process: ProcessId, + launch_attempt: Option, + }, + QuotaStatus, + ListTaskEvents { + process: Option, + }, + ListTaskSnapshots { + process: ProcessId, + }, + ListRecentLogs { + process: ProcessId, + task: Option, + after_sequence: Option, + limit: u32, + }, + RestartTask { + process: ProcessId, + task: TaskInstanceId, + replacement_bundle: Option, + }, + ResolveTaskFailure { + process: ProcessId, + task: TaskInstanceId, + resolution: TaskFailureResolution, + }, + DebugAttach { + process: ProcessId, + }, + CreateDebugEpoch { + process: ProcessId, + stopped_task: TaskInstanceId, + reason: String, + }, + ResumeDebugEpoch { + process: ProcessId, + epoch: u64, + }, + InspectDebugEpoch { + process: ProcessId, + epoch: u64, + }, + ListArtifacts { + process: Option, + cursor: Option, + limit: u32, + }, + GetArtifact { + artifact: ArtifactId, + }, + CreateArtifactDownloadLink { + artifact: ArtifactId, + max_bytes: u64, + ttl_seconds: u64, + }, + OpenArtifactDownloadStream { + artifact: ArtifactId, + max_bytes: u64, + token_digest: Digest, + chunk_bytes: u64, + }, + RevokeArtifactDownloadLink { + artifact: ArtifactId, + token_digest: Digest, + }, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum LoginRequest { + BeginWebBrowserLogin {}, + CancelWebBrowserLogin { + transaction_id: String, + }, + ExchangeWebLoginHandoff { + transaction_id: String, + handoff_code: String, + }, +} + +#[derive(Clone, Deserialize)] +pub(crate) struct WireBrowserSession { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub session_secret: String, + pub expires_at_epoch_seconds: u64, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum WireResponse { + AuthStatus { + tenant: TenantId, + project: ProjectId, + actor: UserId, + authenticated: bool, + account_status: String, + suspended: bool, + disabled: bool, + deleted: bool, + manual_review: bool, + sanitized_reason: Option, + next_actions: Vec, + }, + CliSessionRevoked {}, + ProjectCreated { + project: Project, + }, + ProjectSelected { + project: Project, + }, + Projects { + projects: Vec, + }, + AgentPublicKey { + record: AgentPublicKey, + }, + AgentPublicKeys { + records: Vec, + }, + NodeEnrollmentGrantCreated { + tenant: TenantId, + project: ProjectId, + grant: String, + scope: String, + expires_at_epoch_seconds: u64, + }, + NodeSummaries { + nodes: Vec, + next_cursor: Option, + }, + NodeCredentialRevoked { + node: NodeId, + tenant: TenantId, + project: ProjectId, + actor: UserId, + descriptor_removed: bool, + queued_assignments_removed: usize, + }, + ProcessSummaries { + processes: Vec, + next_cursor: Option, + }, + ProcessCancellationRequested { + process: ProcessId, + cancelled_tasks: Vec, + affected_nodes: Vec, + }, + ProcessAborted { + process: ProcessId, + aborted_tasks: Vec, + affected_nodes: Vec, + }, + QuotaStatus { + tenant: TenantId, + project: ProjectId, + actor: UserId, + policy_label: Option, + limits: ResourceLimits, + window_seconds: BTreeMap, + usage: BTreeMap, + window_started_epoch_seconds: BTreeMap, + }, + TaskEvents { + events: Vec, + }, + TaskSnapshots { + snapshots: Vec, + }, + RecentLogs { + entries: Vec, + next_sequence: Option, + history_truncated: bool, + }, + TaskRestart { + process: ProcessId, + task: TaskInstanceId, + restarted_task_instance: Option, + restarted_attempt_id: Option, + actor: UserId, + accepted: bool, + clean_boundary_available: bool, + active_task: bool, + completed_event_observed: bool, + requires_whole_process_restart: bool, + message: String, + audit_event: DebugAuditEvent, + }, + TaskFailureResolved { + process: ProcessId, + task: TaskInstanceId, + attempt_id: String, + resolution: TaskFailureResolution, + }, + DebugAttach { + process: ProcessId, + actor: UserId, + authorization: Authorization, + audit_event: DebugAuditEvent, + }, + DebugEpoch { + process: ProcessId, + actor: UserId, + epoch: u64, + command: String, + affected_tasks: Vec, + all_stop_requested: bool, + audit_event: DebugAuditEvent, + }, + DebugEpochStatus { + process: ProcessId, + actor: UserId, + epoch: u64, + command: String, + expected_tasks: Vec, + acknowledgements: Vec, + fully_frozen: bool, + partially_frozen: bool, + fully_resumed: bool, + failed: bool, + failure_messages: Vec, + audit_event: DebugAuditEvent, + }, + Artifacts { + artifacts: Vec, + next_cursor: Option, + }, + Artifact { + artifact: ArtifactSummary, + }, + ArtifactDownloadLink { + link: DownloadLink, + }, + ArtifactDownloadLinkRevoked {}, + ArtifactDownloadStream { + #[serde(rename = "link")] + _link: DownloadLink, + content_bytes_available: bool, + content_offset: Option, + content_eof: bool, + content_base64: Option, + }, + WebBrowserLoginStarted { + transaction_id: String, + authorization_url: String, + expires_at_epoch_seconds: u64, + }, + WebBrowserLoginCancelled {}, + WebBrowserSession { + session: WireBrowserSession, + }, + Error { + code: ApiErrorCode, + category: ApiErrorCategory, + message: String, + retryable: bool, + request_id: String, + }, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use serde::Deserialize; + use serde_json::Value; + + use super::*; + + #[derive(Deserialize)] + struct OperationFixture { + operation: String, + boundary: String, + request: Value, + response: Value, + } + + #[test] + fn website_operation_contract_fixtures_cover_every_typed_request() { + let fixtures: Vec = + serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap(); + let expected = BTreeSet::from([ + "abort_process", + "auth_status", + "begin_web_browser_login", + "cancel_web_browser_login", + "cancel_process", + "create_artifact_download_link", + "create_debug_epoch", + "create_node_enrollment_grant", + "create_project", + "debug_attach", + "exchange_web_login_handoff", + "get_artifact", + "inspect_debug_epoch", + "list_agent_public_keys", + "list_artifacts", + "list_node_summaries", + "list_process_summaries", + "list_projects", + "list_recent_logs", + "list_task_events", + "list_task_snapshots", + "open_artifact_download_stream", + "quota_status", + "register_agent_public_key", + "resolve_task_failure", + "restart_task", + "resume_debug_epoch", + "revoke_agent_public_key", + "revoke_artifact_download_link", + "revoke_cli_session", + "revoke_node_credential", + "rotate_agent_public_key", + "select_project", + ]) + .into_iter() + .map(str::to_owned) + .collect(); + let mut observed = BTreeSet::new(); + + for fixture in fixtures { + assert_eq!(fixture.request["type"], fixture.operation); + let round_trip = match fixture.boundary.as_str() { + "control" => { + let typed: AuthenticatedRequest = + serde_json::from_value(fixture.request.clone()).unwrap(); + serde_json::to_value(typed).unwrap() + } + "login" => { + let typed: LoginRequest = + serde_json::from_value(fixture.request.clone()).unwrap(); + serde_json::to_value(typed).unwrap() + } + boundary => panic!("unknown fixture boundary {boundary}"), + }; + assert_eq!(round_trip, fixture.request); + let response: WireResponse = serde_json::from_value(fixture.response).unwrap(); + assert!( + !matches!(response, WireResponse::Error { .. }), + "{} must carry its operation-specific success response fixture", + fixture.operation + ); + assert!(observed.insert(fixture.operation)); + } + assert_eq!(observed, expected); + } +} diff --git a/crates/clusterflux-client/src/transport.rs b/crates/clusterflux-client/src/transport.rs new file mode 100644 index 0000000..b480c0f --- /dev/null +++ b/crates/clusterflux-client/src/transport.rs @@ -0,0 +1,250 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use clusterflux_control::{endpoint_identity, ControlSession}; +use thiserror::Error; + +pub type TransportFuture = + Pin> + Send + 'static>>; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportRequest { + pub api_path: String, + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportResponse { + pub body: Vec, +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum ClientTransportError { + #[error("client transport failed: {0}")] + Failed(String), + #[error("client transport task failed: {0}")] + Task(String), +} + +pub trait ClientTransport: Send + Sync + 'static { + fn send(&self, request: TransportRequest) -> TransportFuture; +} + +type ControlSessionPool = Arc>>>; + +pub struct ControlTransport { + endpoint: String, + connect_timeout: Duration, + io_timeout: Duration, + sessions: Arc>>, + next_session: Arc, +} + +// A small pool allows independent HTMX reads to progress concurrently while +// keeping connection and blocking-worker use strictly bounded. +const SESSIONS_PER_API_PATH: usize = 4; + +impl ControlTransport { + pub fn new(endpoint: impl Into) -> Result { + Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30)) + } + + pub fn with_timeouts( + endpoint: impl Into, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result { + let endpoint = endpoint.into(); + endpoint_identity(&endpoint) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?; + Ok(Self { + endpoint, + connect_timeout, + io_timeout, + sessions: Arc::new(Mutex::new(BTreeMap::new())), + next_session: Arc::new(AtomicU64::new(0)), + }) + } +} + +impl ClientTransport for ControlTransport { + fn send(&self, request: TransportRequest) -> TransportFuture { + let endpoint = self.endpoint.clone(); + let connect_timeout = self.connect_timeout; + let io_timeout = self.io_timeout; + let sessions = Arc::clone(&self.sessions); + let next_session = Arc::clone(&self.next_session); + Box::pin(async move { + let pool = { + let mut sessions = sessions.lock().map_err(|_| { + ClientTransportError::Failed( + "client transport pool lock was poisoned".to_owned(), + ) + })?; + Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| { + Arc::new( + (0..SESSIONS_PER_API_PATH) + .map(|_| Mutex::new(None)) + .collect(), + ) + })) + }; + let slot_index = + next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH; + tokio::task::spawn_blocking(move || { + let value = serde_json::from_slice(&request.body) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?; + let mut selected = None; + for offset in 0..SESSIONS_PER_API_PATH { + let index = (slot_index + offset) % SESSIONS_PER_API_PATH; + match pool[index].try_lock() { + Ok(session) if session.is_some() => { + selected = Some(session); + break; + } + Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {} + Err(std::sync::TryLockError::Poisoned(_)) => { + return Err(ClientTransportError::Failed( + "client transport session lock was poisoned".to_owned(), + )); + } + } + } + if selected.is_none() { + for offset in 0..SESSIONS_PER_API_PATH { + let index = (slot_index + offset) % SESSIONS_PER_API_PATH; + match pool[index].try_lock() { + Ok(session) => { + selected = Some(session); + break; + } + Err(std::sync::TryLockError::WouldBlock) => {} + Err(std::sync::TryLockError::Poisoned(_)) => { + return Err(ClientTransportError::Failed( + "client transport session lock was poisoned".to_owned(), + )); + } + } + } + } + let mut session = match selected { + Some(session) => session, + None => pool[slot_index].lock().map_err(|_| { + ClientTransportError::Failed( + "client transport session lock was poisoned".to_owned(), + ) + })?, + }; + if session.is_none() { + *session = Some( + ControlSession::connect_to_api_path_with_timeouts( + &endpoint, + &request.api_path, + connect_timeout, + io_timeout, + ) + .map_err(|error| ClientTransportError::Failed(error.to_string()))?, + ); + } + let response = session + .as_mut() + .expect("session was initialized for the requested API path") + .request(&value); + match response { + Ok(response) => serde_json::to_vec(&response) + .map(|body| TransportResponse { body }) + .map_err(|error| ClientTransportError::Failed(error.to_string())), + Err(error) => { + *session = None; + Err(ClientTransportError::Failed(error.to_string())) + } + } + }) + .await + .map_err(|error| ClientTransportError::Task(error.to_string()))? + }) + } +} + +#[derive(Clone, Default)] +pub struct MockTransport { + state: Arc>, +} + +#[derive(Default)] +struct MockTransportState { + responses: VecDeque, ClientTransportError>>, + requests: Vec, +} + +impl MockTransport { + pub fn from_json_responses(responses: impl IntoIterator>) -> Self { + let responses = responses + .into_iter() + .map(|response| Ok(response.into().into_bytes())) + .collect(); + Self { + state: Arc::new(Mutex::new(MockTransportState { + responses, + requests: Vec::new(), + })), + } + } + + pub fn push_json_response(&self, response: impl Into) { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .responses + .push_back(Ok(response.into().into_bytes())); + } + + pub fn push_error(&self, message: impl Into) { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .responses + .push_back(Err(ClientTransportError::Failed(message.into()))); + } + + pub fn request_bodies(&self) -> Vec { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .requests + .iter() + .map(|request| String::from_utf8_lossy(&request.body).into_owned()) + .collect() + } + + pub fn requests(&self) -> Vec { + self.state + .lock() + .expect("mock transport lock is not poisoned") + .requests + .clone() + } +} + +impl ClientTransport for MockTransport { + fn send(&self, request: TransportRequest) -> TransportFuture { + let state = Arc::clone(&self.state); + Box::pin(async move { + let mut state = state.lock().map_err(|_| { + ClientTransportError::Failed("mock transport lock was poisoned".to_owned()) + })?; + state.requests.push(request); + state + .responses + .pop_front() + .ok_or_else(|| { + ClientTransportError::Failed("mock transport has no queued response".to_owned()) + })? + .map(|body| TransportResponse { body }) + }) + } +} diff --git a/crates/clusterflux-client/src/types.rs b/crates/clusterflux-client/src/types.rs new file mode 100644 index 0000000..eb17e6e --- /dev/null +++ b/crates/clusterflux-client/src/types.rs @@ -0,0 +1,468 @@ +use std::collections::BTreeMap; + +use clusterflux_core::{ + AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement, + ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy, + TaskInstanceId, TenantId, UserId, VfsPath, +}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccountStatus { + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub authenticated: bool, + pub account_status: String, + pub suspended: bool, + pub disabled: bool, + pub deleted: bool, + pub manual_review: bool, + pub sanitized_reason: Option, + pub next_actions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Project { + pub id: ProjectId, + pub tenant: TenantId, + pub name: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentPublicKey { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub agent: AgentId, + pub public_key: String, + pub public_key_fingerprint: Digest, + pub version: u64, + pub revoked: bool, + pub scopes: Vec, + pub human_account_creation_privilege: bool, + pub browser_interaction_required_each_run: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeEnrollmentGrant { + pub tenant: TenantId, + pub project: ProjectId, + pub grant: String, + pub scope: String, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeSummary { + pub id: NodeId, + pub display_name: String, + pub online: bool, + pub stale: bool, + pub last_seen_epoch_seconds: Option, + pub capabilities: NodeCapabilities, + pub direct_connectivity: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodePage { + pub nodes: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeRevocation { + pub node: NodeId, + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub descriptor_removed: bool, + pub queued_assignments_removed: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessLifecycleState { + Active, + RecentTerminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessActivityState { + Running, + WaitingForNode, + WaitingForTask, + AwaitingAction, + DebugEpochPartial, + Cancelling, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessFinalResult { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochSummary { + pub epoch: u64, + pub command: String, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSummary { + pub process: ProcessId, + pub lifecycle: ProcessLifecycleState, + pub activity: ProcessActivityState, + pub main_wait_state: Option, + pub started_at_epoch_seconds: u64, + pub ended_at_epoch_seconds: Option, + pub final_result: Option, + pub connected_nodes: Vec, + pub current_debug_epoch: Option, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessPage { + pub processes: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskTerminalState { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskExecutor { + CoordinatorMain, + Node, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCompletionEvent { + pub tenant: TenantId, + pub project: ProjectId, + pub process: ProcessId, + pub node: NodeId, + pub executor: TaskExecutor, + pub task_definition: TaskDefinitionId, + pub task: TaskInstanceId, + pub attempt_id: Option, + pub placement: Option, + pub terminal_state: TaskTerminalState, + pub status_code: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, + pub stdout_tail: String, + pub stderr_tail: String, + pub stdout_truncated: bool, + pub stderr_truncated: bool, + pub artifact_path: Option, + pub artifact_digest: Option, + pub artifact_size_bytes: Option, + pub result: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskAttemptState { + Queued, + Running, + FailedAwaitingAction, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskAttemptSnapshot { + pub process: ProcessId, + pub task: TaskInstanceId, + pub attempt_id: String, + pub attempt_number: u32, + pub task_definition: TaskDefinitionId, + pub display_name: String, + pub state: TaskAttemptState, + pub current: bool, + pub node: Option, + pub environment_id: Option, + pub environment_digest: Option, + pub argument_summary: Vec, + pub handle_summary: Vec, + pub command_state: Option, + pub vfs_checkpoint: String, + pub probe_symbol: Option, + pub source_path: Option, + pub source_line: Option, + pub restart_compatible: bool, + pub failure_policy: TaskFailurePolicy, + pub artifact_path: Option, + pub artifact_digest: Option, + pub artifact_size_bytes: Option, + pub status_code: Option, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskLogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogEntry { + pub sequence: u64, + pub process: ProcessId, + pub task: TaskInstanceId, + pub stream: TaskLogStream, + pub text: String, + pub server_timestamp_epoch_seconds: u64, + pub truncated: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogPage { + pub entries: Vec, + pub next_sequence: Option, + pub history_truncated: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactAvailability { + Available, + NodeOffline, + Unavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactRetentionState { + NodeRetained, + ExplicitStorage, + Lost, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactSummary { + pub id: ArtifactId, + pub display_path: String, + pub display_name: String, + pub process: ProcessId, + pub producer_task: TaskInstanceId, + pub safe_node: Option, + pub digest: Digest, + pub size_bytes: u64, + pub availability: ArtifactAvailability, + pub downloadable_now: bool, + pub retention_state: ArtifactRetentionState, + pub explicit_storage: bool, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactPage { + pub artifacts: Vec, + pub next_cursor: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct QuotaStatus { + pub tenant: TenantId, + pub project: ProjectId, + pub actor: UserId, + pub policy_label: Option, + pub limits: ResourceLimits, + pub window_seconds: BTreeMap, + pub usage: BTreeMap, + pub window_started_epoch_seconds: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskCancellationTarget { + pub process: ProcessId, + pub task: TaskInstanceId, + pub node: NodeId, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessCancellation { + pub process: ProcessId, + pub affected_tasks: Vec, + pub affected_nodes: Vec, + pub aborted: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugAuditEvent { + pub tenant: TenantId, + pub project: ProjectId, + pub process: ProcessId, + pub task: Option, + pub actor: UserId, + pub operation: String, + pub allowed: bool, + pub reason: String, + pub charged_debug_read_bytes: u64, + pub used_debug_read_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugAttach { + pub process: ProcessId, + pub actor: UserId, + pub authorization: Authorization, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochControl { + pub process: ProcessId, + pub actor: UserId, + pub epoch: u64, + pub command: String, + pub affected_tasks: Vec, + pub all_stop_requested: bool, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DebugAcknowledgementState { + Frozen, + Running, + Failed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugParticipantAcknowledgement { + pub node: NodeId, + pub task_definition: TaskDefinitionId, + pub task: TaskInstanceId, + pub epoch: u64, + pub state: DebugAcknowledgementState, + pub stack_frames: Vec, + pub local_values: Vec<(String, String)>, + pub task_args: Vec<(String, String)>, + pub handles: Vec<(String, String)>, + pub command_status: Option, + pub recent_output: Vec, + pub message: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochStatus { + pub process: ProcessId, + pub actor: UserId, + pub epoch: u64, + pub command: String, + pub expected_tasks: Vec, + pub acknowledgements: Vec, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, + pub failure_messages: Vec, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskRestart { + pub process: ProcessId, + pub task: TaskInstanceId, + pub restarted_task_instance: Option, + pub restarted_attempt_id: Option, + pub actor: UserId, + pub accepted: bool, + pub clean_boundary_available: bool, + pub active_task: bool, + pub completed_event_observed: bool, + pub requires_whole_process_restart: bool, + pub message: String, + pub audit_event: DebugAuditEvent, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskFailureResolutionResult { + pub process: ProcessId, + pub task: TaskInstanceId, + pub attempt_id: String, + pub resolution: TaskFailureResolution, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskFailureResolution { + AcceptFailure, + Cancel, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskReplacementBundle { + pub bundle_digest: Digest, + pub wasm_module_base64: String, + pub source_snapshot: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserLoginStart { + pub transaction_id: String, + pub authorization_url: String, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct SessionCredential(pub(crate) String); + +impl SessionCredential { + pub fn from_secret(secret: impl Into) -> Self { + Self(secret.into()) + } + + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for SessionCredential { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SessionCredential([REDACTED])") + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowserSession { + pub tenant: TenantId, + pub project: ProjectId, + pub user: UserId, + pub credential: SessionCredential, + pub expires_at_epoch_seconds: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArtifactDownloadPoll { + Pending, + Chunk { + offset: u64, + bytes: Vec, + eof: bool, + }, +} diff --git a/crates/clusterflux-client/tests/client_contract.rs b/crates/clusterflux-client/tests/client_contract.rs new file mode 100644 index 0000000..8851a54 --- /dev/null +++ b/crates/clusterflux-client/tests/client_contract.rs @@ -0,0 +1,254 @@ +use std::net::TcpListener; +use std::time::Duration; + +use clusterflux_client::{ + ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError, + ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId, + UserId, CLIENT_API_VERSION, +}; +use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; +use clusterflux_coordinator::service::CoordinatorService; +use serde_json::{json, Value}; + +#[tokio::test] +async fn mock_transport_exercises_typed_envelope_and_session_plumbing() { + let transport = MockTransport::from_json_responses([json!({ + "type": "projects", + "projects": [{ + "id": "project-one", + "tenant": "tenant-one", + "name": "Project one" + }], + "actor": "user-one" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport.clone()) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let projects = client.list_projects().await.unwrap(); + assert_eq!(projects.len(), 1); + assert_eq!(projects[0].id, ProjectId::from("project-one")); + + let requests = transport.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].api_path, CONTROL_API_PATH); + let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); + assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION); + assert_eq!(envelope["request_id"], "client-1"); + assert_eq!(envelope["operation"], "authenticated"); + assert_eq!(envelope["payload"]["type"], "authenticated"); + assert_eq!(envelope["payload"]["request"]["type"], "list_projects"); + assert_eq!(envelope["payload"]["session_secret"], "test-session-secret"); +} + +#[tokio::test] +async fn structured_errors_retain_machine_fields_and_originating_request_id() { + let transport = MockTransport::from_json_responses([json!({ + "type": "error", + "code": "account_suspended", + "category": "authorization", + "message": "account access is suspended", + "retryable": false, + "request_id": "client-1" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let ClientError::Api(error) = client.account_status().await.unwrap_err() else { + panic!("expected typed API error"); + }; + assert_eq!(error.code, ApiErrorCode::AccountSuspended); + assert_eq!(error.category, ApiErrorCategory::Authorization); + assert_eq!(error.request_id, "client-1"); + assert!(!error.retryable); +} + +#[tokio::test] +async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() { + let transport = MockTransport::from_json_responses([ + json!({ "type": "web_browser_login_cancelled" }).to_string(), + ]); + let client = ClusterfluxClient::with_transport(transport.clone()); + + client + .cancel_browser_login("login-transaction") + .await + .unwrap(); + + let requests = transport.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].api_path, LOGIN_API_PATH); + let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); + assert_eq!(envelope["operation"], "cancel_web_browser_login"); + assert_eq!( + envelope["payload"], + json!({ + "type": "cancel_web_browser_login", + "transaction_id": "login-transaction" + }) + ); +} + +#[tokio::test] +async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() { + let transport = MockTransport::from_json_responses([json!({ + "type": "error", + "code": "validation_error", + "category": "validation", + "message": "bad request", + "retryable": false, + "request_id": "another-request" + }) + .to_string()]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + + let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else { + panic!("expected protocol error"); + }; + assert!(message.contains("does not match client-1")); +} + +#[test] +fn session_credentials_are_redacted_from_debug_output() { + let credential = SessionCredential::from_secret("must-not-appear"); + assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])"); +} + +#[tokio::test] +async fn artifact_download_is_a_typed_bounded_chunk_stream() { + let link = json!({ + "artifact": "artifact-one", + "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "artifact_size_bytes": 5, + "source": { "RetainedNode": "node-one" }, + "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", + "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "expires_at_epoch_seconds": 1000, + "tenant": "tenant-one", + "project": "project-one", + "process": "process-one", + "actor": { "User": "user-one" }, + "max_bytes": 1024, + "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }); + let transport = MockTransport::from_json_responses([ + json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(), + json!({ + "type": "artifact_download_stream", + "link": link.clone(), + "streamed_bytes": 0, + "charged_download_bytes": 0, + "content_bytes_available": false, + "content_eof": false + }) + .to_string(), + json!({ + "type": "artifact_download_stream", + "link": link, + "streamed_bytes": 5, + "charged_download_bytes": 5, + "content_bytes_available": true, + "content_offset": 0, + "content_eof": true, + "content_base64": "aGVsbG8=" + }) + .to_string(), + ]); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("test-session-secret")); + let mut download = client + .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64) + .await + .unwrap(); + assert_eq!( + download.next_chunk().await.unwrap(), + ArtifactDownloadPoll::Pending + ); + assert_eq!( + download.next_chunk().await.unwrap(), + ArtifactDownloadPoll::Chunk { + offset: 0, + bytes: b"hello".to_vec(), + eof: true, + } + ); +} + +#[tokio::test] +async fn typed_client_runs_against_the_real_strict_control_endpoint() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let mut service = CoordinatorService::new(7); + service + .issue_cli_session( + TenantId::from("tenant-one"), + ProjectId::from("project-one"), + UserId::from("user-one"), + "real-endpoint-session", + None, + ) + .unwrap(); + let (stream, _) = listener.accept().unwrap(); + service.handle_stream(stream).unwrap(); + }); + + let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}")) + .unwrap() + .with_session_credential(&SessionCredential::from_secret("real-endpoint-session")); + let projects = client.list_projects().await.unwrap(); + assert_eq!(projects.len(), 1); + assert_eq!(projects[0].id, ProjectId::from("project-one")); + let status = client.account_status().await.unwrap(); + assert!(status.authenticated); + assert_eq!(status.actor, UserId::from("user-one")); + + drop(client); + server.join().unwrap(); +} + +#[tokio::test] +async fn concurrent_requests_expand_the_bounded_pool_without_serializing() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let handlers = (0..2) + .map(|_| { + let (stream, _) = listener.accept().unwrap(); + std::thread::spawn(move || { + let mut service = CoordinatorService::new(7); + service + .issue_cli_session( + TenantId::from("tenant-one"), + ProjectId::from("project-one"), + UserId::from("user-one"), + "concurrent-session", + None, + ) + .unwrap(); + service.handle_stream(stream).unwrap(); + }) + }) + .collect::>(); + for handler in handlers { + handler.join().unwrap(); + } + }); + + let transport = ControlTransport::with_timeouts( + format!("clusterflux+tcp://{address}"), + Duration::from_secs(2), + Duration::from_secs(5), + ) + .unwrap(); + let client = ClusterfluxClient::with_transport(transport) + .with_session_credential(&SessionCredential::from_secret("concurrent-session")); + let (first, second) = tokio::join!(client.account_status(), client.account_status()); + assert!(first.unwrap().authenticated); + assert!(second.unwrap().authenticated); + + drop(client); + server.join().unwrap(); +} diff --git a/crates/clusterflux-client/tests/fixtures/web_operations.json b/crates/clusterflux-client/tests/fixtures/web_operations.json new file mode 100644 index 0000000..45b7b23 --- /dev/null +++ b/crates/clusterflux-client/tests/fixtures/web_operations.json @@ -0,0 +1,200 @@ +[ + { + "operation": "begin_web_browser_login", + "boundary": "login", + "request": { "type": "begin_web_browser_login" }, + "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 } + }, + { + "operation": "cancel_web_browser_login", + "boundary": "login", + "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" }, + "response": { "type": "web_browser_login_cancelled" } + }, + { + "operation": "exchange_web_login_handoff", + "boundary": "login", + "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" }, + "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } } + }, + { + "operation": "auth_status", + "boundary": "control", + "request": { "type": "auth_status" }, + "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] } + }, + { + "operation": "revoke_cli_session", + "boundary": "control", + "request": { "type": "revoke_cli_session" }, + "response": { "type": "cli_session_revoked" } + }, + { + "operation": "create_project", + "boundary": "control", + "request": { "type": "create_project", "project": "project-new", "name": "New project" }, + "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } } + }, + { + "operation": "select_project", + "boundary": "control", + "request": { "type": "select_project", "project": "project-selected" }, + "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } } + }, + { + "operation": "list_projects", + "boundary": "control", + "request": { "type": "list_projects" }, + "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] } + }, + { + "operation": "register_agent_public_key", + "boundary": "control", + "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "list_agent_public_keys", + "boundary": "control", + "request": { "type": "list_agent_public_keys" }, + "response": { "type": "agent_public_keys", "records": [] } + }, + { + "operation": "rotate_agent_public_key", + "boundary": "control", + "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "revoke_agent_public_key", + "boundary": "control", + "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" }, + "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } + }, + { + "operation": "create_node_enrollment_grant", + "boundary": "control", + "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 }, + "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 } + }, + { + "operation": "list_node_summaries", + "boundary": "control", + "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 }, + "response": { "type": "node_summaries", "nodes": [], "next_cursor": null } + }, + { + "operation": "revoke_node_credential", + "boundary": "control", + "request": { "type": "revoke_node_credential", "node": "node-one" }, + "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 } + }, + { + "operation": "list_process_summaries", + "boundary": "control", + "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 }, + "response": { "type": "process_summaries", "processes": [], "next_cursor": null } + }, + { + "operation": "cancel_process", + "boundary": "control", + "request": { "type": "cancel_process", "process": "process-one" }, + "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] } + }, + { + "operation": "abort_process", + "boundary": "control", + "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null }, + "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] } + }, + { + "operation": "quota_status", + "boundary": "control", + "request": { "type": "quota_status" }, + "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } } + }, + { + "operation": "list_task_events", + "boundary": "control", + "request": { "type": "list_task_events", "process": "process-one" }, + "response": { "type": "task_events", "events": [] } + }, + { + "operation": "list_task_snapshots", + "boundary": "control", + "request": { "type": "list_task_snapshots", "process": "process-one" }, + "response": { "type": "task_snapshots", "snapshots": [] } + }, + { + "operation": "list_recent_logs", + "boundary": "control", + "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 }, + "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false } + }, + { + "operation": "restart_task", + "boundary": "control", + "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null }, + "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "resolve_task_failure", + "boundary": "control", + "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" }, + "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" } + }, + { + "operation": "debug_attach", + "boundary": "control", + "request": { "type": "debug_attach", "process": "process-one" }, + "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "create_debug_epoch", + "boundary": "control", + "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" }, + "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "resume_debug_epoch", + "boundary": "control", + "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 }, + "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "inspect_debug_epoch", + "boundary": "control", + "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 }, + "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } + }, + { + "operation": "list_artifacts", + "boundary": "control", + "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 }, + "response": { "type": "artifacts", "artifacts": [], "next_cursor": null } + }, + { + "operation": "get_artifact", + "boundary": "control", + "request": { "type": "get_artifact", "artifact": "artifact-one" }, + "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } } + }, + { + "operation": "create_artifact_download_link", + "boundary": "control", + "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 }, + "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } } + }, + { + "operation": "open_artifact_download_stream", + "boundary": "control", + "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 }, + "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" } + }, + { + "operation": "revoke_artifact_download_link", + "boundary": "control", + "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + "response": { "type": "artifact_download_link_revoked" } + } +] diff --git a/crates/clusterflux-control/src/lib.rs b/crates/clusterflux-control/src/lib.rs index 481e718..39a150d 100644 --- a/crates/clusterflux-control/src/lib.rs +++ b/crates/clusterflux-control/src/lib.rs @@ -61,7 +61,21 @@ impl ControlSession { endpoint: &str, api_path: &str, ) -> Result { - let session = Self::connect(endpoint)?; + Self::connect_to_api_path_with_timeouts( + endpoint, + api_path, + Duration::from_secs(10), + Duration::from_secs(30), + ) + } + + pub fn connect_to_api_path_with_timeouts( + endpoint: &str, + api_path: &str, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result { + let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?; #[cfg(not(target_arch = "wasm32"))] { let mut session = session; diff --git a/crates/clusterflux-coordinator/src/lib.rs b/crates/clusterflux-coordinator/src/lib.rs index 7776307..a110834 100644 --- a/crates/clusterflux-coordinator/src/lib.rs +++ b/crates/clusterflux-coordinator/src/lib.rs @@ -585,6 +585,22 @@ impl Coordinator { .count() } + pub fn tenant_count(&self) -> usize { + self.durable.tenants.len() + } + + pub fn user_count(&self) -> usize { + self.durable.users.len() + } + + pub fn project_count(&self) -> usize { + self.durable.projects.len() + } + + pub fn node_identity_count(&self) -> usize { + self.durable.node_identities.len() + } + pub fn node_identity( &self, tenant: &TenantId, @@ -1046,7 +1062,7 @@ mod tests { } #[test] - fn account_policy_state_summarizes_private_admin_records_safely() { + fn account_policy_state_summarizes_sensitive_admin_records_safely() { let tenant = TenantId::from("tenant"); let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1); diff --git a/crates/clusterflux-coordinator/src/main.rs b/crates/clusterflux-coordinator/src/main.rs index 8c9ce7d..8344f5f 100644 --- a/crates/clusterflux-coordinator/src/main.rs +++ b/crates/clusterflux-coordinator/src/main.rs @@ -5,17 +5,40 @@ use clusterflux_core::{ProjectId, TenantId, UserId}; use serde_json::json; fn main() -> Result<(), Box> { + let raw_args = std::env::args().skip(1).collect::>(); + match raw_args.as_slice() { + [flag] if matches!(flag.as_str(), "--version" | "-V") => { + println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + [flag] if matches!(flag.as_str(), "--help" | "-h") => { + println!( + "Clusterflux coordinator.\n\n\ + Usage: clusterflux-coordinator [OPTIONS]\n\n\ + Options:\n \ + --listen
[default: 127.0.0.1:0]\n \ + --allow-local-trusted-loopback\n \ + -h, --help\n \ + -V, --version" + ); + return Ok(()); + } + _ => {} + } + let mut listen = "127.0.0.1:0".to_owned(); let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK") .ok() .as_deref() == Some("1"); - let mut args = std::env::args().skip(1); + let mut args = raw_args.into_iter(); while let Some(arg) = args.next() { if arg == "--listen" { listen = args.next().ok_or("--listen requires an address")?; } else if arg == "--allow-local-trusted-loopback" { allow_local_trusted = true; + } else { + return Err(format!("unknown argument: {arg}").into()); } } diff --git a/crates/clusterflux-coordinator/src/service.rs b/crates/clusterflux-coordinator/src/service.rs index c1ea1f0..bab6c13 100644 --- a/crates/clusterflux-coordinator/src/service.rs +++ b/crates/clusterflux-coordinator/src/service.rs @@ -7,9 +7,10 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::time::{SystemTime, UNIX_EPOCH}; use clusterflux_core::{ - Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError, - NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId, - RateLimit, TenantId, TransportError, UserId, + Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry, + CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport, + NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit, + TenantId, TransportError, UserId, }; use thiserror::Error; @@ -34,6 +35,7 @@ mod quota; mod relay; mod routing; mod signed_nodes; +mod summaries; mod tcp; mod wire_protocol; use authorization::authorize_authenticated_user_operation; @@ -43,12 +45,14 @@ use keys::{ ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey, }; pub use protocol::{ - ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest, - CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent, - DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, - TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, - TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle, - TaskTerminalState, VirtualProcessStatus, WorkflowActor, + ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment, + AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, + DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement, + NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, + RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment, + TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent, + TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState, + VirtualProcessStatus, WorkflowActor, }; pub use quota::CoordinatorQuotaConfiguration; pub use relay::{ @@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128; const MAX_TASK_EVENTS_TOTAL: usize = 8_192; const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192; const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096; -const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096; +const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000; const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256; const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024; +const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256; +const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024; +const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024; +const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024; +const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32; +const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192; const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30; fn bounded_ttl(requested: u64, maximum: u64) -> u64 { requested.clamp(1, maximum) @@ -111,6 +121,24 @@ pub struct CoordinatorAdmission { pub max_artifact_download_ttl_seconds: u64, } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CoordinatorOperationalMetrics { + pub tenants: usize, + pub users: usize, + pub projects: usize, + pub enrolled_nodes: usize, + pub reported_nodes: usize, + pub live_nodes: usize, + pub active_processes: usize, + pub active_coordinator_mains: usize, + pub max_active_coordinator_mains: usize, + pub active_tasks: usize, + pub queued_tasks: usize, + pub artifacts: usize, + pub retained_download_links: usize, + pub relay: ArtifactRelayUsage, +} + impl Default for CoordinatorAdmission { fn default() -> Self { Self { @@ -151,6 +179,98 @@ pub enum CoordinatorServiceError { Durable(String), } +impl CoordinatorServiceError { + pub fn api_error(&self, request_id: impl Into) -> ApiError { + let request_id = request_id.into(); + let message = self.to_string(); + let (code, category, retryable) = match self { + Self::Io(_) => ( + ApiErrorCode::TemporaryCapacity, + ApiErrorCategory::Availability, + true, + ), + Self::Json(_) + | Self::CapabilityReport(_) + | Self::InvalidArtifactPath(_) + | Self::InvalidTaskLogTail(_) => ( + ApiErrorCode::ValidationError, + ApiErrorCategory::Validation, + false, + ), + Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => { + return ApiError::from_message(request_id, message); + } + Self::Coordinator(CoordinatorError::UnknownNode) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Coordinator(CoordinatorError::Enrollment(_)) => ( + ApiErrorCode::Unauthenticated, + ApiErrorCategory::Authentication, + false, + ), + Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => { + (ApiErrorCode::Conflict, ApiErrorCategory::State, true) + } + Self::Download(DownloadError::NotFound) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Download(DownloadError::Unavailable) + | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => ( + ApiErrorCode::ArtifactUnavailable, + ApiErrorCategory::Availability, + true, + ), + Self::Download(DownloadError::LimitExceeded { .. }) => ( + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCategory::Resource, + false, + ), + Self::Download(DownloadError::Unauthorized(_)) + | Self::Download(DownloadError::InvalidToken) + | Self::Download(DownloadError::Expired) + | Self::Download(DownloadError::Revoked) => ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ), + Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ), + Self::Scheduler(_) => ( + ApiErrorCode::NoCapableNode, + ApiErrorCategory::Availability, + true, + ), + Self::Transport(_) => ( + ApiErrorCode::NodeOffline, + ApiErrorCategory::Availability, + true, + ), + Self::Panel(PanelError::RateLimited) => ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ), + Self::Panel(PanelError::UnknownWidget(_)) => { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } + Self::Panel(_) => ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ), + Self::Durable(_) => ( + ApiErrorCode::InternalError, + ApiErrorCategory::Internal, + true, + ), + }; + ApiError::new(code, category, message, retryable, request_id) + } +} + pub struct CoordinatorService { coordinator: Coordinator, store: RuntimeDurableStore, @@ -161,6 +281,37 @@ pub struct CoordinatorService { enrollment_grants: BTreeMap, task_events: VecDeque, process_scope_history: VecDeque, + process_summaries: BTreeMap, + process_summary_order: VecDeque, + next_process_summary_order: u64, + task_terminal_states: BTreeMap, + recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque>, + recent_log_dropped_through: BTreeMap, + recent_log_accounted_bytes: BTreeMap< + ( + TenantId, + ProjectId, + ProcessId, + clusterflux_core::TaskInstanceId, + String, + ), + u64, + >, + recent_log_truncated_streams: BTreeSet<( + TenantId, + ProjectId, + ProcessId, + clusterflux_core::TaskInstanceId, + String, + )>, + recent_log_quota_truncated_streams: BTreeSet<( + TenantId, + ProjectId, + ProcessId, + clusterflux_core::TaskInstanceId, + String, + )>, + next_recent_log_sequence: u64, debug_audit_events: VecDeque, debug_epochs: BTreeMap, debug_epoch_runtime: BTreeMap, @@ -215,6 +366,29 @@ impl CoordinatorService { self.artifact_relay.usage() } + pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics { + CoordinatorOperationalMetrics { + tenants: self.coordinator.tenant_count(), + users: self.coordinator.user_count(), + projects: self.coordinator.project_count(), + enrolled_nodes: self.coordinator.node_identity_count(), + reported_nodes: self.node_descriptors.len(), + live_nodes: self + .node_descriptors + .keys() + .filter(|scope| self.node_is_live(scope)) + .count(), + active_processes: self.coordinator.active_process_count(), + active_coordinator_mains: self.main_runtime.active_main_count(), + max_active_coordinator_mains: self.main_runtime.max_active_mains(), + active_tasks: self.active_tasks.len(), + queued_tasks: self.pending_task_launches.len(), + artifacts: self.artifact_registry.artifact_count(), + retained_download_links: self.artifact_registry.retained_download_link_count(), + relay: self.artifact_relay.usage(), + } + } + fn commit_artifact_relay( &mut self, candidate: relay::ArtifactRelayLedger, @@ -404,6 +578,16 @@ impl CoordinatorService { enrollment_grants: BTreeMap::new(), task_events: VecDeque::new(), process_scope_history: VecDeque::new(), + process_summaries: BTreeMap::new(), + process_summary_order: VecDeque::new(), + next_process_summary_order: 1, + task_terminal_states: BTreeMap::new(), + recent_logs: BTreeMap::new(), + recent_log_dropped_through: BTreeMap::new(), + recent_log_accounted_bytes: BTreeMap::new(), + recent_log_truncated_streams: BTreeSet::new(), + recent_log_quota_truncated_streams: BTreeSet::new(), + next_recent_log_sequence: 1, debug_audit_events: VecDeque::new(), debug_epochs: BTreeMap::new(), debug_epoch_runtime: BTreeMap::new(), diff --git a/crates/clusterflux-coordinator/src/service/authorization.rs b/crates/clusterflux-coordinator/src/service/authorization.rs index 92c1e6d..f4d60fd 100644 --- a/crates/clusterflux-coordinator/src/service/authorization.rs +++ b/crates/clusterflux-coordinator/src/service/authorization.rs @@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation { RevokeAgentPublicKey, CreateNodeEnrollmentGrant, ListNodeDescriptors, + ListNodeSummaries, RevokeNodeCredential, StartProcess, ScheduleTask, @@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation { CancelProcess, AbortProcess, ListProcesses, + ListProcessSummaries, QuotaStatus, RestartTask, ResolveTaskFailure, @@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation { InspectDebugEpoch, ListTaskEvents, ListTaskSnapshots, + ListRecentLogs, JoinTask, + ListArtifacts, + GetArtifact, CreateArtifactDownloadLink, OpenArtifactDownloadStream, RevokeArtifactDownloadLink, @@ -56,6 +61,7 @@ impl PublicUserOperation { Self::RevokeAgentPublicKey => "revoke_agent_public_key", Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant", Self::ListNodeDescriptors => "list_node_descriptors", + Self::ListNodeSummaries => "list_node_summaries", Self::RevokeNodeCredential => "revoke_node_credential", Self::StartProcess => "start_process", Self::ScheduleTask => "schedule_task", @@ -63,6 +69,7 @@ impl PublicUserOperation { Self::CancelProcess => "cancel_process", Self::AbortProcess => "abort_process", Self::ListProcesses => "list_processes", + Self::ListProcessSummaries => "list_process_summaries", Self::QuotaStatus => "quota_status", Self::RestartTask => "restart_task", Self::ResolveTaskFailure => "resolve_task_failure", @@ -74,7 +81,10 @@ impl PublicUserOperation { Self::InspectDebugEpoch => "inspect_debug_epoch", Self::ListTaskEvents => "list_task_events", Self::ListTaskSnapshots => "list_task_snapshots", + Self::ListRecentLogs => "list_recent_logs", Self::JoinTask => "join_task", + Self::ListArtifacts => "list_artifacts", + Self::GetArtifact => "get_artifact", Self::CreateArtifactDownloadLink => "create_artifact_download_link", Self::OpenArtifactDownloadStream => "open_artifact_download_stream", Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link", @@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { Self::CreateNodeEnrollmentGrant } AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors, + AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries, AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => { Self::RevokeNodeCredential } @@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess, AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess, AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses, + AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => { + Self::ListProcessSummaries + } AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus, AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask, AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure, @@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch, AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents, AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots, + AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs, AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask, + AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts, + AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact, AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => { Self::CreateArtifactDownloadLink } diff --git a/crates/clusterflux-coordinator/src/service/logs.rs b/crates/clusterflux-coordinator/src/service/logs.rs index 79e4ade..1fb575b 100644 --- a/crates/clusterflux-coordinator/src/service/logs.rs +++ b/crates/clusterflux-coordinator/src/service/logs.rs @@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key}; use super::protocol::TaskAttemptState; use super::{ artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError, - TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES, + RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState, + MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES, + MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT, + MAX_TASK_LOG_TAIL_BYTES, }; impl CoordinatorService { @@ -36,23 +39,44 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; validate_task_log_tail("stdout_tail", &stdout_tail)?; validate_task_log_tail("stderr_tail", &stderr_tail)?; - let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - self.quota - .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; - self.quota - .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; + let stdout_retained = self.accept_final_log_stream( + &tenant, + &project, + &process, + &task, + TaskLogStream::Stdout, + stdout_bytes, + &stdout_tail, + stdout_truncated, + now_epoch_seconds, + )?; + let stderr_retained = self.accept_final_log_stream( + &tenant, + &project, + &process, + &task, + TaskLogStream::Stderr, + stderr_bytes, + &stderr_tail, + stderr_truncated, + now_epoch_seconds, + )?; Ok(CoordinatorResponse::TaskLogRecorded { process, task, stdout_bytes, stderr_bytes, - stdout_tail: if stdout_truncated { + stdout_tail: if !stdout_retained { + "[log output truncated at project log quota]".to_owned() + } else if stdout_truncated { format!("{stdout_tail}\n... truncated") } else { stdout_tail }, - stderr_tail: if stderr_truncated { + stderr_tail: if !stderr_retained { + "[log output truncated at project log quota]".to_owned() + } else if stderr_truncated { format!("{stderr_tail}\n... truncated") } else { stderr_tail @@ -61,6 +85,158 @@ impl CoordinatorService { }) } + #[allow(clippy::too_many_arguments)] + pub(super) fn handle_report_task_log_chunk( + &mut self, + tenant: String, + project: String, + process: String, + node: String, + task: String, + stream: TaskLogStream, + offset: u64, + source_bytes: u64, + text: String, + truncated: bool, + ) -> Result { + if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { + return Err(CoordinatorServiceError::InvalidTaskLogTail(format!( + "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}", + text.len() + ))); + } + if source_bytes == 0 && !text.is_empty() && !truncated { + return Err(CoordinatorServiceError::Protocol( + "live log chunk source_bytes must describe non-empty text".to_owned(), + )); + } + if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) { + return Err(CoordinatorServiceError::Protocol( + "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(), + )); + } + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let process = ProcessId::new(process); + let node = NodeId::new(node); + let task = TaskInstanceId::new(task); + self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; + let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream); + let expected = self + .recent_log_accounted_bytes + .get(&key) + .copied() + .unwrap_or(0); + let end = offset.checked_add(source_bytes).ok_or_else(|| { + CoordinatorServiceError::Protocol( + "live log chunk offset exceeds the supported range".to_owned(), + ) + })?; + let state_marker = source_bytes == 0 && truncated; + if end < expected || (end == expected && !state_marker) { + return Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence: None, + next_offset: expected, + }); + } + let now_epoch_seconds = self.current_epoch_seconds()?; + if self.recent_log_quota_truncated_streams.contains(&key) { + if end > expected { + self.recent_log_accounted_bytes.insert(key, end); + } + return Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence: None, + next_offset: end.max(expected), + }); + } + let newly_accounted = end.saturating_sub(expected); + if self + .quota + .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds) + .is_err() + { + if end > expected { + self.recent_log_accounted_bytes.insert(key.clone(), end); + } + let sequence = self.mark_log_quota_truncated( + &tenant, + &project, + &process, + &task, + &stream, + now_epoch_seconds, + ); + return Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence, + next_offset: end.max(expected), + }); + } + if offset > expected { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!("[log output lost: {} bytes]", offset - expected), + true, + now_epoch_seconds, + ); + } else if offset < expected { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!( + "[log output overlap omitted: {} new source bytes]", + end - expected + ), + true, + now_epoch_seconds, + ); + } + let marker_is_new = if truncated { + self.recent_log_truncated_streams.insert(key.clone()) + } else { + false + }; + let text = if state_marker && text.is_empty() { + "[log output truncated at source]".to_owned() + } else { + text + }; + let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new)) + .then(|| { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream, + text, + truncated, + now_epoch_seconds, + ) + }); + if end > expected { + self.recent_log_accounted_bytes.insert(key, end); + } + Ok(CoordinatorResponse::TaskLogChunkRecorded { + process, + task, + sequence, + next_offset: end, + }) + } + pub(super) fn handle_report_vfs_metadata( &mut self, tenant: String, @@ -179,20 +355,37 @@ impl CoordinatorService { artifact_size_bytes, result, }; - let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - self.quota.can_charge_log_bytes( + let stdout_retained = self.accept_final_log_stream( &event.tenant, &event.project, - reported_bytes, + &event.process, + &event.task, + TaskLogStream::Stdout, + event.stdout_bytes, + &event.stdout_tail, + event.stdout_truncated, now_epoch_seconds, )?; - self.quota.charge_log_bytes( + let stderr_retained = self.accept_final_log_stream( &event.tenant, &event.project, - reported_bytes, + &event.process, + &event.task, + TaskLogStream::Stderr, + event.stderr_bytes, + &event.stderr_tail, + event.stderr_truncated, now_epoch_seconds, )?; + if !stdout_retained { + event.stdout_tail = "[log output truncated at project log quota]".to_owned(); + event.stdout_truncated = true; + } + if !stderr_retained { + event.stderr_tail = "[log output truncated at project log quota]".to_owned(); + event.stderr_truncated = true; + } let task_key = task_control_key( &event.tenant, &event.project, @@ -221,6 +414,12 @@ impl CoordinatorService { self.task_aborts.remove(&task_key); self.debug_commands.remove(&task_key); self.active_tasks.remove(&task_key); + self.clear_recent_log_offsets_for_task( + &event.tenant, + &event.project, + &event.process, + &event.task, + ); self.task_assignments.retain(|_, assignments| { assignments.retain(|assignment| { assignment.tenant != event.tenant @@ -248,10 +447,19 @@ impl CoordinatorService { self.notify_coordinator_main_waiters(&event); } self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?; + let events_recorded = self + .task_events + .iter() + .filter(|recorded| { + recorded.tenant == event.tenant + && recorded.project == event.project + && recorded.process == event.process + }) + .count(); Ok(CoordinatorResponse::TaskRecorded { process: event.process, task: event.task, - events_recorded: self.task_events.len(), + events_recorded, }) } @@ -311,7 +519,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::TaskSnapshots { snapshots }) } - fn authorize_task_event_process_scope( + pub(super) fn authorize_task_event_process_scope( &self, tenant: &TenantId, project: &ProjectId, @@ -351,6 +559,47 @@ impl CoordinatorService { Ok(()) } + pub(super) fn handle_list_recent_logs( + &mut self, + tenant: String, + project: String, + actor_user: String, + process: String, + task: Option, + after_sequence: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let process = ProcessId::new(process); + let task = task.map(TaskInstanceId::new); + self.authorize_task_event_process_scope(&tenant, &project, &process)?; + let after_sequence = after_sequence.unwrap_or(0); + let retained = self.recent_logs.get(&(tenant.clone(), project.clone())); + let history_truncated = self + .recent_log_dropped_through + .get(&process_control_key(&tenant, &project, &process)) + .is_some_and(|dropped_through| *dropped_through > after_sequence); + let entries = retained + .into_iter() + .flatten() + .filter(|entry| { + entry.process == process + && entry.sequence > after_sequence + && task.as_ref().is_none_or(|task| &entry.task == task) + }) + .take(limit as usize) + .cloned() + .collect::>(); + let next_sequence = entries.last().map(|entry| entry.sequence); + Ok(CoordinatorResponse::RecentLogs { + entries, + next_sequence, + history_truncated, + }) + } + pub(super) fn handle_join_task( &mut self, tenant: String, @@ -469,6 +718,22 @@ impl CoordinatorService { pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) { event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated); event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated); + match event.executor { + super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state( + &event.tenant, + &event.project, + &event.process, + event.task_definition.clone(), + event.task.clone(), + event.terminal_state.clone(), + ), + super::TaskExecutor::Node => { + self.task_terminal_states.insert( + task_restart_key(&event.tenant, &event.project, &event.process, &event.task), + event.terminal_state.clone(), + ); + } + } let process_scope = ( event.tenant.clone(), event.project.clone(), @@ -506,6 +771,321 @@ impl CoordinatorService { self.task_events.push_back(event); } + #[allow(clippy::too_many_arguments)] + fn record_recent_log( + &mut self, + tenant: TenantId, + project: ProjectId, + process: ProcessId, + task: TaskInstanceId, + stream: TaskLogStream, + mut text: String, + mut truncated: bool, + server_timestamp_epoch_seconds: u64, + ) -> u64 { + if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { + let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES; + while !text.is_char_boundary(boundary) { + boundary -= 1; + } + text.truncate(boundary); + truncated = true; + } + let sequence = self.next_recent_log_sequence; + self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1); + let logs = self + .recent_logs + .entry((tenant.clone(), project.clone())) + .or_default(); + let mut dropped = Vec::new(); + while logs.iter().filter(|entry| entry.process == process).count() + >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS + { + let Some(index) = logs.iter().position(|entry| entry.process == process) else { + break; + }; + if let Some(entry) = logs.remove(index) { + dropped.push(entry); + } + } + while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT + || logs + .iter() + .map(|entry| entry.text.len()) + .sum::() + .saturating_add(text.len()) + > MAX_RECENT_LOG_BYTES_PER_PROJECT + { + match logs.pop_front() { + Some(entry) => dropped.push(entry), + None => break, + } + } + logs.push_back(RecentLogEntry { + sequence, + process, + task, + stream, + text, + server_timestamp_epoch_seconds, + truncated, + }); + for entry in dropped { + let key = process_control_key(&tenant, &project, &entry.process); + self.recent_log_dropped_through + .entry(key) + .and_modify(|dropped_through| { + *dropped_through = (*dropped_through).max(entry.sequence); + }) + .or_insert(entry.sequence); + } + sequence + } + + #[allow(clippy::too_many_arguments)] + fn accept_final_log_stream( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + stream: TaskLogStream, + total_source_bytes: u64, + final_tail: &str, + source_truncated: bool, + now_epoch_seconds: u64, + ) -> Result { + let key = recent_log_offset_key(tenant, project, process, task, &stream); + let accounted = self + .recent_log_accounted_bytes + .get(&key) + .copied() + .unwrap_or(0); + let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| { + let stream_name = match stream { + TaskLogStream::Stdout => "stdout", + TaskLogStream::Stderr => "stderr", + }; + CoordinatorServiceError::Protocol(format!( + "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted" + )) + })?; + if self.recent_log_quota_truncated_streams.contains(&key) { + self.recent_log_accounted_bytes + .insert(key, total_source_bytes); + return Ok(false); + } + if self + .quota + .charge_log_bytes(tenant, project, remaining, now_epoch_seconds) + .is_err() + { + self.recent_log_accounted_bytes + .insert(key, total_source_bytes); + self.mark_log_quota_truncated( + tenant, + project, + process, + task, + &stream, + now_epoch_seconds, + ); + return Ok(false); + } + self.reconcile_final_log_stream( + tenant, + project, + process, + task, + stream, + total_source_bytes, + final_tail, + source_truncated, + now_epoch_seconds, + ); + Ok(true) + } + + #[allow(clippy::too_many_arguments)] + fn mark_log_quota_truncated( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + stream: &TaskLogStream, + now_epoch_seconds: u64, + ) -> Option { + let key = recent_log_offset_key(tenant, project, process, task, stream); + if !self.recent_log_quota_truncated_streams.insert(key.clone()) { + return None; + } + self.recent_log_truncated_streams.insert(key); + Some(self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + "[log output truncated at project log quota]".to_owned(), + true, + now_epoch_seconds, + )) + } + + #[allow(clippy::too_many_arguments)] + fn reconcile_final_log_stream( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + stream: TaskLogStream, + total_source_bytes: u64, + final_tail: &str, + source_truncated: bool, + now_epoch_seconds: u64, + ) { + let key = recent_log_offset_key(tenant, project, process, task, &stream); + let accounted = self + .recent_log_accounted_bytes + .get(&key) + .copied() + .unwrap_or(0); + let mut visible_truncation = false; + if total_source_bytes > accounted { + let missing = total_source_bytes - accounted; + if final_tail.is_empty() { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!("[log output unavailable: {missing} source bytes]"), + true, + now_epoch_seconds, + ); + visible_truncation = true; + } else if (final_tail.len() as u64) <= total_source_bytes { + let tail_source_start = total_source_bytes - final_tail.len() as u64; + if accounted < tail_source_start { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!( + "[log output lost before final tail: {} source bytes]", + tail_source_start - accounted + ), + true, + now_epoch_seconds, + ); + visible_truncation = true; + } + let source_start = accounted.max(tail_source_start) - tail_source_start; + let mut byte_start = usize::try_from(source_start) + .unwrap_or(final_tail.len()) + .min(final_tail.len()); + while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) { + byte_start += 1; + } + let suffix = &final_tail[byte_start..]; + if !suffix.is_empty() { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + suffix.to_owned(), + source_truncated || visible_truncation, + now_epoch_seconds, + ); + visible_truncation |= source_truncated; + } + } else if accounted == 0 { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + final_tail.to_owned(), + source_truncated, + now_epoch_seconds, + ); + visible_truncation |= source_truncated; + } else { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream.clone(), + format!( + "[{missing} additional source bytes could not be merged without duplicating redacted output]" + ), + true, + now_epoch_seconds, + ); + visible_truncation = true; + } + self.recent_log_accounted_bytes + .insert(key.clone(), total_source_bytes); + } + + let marker_is_new = (source_truncated || visible_truncation) + && self.recent_log_truncated_streams.insert(key); + if marker_is_new && !visible_truncation { + self.record_recent_log( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + stream, + "[log output truncated at source]".to_owned(), + true, + now_epoch_seconds, + ); + } + } + + pub(super) fn clear_recent_log_offsets_for_task( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + ) { + self.recent_log_accounted_bytes.retain( + |(entry_tenant, entry_project, entry_process, entry_task, _), _| { + entry_tenant != tenant + || entry_project != project + || entry_process != process + || entry_task != task + }, + ); + self.recent_log_truncated_streams.retain( + |(entry_tenant, entry_project, entry_process, entry_task, _)| { + entry_tenant != tenant + || entry_project != project + || entry_process != process + || entry_task != task + }, + ); + self.recent_log_quota_truncated_streams.retain( + |(entry_tenant, entry_project, entry_process, entry_task, _)| { + entry_tenant != tenant + || entry_project != project + || entry_process != process + || entry_task != task + }, + ); + } + fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool { let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task); let Some(attempt) = self @@ -589,13 +1169,11 @@ impl CoordinatorService { return Ok(false); } - let main_completed = self.task_events.iter().rev().any(|event| { - &event.tenant == tenant - && &event.project == project - && &event.process == process - && matches!(event.executor, super::TaskExecutor::CoordinatorMain) - && matches!(event.terminal_state, TaskTerminalState::Completed) - }); + let main_completed = self + .process_summaries + .get(&process_key) + .and_then(|summary| summary.main_terminal_state.as_ref()) + .is_some_and(|state| matches!(state, TaskTerminalState::Completed)); let cancellation_completed = self.process_cancellations.contains(&process_key); if !main_completed && !cancellation_completed { return Ok(false); @@ -610,9 +1188,47 @@ impl CoordinatorService { { return Ok(false); } + let final_result = if cancellation_completed { + super::ProcessFinalResult::Cancelled + } else if self.task_terminal_states.iter().any( + |((task_tenant, task_project, task_process, _), terminal_state)| { + task_tenant == tenant + && task_project == project + && task_process == process + && terminal_state == &TaskTerminalState::Failed + }, + ) { + super::ProcessFinalResult::Failed + } else if self.task_terminal_states.iter().any( + |((task_tenant, task_project, task_process, _), terminal_state)| { + task_tenant == tenant + && task_project == project + && task_process == process + && terminal_state == &TaskTerminalState::Cancelled + }, + ) { + super::ProcessFinalResult::Cancelled + } else { + super::ProcessFinalResult::Completed + }; + self.record_process_terminal( + tenant, + project, + process, + final_result, + self.current_epoch_seconds()?, + ); self.coordinator.abort_process(tenant, project, process)?; self.clear_debug_state_for_process(tenant, project, process); self.clear_operator_panel_state(tenant, project, process); + let (pinned, protected_processes) = + self.artifact_retention_guards_for_project(tenant, project); + self.artifact_registry.enforce_project_metadata_limit( + tenant, + project, + &pinned, + &protected_processes, + ); Ok(true) } @@ -623,8 +1239,30 @@ impl CoordinatorService { let now_epoch_seconds = self.current_epoch_seconds()?; self.artifact_registry .expire_download_links(now_epoch_seconds); + let tenant = flush.tenant.clone(); + let project = flush.project.clone(); + let (pinned, protected_processes) = + self.artifact_retention_guards_for_project(&tenant, &project); + self.artifact_registry + .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes) + .map(|_| ()) + .map_err(CoordinatorServiceError::Protocol) + } + + fn artifact_retention_guards_for_project( + &self, + tenant: &TenantId, + project: &ProjectId, + ) -> ( + std::collections::BTreeSet, + std::collections::BTreeSet, + ) { let mut pinned = std::collections::BTreeSet::new(); for checkpoint in self.task_restart_checkpoints.values() { + if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project + { + continue; + } for artifact in &checkpoint.assignment.task_spec.required_artifacts { pinned.insert(ArtifactScopeKey::from_refs( &checkpoint.assignment.tenant, @@ -634,6 +1272,9 @@ impl CoordinatorService { } } for pending in &self.pending_task_launches { + if &pending.tenant != tenant || &pending.project != project { + continue; + } for artifact in &pending.task_spec.required_artifacts { pinned.insert(ArtifactScopeKey::from_refs( &pending.tenant, @@ -642,10 +1283,13 @@ impl CoordinatorService { )); } } - self.artifact_registry - .flush_metadata_bounded(flush, &pinned) - .map(|_| ()) - .map_err(CoordinatorServiceError::Protocol) + let protected_processes = self + .coordinator + .active_processes_for_project(tenant, project) + .into_iter() + .map(|process| process.id) + .collect(); + (pinned, protected_processes) } fn task_is_known_or_active( @@ -680,15 +1324,24 @@ impl CoordinatorService { } } -fn checked_reported_log_bytes( - stdout_bytes: u64, - stderr_bytes: u64, -) -> Result { - stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| { - CoordinatorServiceError::Protocol( - "reported task log byte counts exceed the supported range".to_owned(), - ) - }) +fn recent_log_offset_key( + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task: &TaskInstanceId, + stream: &TaskLogStream, +) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) { + ( + tenant.clone(), + project.clone(), + process.clone(), + task.clone(), + match stream { + TaskLogStream::Stdout => "stdout", + TaskLogStream::Stderr => "stderr", + } + .to_owned(), + ) } fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> { @@ -705,11 +1358,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String { if value.len() <= MAX_TASK_LOG_TAIL_BYTES { return value; } - let mut boundary = MAX_TASK_LOG_TAIL_BYTES; - while !value.is_char_boundary(boundary) { - boundary -= 1; + let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES; + while boundary < value.len() && !value.is_char_boundary(boundary) { + boundary += 1; } - value.truncate(boundary); + value.drain(..boundary); *truncated = true; value } diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index 735d2f1..30b81a8 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime { } impl CoordinatorMainRuntime { + pub(super) fn active_main_count(&self) -> usize { + self.controls.len() + } + + pub(super) fn max_active_mains(&self) -> usize { + self.max_active_mains + } + pub(super) fn configure( &mut self, configuration: super::CoordinatorMainRuntimeConfiguration, @@ -729,6 +737,13 @@ impl CoordinatorService { &process, "coordinator main launch failed admission or validation", ); + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Failed, + self.liveness_now_epoch_seconds(), + ); let _ = self.coordinator.abort_process(&tenant, &project, &process); } result @@ -992,6 +1007,13 @@ impl CoordinatorService { } } self.process_aborts.insert(process_key.clone()); + self.record_process_terminal( + &scope.tenant, + &scope.project, + &scope.process, + super::ProcessFinalResult::Failed, + self.liveness_now_epoch_seconds(), + ); let _ = self .coordinator .abort_process(&scope.tenant, &scope.project, &scope.process); diff --git a/crates/clusterflux-coordinator/src/service/nodes.rs b/crates/clusterflux-coordinator/src/service/nodes.rs index b7a7b4e..cedae30 100644 --- a/crates/clusterflux-coordinator/src/service/nodes.rs +++ b/crates/clusterflux-coordinator/src/service/nodes.rs @@ -300,8 +300,8 @@ impl CoordinatorService { node_scope, NodeDescriptor { id: node.clone(), - tenant, - project, + tenant: tenant.clone(), + project: project.clone(), capabilities, cached_environments: cached_environment_digests.into_iter().collect(), dependency_caches: dependency_cache_digests.into_iter().collect(), @@ -311,9 +311,14 @@ impl CoordinatorService { online, }, ); + let node_descriptors = self + .node_descriptors + .values() + .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project) + .count(); Ok(CoordinatorResponse::NodeCapabilitiesRecorded { node, - node_descriptors: self.node_descriptors.len(), + node_descriptors, }) } diff --git a/crates/clusterflux-coordinator/src/service/process_launch.rs b/crates/clusterflux-coordinator/src/service/process_launch.rs index 31fda49..81737c3 100644 --- a/crates/clusterflux-coordinator/src/service/process_launch.rs +++ b/crates/clusterflux-coordinator/src/service/process_launch.rs @@ -545,13 +545,22 @@ impl CoordinatorService { task_spec, wasm_module_base64, }); + let queued_tasks = self + .pending_task_launches + .iter() + .filter(|pending| { + pending.tenant == tenant + && pending.project == project + && pending.process == process + }) + .count(); return Ok(CoordinatorResponse::TaskQueued { process, task, actor, reason, charged_spawns, - queued_tasks: self.pending_task_launches.len(), + queued_tasks, }); } Err(err) => return Err(err.into()), @@ -662,7 +671,9 @@ impl CoordinatorService { )); }; self.task_attempts.remove(&removable); + self.task_terminal_states.remove(&removable); } + self.task_terminal_states.remove(&key); let attempts = self.task_attempts.entry(key).or_default(); for attempt in attempts.iter_mut() { attempt.current = false; diff --git a/crates/clusterflux-coordinator/src/service/processes.rs b/crates/clusterflux-coordinator/src/service/processes.rs index 54f2f32..646c607 100644 --- a/crates/clusterflux-coordinator/src/service/processes.rs +++ b/crates/clusterflux-coordinator/src/service/processes.rs @@ -382,6 +382,10 @@ impl CoordinatorService { || attempt_project != &project || attempt_process != &process }); + self.task_terminal_states + .retain(|(task_tenant, task_project, task_process, _), _| { + task_tenant != &tenant || task_project != &project || task_process != &process + }); self.restart_launches .retain(|(attempt_tenant, attempt_project, attempt_process, _)| { attempt_tenant != &tenant @@ -392,11 +396,12 @@ impl CoordinatorService { event.tenant != tenant || event.project != project || event.process != process }); let active = self.coordinator.start_process_for_launch_attempt( - tenant, - project, + tenant.clone(), + project.clone(), process.clone(), launch_attempt.map(clusterflux_core::LaunchAttemptId::new), ); + self.record_process_started(&tenant, &project, &process, now_epoch_seconds); Ok(CoordinatorResponse::ProcessStarted { process, launch_attempt: active @@ -513,6 +518,13 @@ impl CoordinatorService { } let process_key = process_control_key(&tenant, &project, &process); if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) { + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Cancelled, + self.current_epoch_seconds()?, + ); self.coordinator .abort_process(&tenant, &project, &process)?; self.clear_operator_panel_state(&tenant, &project, &process); @@ -598,6 +610,13 @@ impl CoordinatorService { } } + self.record_process_terminal( + &tenant, + &project, + &process, + super::ProcessFinalResult::Cancelled, + self.current_epoch_seconds()?, + ); if let Some(launch_attempt) = launch_attempt.as_ref() { self.coordinator.abort_process_for_launch_attempt( &tenant, @@ -656,10 +675,18 @@ impl CoordinatorService { .map(|active| { let process_key = process_control_key(&active.tenant, &active.project, &active.id); let main = self.main_runtime.controls.get(&process_key); + let stored = self.process_summaries.get(&process_key); + let stored_main_state = stored + .and_then(|summary| summary.main_terminal_state.as_ref()) + .map(|state| match state { + super::TaskTerminalState::Completed => "completed", + super::TaskTerminalState::Failed => "failed", + super::TaskTerminalState::Cancelled => "cancelled", + }); let state = if self.process_cancellations.contains(&process_key) { "cancelling" } else { - main.map_or("running", |main| main.state.as_str()) + "running" }; let main_wait_state = main.and_then(|main| { if main.state != "running" { @@ -684,9 +711,15 @@ impl CoordinatorService { VirtualProcessStatus { process: active.id, state: state.to_owned(), - main_task_definition: main.map(|main| main.task_definition.clone()), - main_task_instance: main.map(|main| main.task_instance.clone()), - main_state: main.map(|main| main.state.clone()), + main_task_definition: main.map(|main| main.task_definition.clone()).or_else( + || stored.and_then(|summary| summary.main_task_definition.clone()), + ), + main_task_instance: main + .map(|main| main.task_instance.clone()) + .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())), + main_state: main + .map(|main| main.state.clone()) + .or_else(|| stored_main_state.map(str::to_owned)), main_wait_state, main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()), connected_nodes: active.connected_nodes.into_iter().collect(), diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index edaa988..61faf98 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -150,6 +150,15 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, + ListNodeSummaries { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, RevokeNodeCredential { tenant: String, project: String, @@ -303,6 +312,15 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, + ListProcessSummaries { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, QuotaStatus { tenant: String, project: String, @@ -429,6 +447,19 @@ pub enum CoordinatorRequest { stderr_truncated: bool, backpressured: bool, }, + ReportTaskLogChunk { + tenant: String, + project: String, + process: String, + node: String, + task: String, + stream: TaskLogStream, + offset: u64, + source_bytes: u64, + text: String, + #[serde(default)] + truncated: bool, + }, ReportVfsMetadata { tenant: String, project: String, @@ -478,6 +509,18 @@ pub enum CoordinatorRequest { actor_user: String, process: String, }, + ListRecentLogs { + tenant: String, + project: String, + actor_user: String, + process: String, + #[serde(default)] + task: Option, + #[serde(default)] + after_sequence: Option, + #[serde(default = "default_log_page_limit")] + limit: u32, + }, JoinTask { tenant: String, project: String, @@ -510,6 +553,23 @@ pub enum CoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, + ListArtifacts { + tenant: String, + project: String, + actor_user: String, + #[serde(default)] + process: Option, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, + GetArtifact { + tenant: String, + project: String, + actor_user: String, + artifact: String, + }, OpenArtifactDownloadStream { tenant: String, project: String, @@ -666,6 +726,18 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } + CoordinatorRequest::ListNodeSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::ExchangeNodeEnrollmentGrant { tenant, project, @@ -901,6 +973,14 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res task, .. } + | CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + process, + node, + task, + .. + } | CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -979,6 +1059,23 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_user(actor_user, &format!("{path}.actor_user"))?; validate_process(process, &format!("{path}.process")) } + CoordinatorRequest::ListRecentLogs { + tenant, + project, + actor_user, + process, + task, + limit, + .. + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_process(process, &format!("{path}.process"))?; + if let Some(task) = task { + validate_task_instance(task, &format!("{path}.task"))?; + } + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::AbortProcess { tenant, project, @@ -1007,6 +1104,18 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user")) } + CoordinatorRequest::ListProcessSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 100) + } CoordinatorRequest::RestartTask { tenant, project, @@ -1080,6 +1189,20 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res validate_process(process, &format!("{path}.process"))?; validate_external_token(widget_id, &format!("{path}.widget_id"), 256) } + CoordinatorRequest::ListArtifacts { + tenant, + project, + actor_user, + process, + cursor, + limit, + } => { + validate_tenant_project(tenant, project, path)?; + validate_user(actor_user, &format!("{path}.actor_user"))?; + validate_optional_process(process.as_deref(), &format!("{path}.process"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } CoordinatorRequest::CreateArtifactDownloadLink { tenant, project, @@ -1100,6 +1223,12 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res actor_user, artifact, .. + } + | CoordinatorRequest::GetArtifact { + tenant, + project, + actor_user, + artifact, } => { validate_tenant_project(tenant, project, path)?; validate_user(actor_user, &format!("{path}.actor_user"))?; @@ -1133,6 +1262,14 @@ fn validate_authenticated_request( | AuthenticatedCoordinatorRequest::ListNodeDescriptors | AuthenticatedCoordinatorRequest::ListProcesses | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), + AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => { + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } + AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => { + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 100) + } AuthenticatedCoordinatorRequest::CreateProject { project, .. } | AuthenticatedCoordinatorRequest::SelectProject { project } => { validate_project(project, &format!("{path}.project")) @@ -1188,6 +1325,18 @@ fn validate_authenticated_request( | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { validate_process(process, &format!("{path}.process")) } + AuthenticatedCoordinatorRequest::ListRecentLogs { + process, + task, + limit, + .. + } => { + validate_process(process, &format!("{path}.process"))?; + if let Some(task) = task { + validate_task_instance(task, &format!("{path}.task"))?; + } + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { @@ -1205,9 +1354,19 @@ fn validate_authenticated_request( AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { validate_optional_process(process.as_deref(), &format!("{path}.process")) } + AuthenticatedCoordinatorRequest::ListArtifacts { + process, + cursor, + limit, + } => { + validate_optional_process(process.as_deref(), &format!("{path}.process"))?; + validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; + validate_page_limit(*limit, &format!("{path}.limit"), 200) + } AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } - | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } => { + | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } + | AuthenticatedCoordinatorRequest::GetArtifact { artifact } => { validate_artifact(artifact, &format!("{path}.artifact")) } AuthenticatedCoordinatorRequest::ExportArtifactToNode { @@ -1362,6 +1521,19 @@ fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), Stri value.map_or(Ok(()), |value| validate_process(value, path)) } +fn validate_optional_cursor(value: Option<&str>, path: &str) -> Result<(), String> { + value.map_or(Ok(()), |value| validate_external_token(value, path, 256)) +} + +fn validate_page_limit(value: u32, path: &str, maximum: u32) -> Result<(), String> { + if value == 0 || value > maximum { + return Err(format!( + "malformed pagination limit {path}: expected 1 through {maximum}, received {value}" + )); + } + Ok(()) +} + fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) } @@ -1686,6 +1858,12 @@ pub enum AuthenticatedCoordinatorRequest { ttl_seconds: u64, }, ListNodeDescriptors, + ListNodeSummaries { + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, RevokeNodeCredential { node: String, }, @@ -1722,6 +1900,12 @@ pub enum AuthenticatedCoordinatorRequest { launch_attempt: Option, }, ListProcesses, + ListProcessSummaries { + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, QuotaStatus, RestartTask { process: String, @@ -1766,6 +1950,15 @@ pub enum AuthenticatedCoordinatorRequest { ListTaskSnapshots { process: String, }, + ListRecentLogs { + process: String, + #[serde(default)] + task: Option, + #[serde(default)] + after_sequence: Option, + #[serde(default = "default_log_page_limit")] + limit: u32, + }, JoinTask { process: String, task: String, @@ -1776,6 +1969,17 @@ pub enum AuthenticatedCoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, + ListArtifacts { + #[serde(default)] + process: Option, + #[serde(default)] + cursor: Option, + #[serde(default = "default_page_limit")] + limit: u32, + }, + GetArtifact { + artifact: String, + }, OpenArtifactDownloadStream { artifact: String, max_bytes: u64, @@ -1798,6 +2002,14 @@ fn default_download_ttl_seconds() -> u64 { 900 } +fn default_page_limit() -> u32 { + 50 +} + +fn default_log_page_limit() -> u32 { + 100 +} + fn default_node_enrollment_ttl_seconds() -> u64 { 900 } diff --git a/crates/clusterflux-coordinator/src/service/protocol/responses.rs b/crates/clusterflux-coordinator/src/service/protocol/responses.rs index dbb3874..8245536 100644 --- a/crates/clusterflux-coordinator/src/service/protocol/responses.rs +++ b/crates/clusterflux-coordinator/src/service/protocol/responses.rs @@ -183,6 +183,121 @@ pub struct VirtualProcessStatus { pub coordinator_epoch: u64, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeSummary { + pub id: NodeId, + pub display_name: String, + pub online: bool, + pub stale: bool, + pub last_seen_epoch_seconds: Option, + pub capabilities: NodeCapabilities, + pub direct_connectivity: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessLifecycleState { + Active, + RecentTerminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessActivityState { + Running, + WaitingForNode, + WaitingForTask, + AwaitingAction, + DebugEpochPartial, + Cancelling, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessFinalResult { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DebugEpochSummary { + pub epoch: u64, + pub command: String, + pub fully_frozen: bool, + pub partially_frozen: bool, + pub fully_resumed: bool, + pub failed: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSummary { + pub process: ProcessId, + pub lifecycle: ProcessLifecycleState, + pub activity: ProcessActivityState, + pub main_wait_state: Option, + pub started_at_epoch_seconds: u64, + pub ended_at_epoch_seconds: Option, + pub final_result: Option, + pub connected_nodes: Vec, + pub current_debug_epoch: Option, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactAvailability { + Available, + NodeOffline, + Unavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactRetentionState { + NodeRetained, + ExplicitStorage, + Lost, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactSummary { + pub id: ArtifactId, + pub display_path: String, + pub display_name: String, + pub process: ProcessId, + pub producer_task: TaskInstanceId, + pub safe_node: Option, + pub digest: Digest, + pub size_bytes: u64, + pub availability: ArtifactAvailability, + pub downloadable_now: bool, + pub retention_state: ArtifactRetentionState, + pub explicit_storage: bool, + pub order_cursor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskLogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecentLogEntry { + pub sequence: u64, + pub process: ProcessId, + pub task: TaskInstanceId, + pub stream: TaskLogStream, + pub text: String, + pub server_timestamp_epoch_seconds: u64, + pub truncated: bool, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum SourcePreparationDisposition { Pending { reason: String }, @@ -213,7 +328,7 @@ pub enum CoordinatorResponse { manual_review: bool, sanitized_reason: Option, next_actions: Vec, - private_moderation_details_exposed: bool, + sensitive_moderation_details_exposed: bool, signup_failure_details_exposed: bool, }, AdminStatus { @@ -282,6 +397,11 @@ pub enum CoordinatorResponse { descriptors: Vec, actor: UserId, }, + NodeSummaries { + nodes: Vec, + next_cursor: Option, + actor: UserId, + }, NodeCredentialRevoked { node: NodeId, tenant: TenantId, @@ -373,6 +493,11 @@ pub enum CoordinatorResponse { processes: Vec, actor: UserId, }, + ProcessSummaries { + processes: Vec, + next_cursor: Option, + actor: UserId, + }, QuotaStatus { tenant: TenantId, project: ProjectId, @@ -483,6 +608,17 @@ pub enum CoordinatorResponse { stderr_tail: String, backpressured: bool, }, + TaskLogChunkRecorded { + process: ProcessId, + task: TaskInstanceId, + sequence: Option, + next_offset: u64, + }, + RecentLogs { + entries: Vec, + next_sequence: Option, + history_truncated: bool, + }, VfsMetadataRecorded { process: ProcessId, task: TaskInstanceId, @@ -519,6 +655,13 @@ pub enum CoordinatorResponse { ArtifactDownloadLink { link: DownloadLink, }, + Artifacts { + artifacts: Vec, + next_cursor: Option, + }, + Artifact { + artifact: ArtifactSummary, + }, ArtifactDownloadLinkRevoked { link: DownloadLink, }, @@ -543,6 +686,24 @@ pub enum CoordinatorResponse { artifact_size_bytes: u64, }, Error { - message: String, + #[serde(flatten)] + error: clusterflux_core::ApiError, }, } + +impl CoordinatorResponse { + pub fn error(request_id: impl Into, message: impl Into) -> Self { + Self::Error { + error: clusterflux_core::ApiError::from_message(request_id, message), + } + } + + pub fn service_error( + request_id: impl Into, + error: &crate::service::CoordinatorServiceError, + ) -> Self { + Self::Error { + error: error.api_error(request_id), + } + } +} diff --git a/crates/clusterflux-coordinator/src/service/quota.rs b/crates/clusterflux-coordinator/src/service/quota.rs index 6fcd90f..78b65ae 100644 --- a/crates/clusterflux-coordinator/src/service/quota.rs +++ b/crates/clusterflux-coordinator/src/service/quota.rs @@ -260,22 +260,6 @@ impl CoordinatorQuota { self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds) } - pub(super) fn can_charge_log_bytes( - &self, - tenant: &TenantId, - project: &ProjectId, - bytes: u64, - now_epoch_seconds: u64, - ) -> Result<(), LimitError> { - self.can_charge( - tenant, - project, - LimitKind::LogBytes, - bytes, - now_epoch_seconds, - ) - } - pub(super) fn charge_log_bytes( &mut self, tenant: &TenantId, diff --git a/crates/clusterflux-coordinator/src/service/relay.rs b/crates/clusterflux-coordinator/src/service/relay.rs index cbe92dd..b79db3d 100644 --- a/crates/clusterflux-coordinator/src/service/relay.rs +++ b/crates/clusterflux-coordinator/src/service/relay.rs @@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage { pub egress_bytes: u64, pub abandoned_or_failed_bytes: u64, pub reserved_bytes: u64, + pub lifetime_ingress_bytes: u64, + pub lifetime_egress_bytes: u64, + pub lifetime_abandoned_or_failed_bytes: u64, + pub completed_transfers: u64, + pub failed_transfers: u64, + pub cancelled_transfers: u64, + pub expired_transfers: u64, + pub tracked_scopes: usize, + pub max_active_global: usize, + pub max_tracked_scopes: usize, } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState { pub ingress_used: u64, pub egress_used: u64, pub abandoned_or_failed_used: u64, + #[serde(default)] + pub lifetime_ingress_bytes: u64, + #[serde(default)] + pub lifetime_egress_bytes: u64, + #[serde(default)] + pub lifetime_abandoned_or_failed_bytes: u64, + #[serde(default)] + pub completed_transfers: u64, + #[serde(default)] + pub failed_transfers: u64, + #[serde(default)] + pub cancelled_transfers: u64, + #[serde(default)] + pub expired_transfers: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger { ingress_used: u64, egress_used: u64, abandoned_or_failed_used: u64, + lifetime_ingress_bytes: u64, + lifetime_egress_bytes: u64, + lifetime_abandoned_or_failed_bytes: u64, + completed_transfers: u64, + failed_transfers: u64, + cancelled_transfers: u64, + expired_transfers: u64, } impl Default for ArtifactRelayLedger { @@ -165,6 +196,13 @@ impl ArtifactRelayLedger { ingress_used: 0, egress_used: 0, abandoned_or_failed_used: 0, + lifetime_ingress_bytes: 0, + lifetime_egress_bytes: 0, + lifetime_abandoned_or_failed_bytes: 0, + completed_transfers: 0, + failed_transfers: 0, + cancelled_transfers: 0, + expired_transfers: 0, } } @@ -218,6 +256,13 @@ impl ArtifactRelayLedger { ingress_used: state.ingress_used, egress_used: state.egress_used, abandoned_or_failed_used: state.abandoned_or_failed_used, + lifetime_ingress_bytes: state.lifetime_ingress_bytes, + lifetime_egress_bytes: state.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes, + completed_transfers: state.completed_transfers, + failed_transfers: state.failed_transfers, + cancelled_transfers: state.cancelled_transfers, + expired_transfers: state.expired_transfers, } } @@ -260,6 +305,13 @@ impl ArtifactRelayLedger { ingress_used: self.ingress_used, egress_used: self.egress_used, abandoned_or_failed_used: self.abandoned_or_failed_used, + lifetime_ingress_bytes: self.lifetime_ingress_bytes, + lifetime_egress_bytes: self.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, + completed_transfers: self.completed_transfers, + failed_transfers: self.failed_transfers, + cancelled_transfers: self.cancelled_transfers, + expired_transfers: self.expired_transfers, } } @@ -491,9 +543,11 @@ impl ArtifactRelayLedger { if ingress { reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes); self.ingress_used = self.ingress_used.saturating_add(bytes); + self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes); } else { reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes); self.egress_used = self.egress_used.saturating_add(bytes); + self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes); } let project_key = ( reservation.scope.tenant.clone(), @@ -554,10 +608,29 @@ impl ArtifactRelayLedger { pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) { if let Some(reservation) = self.reservations.remove(key) { if reason != RelayFinishReason::Completed { + let abandoned_or_failed_bytes = reservation + .ingress_bytes + .saturating_add(reservation.egress_bytes); self.abandoned_or_failed_used = self .abandoned_or_failed_used - .saturating_add(reservation.ingress_bytes) - .saturating_add(reservation.egress_bytes); + .saturating_add(abandoned_or_failed_bytes); + self.lifetime_abandoned_or_failed_bytes = self + .lifetime_abandoned_or_failed_bytes + .saturating_add(abandoned_or_failed_bytes); + } + match reason { + RelayFinishReason::Completed => { + self.completed_transfers = self.completed_transfers.saturating_add(1); + } + RelayFinishReason::Failed => { + self.failed_transfers = self.failed_transfers.saturating_add(1); + } + RelayFinishReason::Cancelled => { + self.cancelled_transfers = self.cancelled_transfers.saturating_add(1); + } + RelayFinishReason::Expired => { + self.expired_transfers = self.expired_transfers.saturating_add(1); + } } } } @@ -580,6 +653,18 @@ impl ArtifactRelayLedger { ingress_bytes: self.ingress_used, egress_bytes: self.egress_used, abandoned_or_failed_bytes: self.abandoned_or_failed_used, + lifetime_ingress_bytes: self.lifetime_ingress_bytes, + lifetime_egress_bytes: self.lifetime_egress_bytes, + lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, + completed_transfers: self.completed_transfers, + failed_transfers: self.failed_transfers, + cancelled_transfers: self.cancelled_transfers, + expired_transfers: self.expired_transfers, + tracked_scopes: self.project_used.len() + + self.tenant_used.len() + + self.account_used.len(), + max_active_global: self.configuration.max_active_global, + max_tracked_scopes: self.configuration.max_tracked_scopes, reserved_bytes: self .reservations .values() @@ -588,3 +673,68 @@ impl ArtifactRelayLedger { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() { + let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited()); + ledger + .reserve( + "transfer".to_owned(), + TenantId::from("tenant"), + ProjectId::from("project"), + UserId::from("user"), + 16, + 16, + 100, + 1, + ) + .unwrap(); + ledger.charge_ingress("transfer", 24, 1).unwrap(); + ledger.charge_egress("transfer", 24, 1).unwrap(); + ledger.finish("transfer", RelayFinishReason::Completed); + + let usage = ledger.usage(); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + assert_eq!(usage.completed_transfers, 1); + + ledger.prepare_period(u64::MAX); + let usage = ledger.usage(); + assert_eq!(usage.ingress_bytes, 0); + assert_eq!(usage.egress_bytes, 0); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + + let restored = ArtifactRelayLedger::from_durable( + ArtifactRelayConfiguration::unlimited(), + ledger.durable_state(), + ); + let usage = restored.usage(); + assert_eq!(usage.lifetime_ingress_bytes, 24); + assert_eq!(usage.lifetime_egress_bytes, 24); + assert_eq!(usage.completed_transfers, 1); + } + + #[test] + fn older_durable_relay_state_defaults_new_metrics() { + let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({ + "reservations": {}, + "period": 1, + "project_used": [], + "tenant_used": [], + "account_used": [], + "ingress_used": 3, + "egress_used": 4, + "abandoned_or_failed_used": 2 + })) + .unwrap(); + + assert_eq!(state.lifetime_ingress_bytes, 0); + assert_eq!(state.lifetime_egress_bytes, 0); + assert_eq!(state.completed_transfers, 0); + } +} diff --git a/crates/clusterflux-coordinator/src/service/routing.rs b/crates/clusterflux-coordinator/src/service/routing.rs index d57af3f..e05a670 100644 --- a/crates/clusterflux-coordinator/src/service/routing.rs +++ b/crates/clusterflux-coordinator/src/service/routing.rs @@ -45,7 +45,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - private_moderation_details_exposed: false, + sensitive_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -298,6 +298,13 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_node_descriptors(tenant, project, actor_user), + CoordinatorRequest::ListNodeSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::RevokeNodeCredential { tenant, project, @@ -421,6 +428,13 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_processes(tenant, project, actor_user), + CoordinatorRequest::ListProcessSummaries { + tenant, + project, + actor_user, + cursor, + limit, + } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::QuotaStatus { tenant, project, @@ -466,7 +480,8 @@ impl CoordinatorService { CoordinatorRequest::PollDebugCommand { .. } | CoordinatorRequest::ReportDebugState { .. } | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(), - CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(), + CoordinatorRequest::ReportTaskLog { .. } + | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ListTaskEvents { @@ -481,6 +496,23 @@ impl CoordinatorService { actor_user, process, } => self.handle_list_task_snapshots(tenant, project, actor_user, process), + CoordinatorRequest::ListRecentLogs { + tenant, + project, + actor_user, + process, + task, + after_sequence, + limit, + } => self.handle_list_recent_logs( + tenant, + project, + actor_user, + process, + task, + after_sequence, + limit, + ), CoordinatorRequest::JoinTask { tenant, project, @@ -527,6 +559,20 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), + CoordinatorRequest::ListArtifacts { + tenant, + project, + actor_user, + process, + cursor, + limit, + } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit), + CoordinatorRequest::GetArtifact { + tenant, + project, + actor_user, + artifact, + } => self.handle_get_artifact(tenant, project, actor_user, artifact), CoordinatorRequest::OpenArtifactDownloadStream { tenant, project, @@ -609,7 +655,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - private_moderation_details_exposed: false, + sensitive_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -696,6 +742,14 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), + AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self + .handle_list_node_summaries( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + cursor, + limit, + ), AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self .handle_revoke_node_credential( context.tenant.as_str().to_owned(), @@ -747,6 +801,14 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), + AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self + .handle_list_process_summaries( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + cursor, + limit, + ), AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -802,6 +864,20 @@ impl CoordinatorService { actor.as_str().to_owned(), process, ), + AuthenticatedCoordinatorRequest::ListRecentLogs { + process, + task, + after_sequence, + limit, + } => self.handle_list_recent_logs( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + process, + task, + after_sequence, + limit, + ), AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -821,6 +897,24 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), + AuthenticatedCoordinatorRequest::ListArtifacts { + process, + cursor, + limit, + } => self.handle_list_artifacts( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + process, + cursor, + limit, + ), + AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + actor.as_str().to_owned(), + artifact, + ), AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, max_bytes, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index 19384df..40f633a 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -253,6 +253,29 @@ impl CoordinatorService { stderr_truncated, backpressured, ), + CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + process, + node, + task, + stream, + offset, + source_bytes, + text, + truncated, + } => self.handle_report_task_log_chunk( + tenant, + project, + process, + node, + task, + stream, + offset, + source_bytes, + text, + truncated, + ), CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -346,6 +369,7 @@ fn signed_node_request_kind( CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"), CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"), CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"), + CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"), CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"), CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"), _ => Err(CoordinatorError::Unauthorized( @@ -441,6 +465,12 @@ fn signed_node_request_scope( node, .. } + | CoordinatorRequest::ReportTaskLogChunk { + tenant, + project, + node, + .. + } | CoordinatorRequest::ReportVfsMetadata { tenant, project, diff --git a/crates/clusterflux-coordinator/src/service/summaries.rs b/crates/clusterflux-coordinator/src/service/summaries.rs new file mode 100644 index 0000000..1e5ad92 --- /dev/null +++ b/crates/clusterflux-coordinator/src/service/summaries.rs @@ -0,0 +1,565 @@ +use std::collections::BTreeSet; +use std::time::Instant; + +use clusterflux_core::{ + ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, + TenantId, UserId, +}; + +use super::keys::{process_control_key, ProcessControlKey}; +use super::{ + ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse, + CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary, + NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, + TaskAttemptState, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct StoredProcessSummary { + pub(super) started_at_epoch_seconds: u64, + pub(super) ended_at_epoch_seconds: Option, + pub(super) final_result: Option, + pub(super) connected_nodes: Vec, + pub(super) main_task_definition: Option, + pub(super) main_task_instance: Option, + pub(super) main_terminal_state: Option, + pub(super) order: u64, +} + +impl CoordinatorService { + pub(super) fn handle_list_node_summaries( + &mut self, + tenant: String, + project: String, + actor_user: String, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let actor = UserId::new(actor_user); + let cursor = cursor.as_deref(); + let mut nodes = self + .node_descriptors + .iter() + .filter(|(scope, descriptor)| { + scope.tenant == tenant + && scope.project == project + && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor) + }) + .map(|(scope, descriptor)| { + let online = self.node_is_live(scope); + NodeSummary { + id: descriptor.id.clone(), + display_name: descriptor.id.as_str().to_owned(), + online, + stale: !online, + last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(), + capabilities: descriptor.capabilities.clone(), + direct_connectivity: descriptor.direct_connectivity, + } + }) + .collect::>(); + nodes.sort_by(|left, right| left.id.cmp(&right.id)); + let has_more = nodes.len() > limit as usize; + nodes.truncate(limit as usize); + let next_cursor = has_more + .then(|| nodes.last().map(|node| node.id.as_str().to_owned())) + .flatten(); + Ok(CoordinatorResponse::NodeSummaries { + nodes, + next_cursor, + actor, + }) + } + + pub(super) fn record_process_started( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + now_epoch_seconds: u64, + ) { + let key = process_control_key(tenant, project, process); + if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) { + logs.retain(|entry| &entry.process != process); + if logs.is_empty() { + self.recent_logs.remove(&(tenant.clone(), project.clone())); + } + } + self.recent_log_dropped_through.remove(&key); + self.recent_log_accounted_bytes.retain( + |(entry_tenant, entry_project, entry_process, _, _), _| { + entry_tenant != tenant || entry_project != project || entry_process != process + }, + ); + self.recent_log_truncated_streams.retain( + |(entry_tenant, entry_project, entry_process, _, _)| { + entry_tenant != tenant || entry_project != project || entry_process != process + }, + ); + self.recent_log_quota_truncated_streams.retain( + |(entry_tenant, entry_project, entry_process, _, _)| { + entry_tenant != tenant || entry_project != project || entry_process != process + }, + ); + self.process_summary_order + .retain(|retained| retained != &key); + self.evict_process_summaries_for_project(tenant, project); + self.evict_process_summaries_total(); + let order = self.next_process_summary_order; + self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); + self.process_summaries.insert( + key.clone(), + StoredProcessSummary { + started_at_epoch_seconds: now_epoch_seconds, + ended_at_epoch_seconds: None, + final_result: None, + connected_nodes: Vec::new(), + main_task_definition: None, + main_task_instance: None, + main_terminal_state: None, + order, + }, + ); + self.process_summary_order.push_back(key); + } + + pub(super) fn record_process_terminal( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + final_result: ProcessFinalResult, + now_epoch_seconds: u64, + ) { + let key = process_control_key(tenant, project, process); + let connected_nodes = self + .coordinator + .active_process(tenant, project, process) + .map(|active| active.connected_nodes.iter().cloned().collect()) + .unwrap_or_default(); + let order = self.next_process_summary_order; + let entry = self + .process_summaries + .entry(key.clone()) + .or_insert_with(|| { + self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); + self.process_summary_order.push_back(key.clone()); + StoredProcessSummary { + started_at_epoch_seconds: now_epoch_seconds, + ended_at_epoch_seconds: None, + final_result: None, + connected_nodes: Vec::new(), + main_task_definition: None, + main_task_instance: None, + main_terminal_state: None, + order, + } + }); + entry.ended_at_epoch_seconds = Some(now_epoch_seconds); + entry.final_result = Some(final_result); + entry.connected_nodes = connected_nodes; + } + + pub(super) fn record_main_terminal_state( + &mut self, + tenant: &TenantId, + project: &ProjectId, + process: &ProcessId, + task_definition: TaskDefinitionId, + task_instance: TaskInstanceId, + terminal_state: super::TaskTerminalState, + ) { + let key = process_control_key(tenant, project, process); + if !self.process_summaries.contains_key(&key) { + self.record_process_started( + tenant, + project, + process, + self.liveness_now_epoch_seconds(), + ); + } + if let Some(summary) = self.process_summaries.get_mut(&key) { + summary.main_task_definition = Some(task_definition); + summary.main_task_instance = Some(task_instance); + summary.main_terminal_state = Some(terminal_state); + } + } + + pub(super) fn handle_list_process_summaries( + &mut self, + tenant: String, + project: String, + actor_user: String, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let actor = UserId::new(actor_user); + let cursor = parse_order_cursor(cursor.as_deref(), "process")?; + let mut stored = self + .process_summaries + .iter() + .filter(|((entry_tenant, entry_project, _), summary)| { + entry_tenant == &tenant + && entry_project == &project + && cursor.is_none_or(|cursor| summary.order < cursor) + }) + .map(|(key, summary)| (key.clone(), summary.clone())) + .collect::>(); + stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order)); + let has_more = stored.len() > limit as usize; + stored.truncate(limit as usize); + let processes = stored + .into_iter() + .map(|(key, stored)| self.process_summary_from_stored(&key, stored)) + .collect::>(); + let next_cursor = has_more + .then(|| processes.last().map(|process| process.order_cursor.clone())) + .flatten(); + Ok(CoordinatorResponse::ProcessSummaries { + processes, + next_cursor, + actor, + }) + } + + fn process_summary_from_stored( + &self, + key: &ProcessControlKey, + stored: StoredProcessSummary, + ) -> ProcessSummary { + let (tenant, project, process) = key; + let active = self.coordinator.active_process(tenant, project, process); + let process_key = process_control_key(tenant, project, process); + let main_wait_state = active.and_then(|_| { + if self.pending_task_launches.iter().any(|pending| { + &pending.tenant == tenant + && &pending.project == project + && &pending.process == process + }) { + Some("waiting_for_node".to_owned()) + } else if self + .main_runtime + .is_waiting_for_task(tenant, project, process) + { + Some("waiting_for_task".to_owned()) + } else { + None + } + }); + let current_debug_epoch = self.debug_epoch_summary(&process_key); + let awaiting_action = self.task_attempts.iter().any( + |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { + attempt_tenant == tenant + && attempt_project == project + && attempt_process == process + && attempts.iter().any(|attempt| { + attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction + }) + }, + ); + let activity = if let Some(result) = &stored.final_result { + match result { + ProcessFinalResult::Completed => ProcessActivityState::Completed, + ProcessFinalResult::Failed => ProcessActivityState::Failed, + ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled, + } + } else if self.process_cancellations.contains(&process_key) { + ProcessActivityState::Cancelling + } else if current_debug_epoch + .as_ref() + .is_some_and(|epoch| epoch.partially_frozen) + { + ProcessActivityState::DebugEpochPartial + } else if awaiting_action { + ProcessActivityState::AwaitingAction + } else { + match main_wait_state.as_deref() { + Some("waiting_for_node") => ProcessActivityState::WaitingForNode, + Some("waiting_for_task") => ProcessActivityState::WaitingForTask, + _ => ProcessActivityState::Running, + } + }; + let connected_nodes = active + .map(|active| active.connected_nodes.iter().cloned().collect()) + .unwrap_or(stored.connected_nodes); + ProcessSummary { + process: process.clone(), + lifecycle: if active.is_some() { + ProcessLifecycleState::Active + } else { + ProcessLifecycleState::RecentTerminal + }, + activity, + main_wait_state, + started_at_epoch_seconds: stored.started_at_epoch_seconds, + ended_at_epoch_seconds: stored.ended_at_epoch_seconds, + final_result: stored.final_result, + connected_nodes, + current_debug_epoch, + order_cursor: format!("process:{}", stored.order), + } + } + + fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option { + let runtime = self.debug_epoch_runtime.get(key)?; + let acknowledgements = runtime.acknowledgements.values().collect::>(); + let all_acknowledged = !runtime.expected.is_empty() + && runtime + .expected + .iter() + .all(|participant| runtime.acknowledgements.contains_key(participant)); + let fully_frozen = runtime.command == "freeze" + && all_acknowledged + && acknowledgements + .iter() + .all(|ack| ack.state == DebugAcknowledgementState::Frozen); + let freeze_deadline_elapsed = + runtime.command == "freeze" && Instant::now() >= runtime.deadline; + let frozen_count = acknowledgements + .iter() + .filter(|ack| ack.state == DebugAcknowledgementState::Frozen) + .count(); + let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen; + let fully_resumed = runtime.command == "resume" + && all_acknowledged + && acknowledgements + .iter() + .all(|ack| ack.state == DebugAcknowledgementState::Running); + let failed = acknowledgements + .iter() + .any(|ack| ack.state == DebugAcknowledgementState::Failed) + || (freeze_deadline_elapsed + && runtime + .expected + .iter() + .any(|participant| !runtime.acknowledgements.contains_key(participant))); + Some(DebugEpochSummary { + epoch: runtime.epoch, + command: runtime.command.clone(), + fully_frozen, + partially_frozen, + fully_resumed, + failed, + }) + } + + pub(super) fn handle_list_artifacts( + &mut self, + tenant: String, + project: String, + actor_user: String, + process: Option, + cursor: Option, + limit: u32, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let process = process.map(ProcessId::new); + if let Some(process) = &process { + self.authorize_task_event_process_scope(&tenant, &project, process)?; + } + let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?; + let mut metadata = self + .artifact_registry + .metadata_for_project(&tenant, &project) + .filter(|metadata| { + process + .as_ref() + .is_none_or(|process| &metadata.process == process) + && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor) + }) + .cloned() + .collect::>(); + metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch)); + let has_more = metadata.len() > limit as usize; + metadata.truncate(limit as usize); + let artifacts = metadata + .into_iter() + .map(|metadata| self.artifact_summary(metadata)) + .collect::>(); + let next_cursor = has_more + .then(|| { + artifacts + .last() + .map(|artifact| artifact.order_cursor.clone()) + }) + .flatten(); + Ok(CoordinatorResponse::Artifacts { + artifacts, + next_cursor, + }) + } + + pub(super) fn handle_get_artifact( + &mut self, + tenant: String, + project: String, + actor_user: String, + artifact: String, + ) -> Result { + let tenant = TenantId::new(tenant); + let project = ProjectId::new(project); + let _actor = UserId::new(actor_user); + let artifact = ArtifactId::new(artifact); + let metadata = self + .artifact_registry + .metadata(&tenant, &project, &artifact) + .cloned() + .ok_or(clusterflux_core::DownloadError::NotFound)?; + Ok(CoordinatorResponse::Artifact { + artifact: self.artifact_summary(metadata), + }) + } + + fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary { + let live_retaining_nodes = metadata + .retaining_nodes + .iter() + .filter(|node| { + self.node_is_live(&crate::NodeScopeKey::from_refs( + &metadata.tenant, + &metadata.project, + node, + )) + }) + .cloned() + .collect::>(); + let safe_node = live_retaining_nodes + .iter() + .next() + .cloned() + .or_else(|| metadata.retaining_nodes.iter().next().cloned()); + let explicit_storage = !metadata.explicit_locations.is_empty(); + let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage; + let availability = if downloadable_now { + ArtifactAvailability::Available + } else if !metadata.retaining_nodes.is_empty() { + ArtifactAvailability::NodeOffline + } else { + ArtifactAvailability::Unavailable + }; + let retention_state = if explicit_storage { + ArtifactRetentionState::ExplicitStorage + } else if !metadata.retaining_nodes.is_empty() { + ArtifactRetentionState::NodeRetained + } else { + ArtifactRetentionState::Lost + }; + let display_suffix = metadata.id.as_str().replace(':', "/"); + let display_path = format!("/vfs/artifacts/{display_suffix}"); + let display_name = display_suffix + .rsplit('/') + .next() + .unwrap_or(metadata.id.as_str()) + .to_owned(); + ArtifactSummary { + id: metadata.id, + display_path, + display_name, + process: metadata.process, + producer_task: metadata.producer_task, + safe_node, + digest: metadata.digest, + size_bytes: metadata.size, + availability, + downloadable_now, + retention_state, + explicit_storage, + order_cursor: format!("artifact:{}", metadata.flushed_epoch), + } + } + + fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) { + while self + .process_summaries + .keys() + .filter(|(entry_tenant, entry_project, _)| { + entry_tenant == tenant && entry_project == project + }) + .count() + >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT + { + let candidate = self.process_summary_order.iter().find(|key| { + &key.0 == tenant + && &key.1 == project + && self + .process_summaries + .get(*key) + .is_some_and(|summary| summary.final_result.is_some()) + }); + let Some(candidate) = candidate.cloned() else { + break; + }; + self.remove_process_summary_state(&candidate); + } + } + + fn evict_process_summaries_total(&mut self) { + while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL { + let candidate = self.process_summary_order.iter().find(|key| { + self.process_summaries + .get(*key) + .is_some_and(|summary| summary.final_result.is_some()) + }); + let Some(candidate) = candidate.cloned() else { + break; + }; + self.remove_process_summary_state(&candidate); + } + } + + fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) { + self.process_summaries.remove(key); + self.recent_log_dropped_through.remove(key); + if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) { + logs.retain(|entry| entry.process != key.2); + if logs.is_empty() { + self.recent_logs.remove(&(key.0.clone(), key.1.clone())); + } + } + self.recent_log_accounted_bytes + .retain(|(tenant, project, process, _, _), _| { + tenant != &key.0 || project != &key.1 || process != &key.2 + }); + self.recent_log_truncated_streams + .retain(|(tenant, project, process, _, _)| { + tenant != &key.0 || project != &key.1 || process != &key.2 + }); + self.recent_log_quota_truncated_streams + .retain(|(tenant, project, process, _, _)| { + tenant != &key.0 || project != &key.1 || process != &key.2 + }); + self.process_summary_order + .retain(|retained| retained != key); + } +} + +fn parse_order_cursor( + cursor: Option<&str>, + expected_kind: &str, +) -> Result, CoordinatorServiceError> { + cursor + .map(|cursor| { + let (kind, order) = cursor.split_once(':').ok_or_else(|| { + CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + )) + })?; + if kind != expected_kind { + return Err(CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + ))); + } + order.parse::().map_err(|_| { + CoordinatorServiceError::Protocol(format!( + "invalid {expected_kind} pagination cursor" + )) + }) + }) + .transpose() +} diff --git a/crates/clusterflux-coordinator/src/service/tcp.rs b/crates/clusterflux-coordinator/src/service/tcp.rs index 0b42969..2b6e9c9 100644 --- a/crates/clusterflux-coordinator/src/service/tcp.rs +++ b/crates/clusterflux-coordinator/src/service/tcp.rs @@ -79,17 +79,15 @@ impl CoordinatorService { continue; } let response = match decode_wire_request(&line) { - Ok(request) => match authorize_client_request(&request, authority_mode) - .and_then(|()| self.handle_request(request)) - { - Ok(response) => response, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, + Ok((request_id, request)) => { + match authorize_client_request(&request, authority_mode) + .and_then(|()| self.handle_request(request)) + { + Ok(response) => response, + Err(err) => CoordinatorResponse::service_error(request_id, &err), + } + } + Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -114,25 +112,19 @@ fn handle_shared_stream( continue; } let response = match decode_wire_request(&line) { - Ok(request) => match authorize_client_request(&request, authority_mode) { + Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) { Ok(()) => match service.lock() { Ok(mut service) => match service.handle_request(request) { Ok(response) => response, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(_) => CoordinatorResponse::Error { - message: "coordinator service lock poisoned".to_owned(), + Err(err) => CoordinatorResponse::service_error(request_id, &err), }, + Err(_) => { + CoordinatorResponse::error(request_id, "coordinator service lock poisoned") + } }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), - }, - }, - Err(err) => CoordinatorResponse::Error { - message: err.to_string(), + Err(err) => CoordinatorResponse::service_error(request_id, &err), }, + Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato Ok((listener, addr)) } -fn decode_wire_request(line: &str) -> Result { +fn decode_wire_request( + line: &str, +) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> { serde_json::from_str::(line)? - .into_request() + .into_parts() .map_err(CoordinatorServiceError::Protocol) } +fn wire_request_id_hint(line: &str) -> String { + serde_json::from_str::(line) + .ok() + .and_then(|value| { + value + .get("request_id") + .and_then(|value| value.as_str()) + .map(str::to_owned) + }) + .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok()) + .unwrap_or_else(|| "unavailable".to_owned()) +} + fn authorize_client_request( request: &CoordinatorRequest, authority_mode: ClientAuthorityMode, @@ -180,10 +187,11 @@ fn authorize_client_request( } | CoordinatorRequest::AdminStatus { .. } | CoordinatorRequest::SuspendTenant { .. } => Ok(()), - _ => Err(CoordinatorServiceError::Protocol( + _ => Err(crate::CoordinatorError::Unauthorized( "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority" .to_owned(), - )), + ) + .into()), } } @@ -253,16 +261,19 @@ mod transport_boundary_tests { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { message } = + let CoordinatorResponse::Error { error } = serde_json::from_str::(&line).unwrap() else { panic!("malformed identifier request unexpectedly succeeded"); }; assert!( - message.contains("malformed external identifier") - && message.contains("request.request.process"), - "unexpected malformed identifier response: {message}" + error.message.contains("malformed external identifier") + && error.message.contains("request.request.process"), + "unexpected malformed identifier response: {}", + error.message ); + assert_eq!(error.request_id, format!("malformed-{index}")); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); let valid = coordinator_wire_request( format!("healthy-{index}"), diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index 3369b5b..339860d 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -512,6 +512,20 @@ fn service_with_completed_main_and_final_child( public_key: test_node_public_key(node.as_str()), }) .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: tenant.to_string(), + project: project.to_string(), + node: node.to_string(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: Vec::new(), + direct_connectivity: true, + online: true, + }) + .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { launch_attempt: None, @@ -815,6 +829,7 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { | CoordinatorRequest::ReportDebugState { node, .. } | CoordinatorRequest::ReportDebugProbeHit { node, .. } | CoordinatorRequest::ReportTaskLog { node, .. } + | CoordinatorRequest::ReportTaskLogChunk { node, .. } | CoordinatorRequest::ReportVfsMetadata { node, .. } | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), @@ -860,6 +875,9 @@ fn signed_node_request_auto_with_private_key( (node.clone(), "report_debug_probe_hit") } CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"), + CoordinatorRequest::ReportTaskLogChunk { node, .. } => { + (node.clone(), "report_task_log_chunk") + } CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"), CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"), _ => panic!("test helper only signs node-originated requests"), @@ -1485,7 +1503,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { suspended, disabled, sanitized_reason, - private_moderation_details_exposed, + sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1506,7 +1524,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(!suspended); assert!(!disabled); assert!(sanitized_reason.is_none()); - assert!(!private_moderation_details_exposed); + assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); for (tenant, policy_name, expected_status, expected_reason) in [ @@ -1542,7 +1560,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { manual_review, sanitized_reason, next_actions, - private_moderation_details_exposed, + sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1564,7 +1582,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!private_moderation_details_exposed); + assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); } @@ -1703,7 +1721,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { disabled, sanitized_reason, next_actions, - private_moderation_details_exposed, + sensitive_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1726,7 +1744,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!private_moderation_details_exposed); + assert!(!sensitive_moderation_details_exposed); assert!(!signup_failure_details_exposed); let create = service @@ -2417,7 +2435,7 @@ fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() } #[test] -fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { +fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::LogBytes, 4); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); @@ -2476,23 +2494,37 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { backpressured: false, }) .unwrap(); - let denied = service + let truncated = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "task".to_owned(), - stdout_bytes: 1, - stderr_bytes: 0, - stdout_tail: "x".to_owned(), - stderr_tail: String::new(), + stdout_bytes: 4, + stderr_bytes: 1, + stdout_tail: "outx".to_owned(), + stderr_tail: "e".to_owned(), stdout_truncated: false, stderr_truncated: false, backpressured: true, }) - .unwrap_err(); - assert!(denied.to_string().contains("LogBytes")); + .unwrap(); + let CoordinatorResponse::TaskLogRecorded { + stdout_tail, + stdout_bytes: 4, + .. + } = truncated + else { + panic!("expected a successful truncated task-log report"); + }; + assert_eq!(stdout_tail, "[log output truncated at project log quota]"); + assert!(service + .recent_logs + .get(&(TenantId::from("tenant"), ProjectId::from("project"))) + .unwrap() + .iter() + .any(|entry| entry.text.contains("project log quota") && entry.truncated)); assert_eq!( service .quota @@ -2501,6 +2533,140 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { ); } +#[test] +fn log_quota_exhaustion_cannot_strand_task_completion_or_artifact_publication() { + let mut limits = ResourceLimits::unlimited(); + limits.limits.insert(LimitKind::LogBytes, 4); + let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); + let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( + 7, + "test-admin-token", + None, + quota, + ) + .unwrap(); + service.set_server_time(30); + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: "node".to_owned(), + public_key: test_node_public_key("node"), + }) + .unwrap(); + service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "process".to_owned(), + restart: false, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: "node".to_owned(), + process: "process".to_owned(), + epoch: 7, + }) + .unwrap(); + register_test_task_assignment( + &mut service, + "tenant", + "project", + "process", + "node", + "compile", + "compile-one", + 7, + ); + service.record_task_completion_event(TaskCompletionEvent { + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), + node: NodeId::from("coordinator-main"), + executor: TaskExecutor::CoordinatorMain, + task_definition: TaskDefinitionId::from("build"), + task: TaskInstanceId::from("main"), + attempt_id: None, + placement: None, + terminal_state: TaskTerminalState::Completed, + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }); + + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + process: "process".to_owned(), + node: "node".to_owned(), + task: "compile-one".to_owned(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 64, + stderr_bytes: 0, + stdout_tail: "the-real-final-tail".to_owned(), + stderr_tail: String::new(), + stdout_truncated: true, + stderr_truncated: false, + artifact_path: Some("/vfs/artifacts/result.bin".to_owned()), + artifact_digest: Some(Digest::sha256("artifact bytes")), + artifact_size_bytes: Some(14), + result: None, + }) + .unwrap(); + + let event = service + .task_events + .iter() + .find(|event| event.task == TaskInstanceId::from("compile-one")) + .unwrap(); + assert_eq!( + event.stdout_tail, + "[log output truncated at project log quota]" + ); + assert!(event.stdout_truncated); + assert!(!service + .active_tasks + .iter() + .any(|key| key.4 == TaskInstanceId::from("compile-one"))); + assert!(service + .coordinator + .active_process( + &TenantId::from("tenant"), + &ProjectId::from("project"), + &ProcessId::from("process"), + ) + .is_none()); + assert!(matches!( + service + .handle_request(CoordinatorRequest::GetArtifact { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + actor_user: "user".to_owned(), + artifact: "result.bin".to_owned(), + }) + .unwrap(), + CoordinatorResponse::Artifact { .. } + )); +} + #[test] fn service_attaches_node_starts_process_and_records_scoped_task_event() { let mut service = CoordinatorService::new(7); @@ -4456,6 +4622,63 @@ fn completed_main_unpolled_final_assignment_completion_retires_process() { .expect("unpolled terminal completion must release the one-process slot"); } +#[test] +fn completed_main_retires_from_authoritative_state_after_event_history_rotates() { + let mut service = + service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + + for index in 0..=MAX_TASK_EVENTS_PER_PROCESS { + service.record_task_completion_event(TaskCompletionEvent { + tenant: tenant.clone(), + project: project.clone(), + process: process.clone(), + node: NodeId::from("worker"), + executor: TaskExecutor::Node, + task_definition: TaskDefinitionId::from("historical"), + task: TaskInstanceId::new(format!("historical-{index}")), + attempt_id: None, + placement: None, + terminal_state: TaskTerminalState::Completed, + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }); + } + assert!( + service.task_events.iter().all(|event| { + event.process != process || event.executor != TaskExecutor::CoordinatorMain + }), + "the regression requires bounded history to have rotated the main event" + ); + + complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + let summary = service + .process_summaries + .get(&process_control_key(&tenant, &project, &process)) + .expect("the terminal process summary must remain authoritative"); + assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed)); + assert_eq!( + summary.main_terminal_state, + Some(TaskTerminalState::Completed) + ); +} + #[test] fn completed_main_await_operator_blocks_retirement_until_each_resolution() { for resolution in [ @@ -4549,6 +4772,89 @@ fn completed_main_await_operator_blocks_retirement_until_each_resolution() { } } +#[test] +fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() { + let mut service = service_with_completed_main_and_final_child( + clusterflux_core::TaskFailurePolicy::AwaitOperator, + ); + complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let process = ProcessId::from("terminal-matrix"); + let task = TaskInstanceId::from("final-child"); + + let CoordinatorResponse::TaskRestart { accepted, .. } = service + .handle_request(CoordinatorRequest::RestartTask { + tenant: tenant.to_string(), + project: project.to_string(), + actor_user: "user".to_owned(), + process: process.to_string(), + task: task.to_string(), + replacement_bundle: None, + }) + .unwrap() + else { + panic!("expected task restart"); + }; + assert!(accepted); + + service + .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { + tenant: tenant.to_string(), + project: project.to_string(), + process: process.to_string(), + node: "worker".to_owned(), + task: task.to_string(), + terminal_state: Some(TaskTerminalState::Completed), + status_code: Some(0), + stdout_bytes: 2, + stderr_bytes: 0, + stdout_tail: "ok".to_owned(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: Some(TaskBoundaryValue::SmallJson(json!("ok"))), + }) + .unwrap(); + + assert!(service + .coordinator + .active_process(&tenant, &project, &process) + .is_none()); + let attempts = service + .task_attempts + .get(&super::keys::task_restart_key( + &tenant, &project, &process, &task, + )) + .unwrap(); + assert!( + attempts + .iter() + .any(|attempt| !attempt.current + && attempt.state == TaskAttemptState::FailedAwaitingAction) + ); + assert!(attempts + .iter() + .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed)); + assert_eq!( + service + .process_summaries + .get(&process_control_key(&tenant, &project, &process)) + .and_then(|summary| summary.final_result.clone()), + Some(ProcessFinalResult::Completed), + "a successful current retry must override the superseded failed attempt" + ); + assert_eq!( + service + .task_join_result(tenant, project, process, task) + .state, + TaskJoinState::Completed + ); +} + #[test] fn completed_main_failed_child_does_not_abort_another_active_child() { let mut service = @@ -5570,6 +5876,29 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() { #[test] fn windows_task_events_share_the_virtual_process_scope() { let mut service = CoordinatorService::new(7); + service.record_task_completion_event(TaskCompletionEvent { + tenant: TenantId::from("other-tenant"), + project: ProjectId::from("other-project"), + process: ProcessId::from("other-process"), + node: NodeId::from("other-node"), + executor: super::TaskExecutor::Node, + task_definition: TaskDefinitionId::from("other-task"), + task: TaskInstanceId::from("other-task"), + attempt_id: None, + placement: None, + terminal_state: TaskTerminalState::Completed, + status_code: Some(0), + stdout_bytes: 0, + stderr_bytes: 0, + stdout_tail: String::new(), + stderr_tail: String::new(), + stdout_truncated: false, + stderr_truncated: false, + artifact_path: None, + artifact_digest: None, + artifact_size_bytes: None, + result: None, + }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), @@ -5670,6 +5999,28 @@ fn windows_task_events_share_the_virtual_process_scope() { #[test] fn service_schedules_task_across_reported_node_descriptors() { let mut service = CoordinatorService::new(7); + service + .handle_request(CoordinatorRequest::AttachNode { + tenant: "other-tenant".to_owned(), + project: "other-project".to_owned(), + node: "other-node".to_owned(), + public_key: test_node_public_key("other-node"), + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: "other-tenant".to_owned(), + project: "other-project".to_owned(), + node: "other-node".to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: Vec::new(), + direct_connectivity: false, + online: true, + }) + .unwrap(); for node in ["cold-node", "warm-node"] { service .handle_request(CoordinatorRequest::AttachNode { @@ -6957,6 +7308,52 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { #[test] fn coordinator_side_task_launch_can_wait_for_capable_worker() { let mut service = CoordinatorService::new(11); + let CoordinatorResponse::ProcessStarted { + epoch: other_epoch, .. + } = service + .handle_request(CoordinatorRequest::StartProcess { + launch_attempt: None, + tenant: "other-tenant".to_owned(), + project: "other-project".to_owned(), + actor_user: None, + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + process: "other-vp-wait".to_owned(), + restart: false, + }) + .unwrap() + else { + panic!("expected unrelated process start"); + }; + let CoordinatorResponse::TaskQueued { + queued_tasks: other_queued_tasks, + .. + } = service + .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { + task_spec: test_task_spec( + "other-tenant", + "other-project", + "other-vp-wait", + "other-compile", + other_epoch, + [Capability::Command], + ), + tenant: "other-tenant".to_owned(), + project: "other-project".to_owned(), + actor_user: Some("other-user".to_owned()), + actor_agent: None, + agent_public_key_fingerprint: None, + agent_signature: None, + wait_for_node: true, + artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(), + wasm_module_base64: test_wasm_module_base64(), + }) + .unwrap() + else { + panic!("expected unrelated queued task launch"); + }; + assert_eq!(other_queued_tasks, 1); let CoordinatorResponse::ProcessStarted { launch_attempt: None, epoch, @@ -7670,12 +8067,16 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { message } = + let CoordinatorResponse::Error { error } = serde_json::from_str::(&line).unwrap() else { panic!("expected strict body-authority denial"); }; - assert!(message.contains("request-body identity fields are not authority")); + assert!(error + .message + .contains("request-body identity fields are not authority")); + assert_eq!(error.request_id, "strict-stream-forged"); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden); write_coordinator_wire_request( &mut stream, @@ -7731,10 +8132,14 @@ fn service_stream_rejects_invalid_versioned_envelope_metadata() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); let response = serde_json::from_str::(&line).unwrap(); - let CoordinatorResponse::Error { message } = response else { + let CoordinatorResponse::Error { error } = response else { panic!("expected invalid wire envelope response"); }; - assert!(message.contains("operation attach_node does not match payload operation ping")); + assert!(error + .message + .contains("operation attach_node does not match payload operation ping")); + assert_eq!(error.request_id, "bad-operation"); + assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); @@ -7831,3 +8236,879 @@ fn coordinator_generates_and_bounds_node_enrollment_grants() { assert_ne!(first_grant, second_grant); assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60); } + +#[test] +fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret) in [ + ("tenant-a", "project-a", "user-a", "session-a"), + ("tenant-b", "project-b", "user-b", "session-b"), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + } + service.set_server_time(100); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: "process-one".to_owned(), + restart: false, + }, + }) + .unwrap(); + + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected process summaries"); + }; + assert_eq!(processes.len(), 1); + assert_eq!(processes[0].process, ProcessId::from("process-one")); + assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active); + assert_eq!(processes[0].activity, ProcessActivityState::Running); + assert_eq!(processes[0].started_at_epoch_seconds, 100); + assert!(next_cursor.is_none()); + + let CoordinatorResponse::ProcessSummaries { processes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 10, + }, + }) + .unwrap() + else { + panic!("expected scoped process summaries"); + }; + assert!(processes.is_empty()); + + service.set_server_time(120); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::AbortProcess { + process: "process-one".to_owned(), + launch_attempt: None, + }, + }) + .unwrap(); + let CoordinatorResponse::ProcessSummaries { processes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 10, + }, + }) + .unwrap() + else { + panic!("expected terminal process summary"); + }; + assert_eq!( + processes[0].lifecycle, + ProcessLifecycleState::RecentTerminal + ); + assert_eq!(processes[0].activity, ProcessActivityState::Cancelled); + assert_eq!( + processes[0].final_result, + Some(ProcessFinalResult::Cancelled) + ); + assert_eq!(processes[0].ended_at_epoch_seconds, Some(120)); + + service.set_server_time(130); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: "process-two".to_owned(), + restart: false, + }, + }) + .unwrap(); + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor: Some(cursor), + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected first process summary page"); + }; + assert_eq!(processes[0].process, ProcessId::from("process-two")); + let CoordinatorResponse::ProcessSummaries { + processes, + next_cursor: None, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: Some(cursor), + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected final process summary page"); + }; + assert_eq!(processes[0].process, ProcessId::from("process-one")); + + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListProcessSummaries { + cursor: None, + limit: 101, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("100")); +} + +#[test] +fn process_summary_eviction_releases_live_log_accounting_state() { + let mut service = CoordinatorService::new(7); + let tenant = TenantId::from("tenant-summary-bound"); + let project = ProjectId::from("project-summary-bound"); + let task = TaskInstanceId::from("task"); + + for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT { + let process = ProcessId::new(format!("process-{index:03}")); + service.record_process_started(&tenant, &project, &process, index as u64); + service.record_process_terminal( + &tenant, + &project, + &process, + ProcessFinalResult::Completed, + index as u64 + 1, + ); + let key = ( + tenant.clone(), + project.clone(), + process, + task.clone(), + "stdout".to_owned(), + ); + service.recent_log_accounted_bytes.insert(key.clone(), 10); + service.recent_log_truncated_streams.insert(key); + } + + let evicted = ProcessId::from("process-000"); + service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000); + + assert!(!service.process_summaries.contains_key(&( + tenant.clone(), + project.clone(), + evicted.clone() + ))); + assert!(!service.recent_log_accounted_bytes.keys().any( + |(entry_tenant, entry_project, process, _, _)| { + entry_tenant == &tenant && entry_project == &project && process == &evicted + } + )); + assert!(!service.recent_log_truncated_streams.iter().any( + |(entry_tenant, entry_project, process, _, _)| { + entry_tenant == &tenant && entry_project == &project && process == &evicted + } + )); +} + +#[test] +fn web_node_summaries_are_scoped_paginated_and_hard_bounded() { + let mut service = CoordinatorService::new(7); + service + .issue_cli_session( + TenantId::from("tenant"), + ProjectId::from("project"), + UserId::from("user"), + "session", + None, + ) + .unwrap(); + for node in ["node-a", "node-b", "node-c"] { + enroll_test_node( + &mut service, + "tenant", + "project", + node, + &test_node_public_key(node), + ); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: "tenant".to_owned(), + project: "project".to_owned(), + node: node.to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: Vec::new(), + direct_connectivity: true, + online: true, + }) + .unwrap(); + } + + let CoordinatorResponse::NodeSummaries { + nodes, + next_cursor: Some(cursor), + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected first node-summary page"); + }; + assert_eq!( + nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + ["node-a", "node-b"] + ); + + let CoordinatorResponse::NodeSummaries { + nodes, + next_cursor: None, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: Some(cursor), + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected final node-summary page"); + }; + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].id, NodeId::from("node-c")); + + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 201, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("200")); +} + +#[test] +fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret, node) in [ + ("tenant-a", "project-a", "user-a", "session-a", "node-a"), + ("tenant-b", "project-b", "user-b", "session-b", "node-b"), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + enroll_test_node( + &mut service, + tenant, + project, + node, + &test_node_public_key(node), + ); + } + service.set_server_time(100); + for (tenant, project, node) in [ + ("tenant-a", "project-a", "node-a"), + ("tenant-b", "project-b", "node-b"), + ] { + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + capabilities: linux_capabilities(), + cached_environment_digests: Vec::new(), + dependency_cache_digests: Vec::new(), + source_snapshots: Vec::new(), + artifact_locations: vec!["shared-artifact".to_owned()], + direct_connectivity: true, + online: false, + }) + .unwrap(); + } + let CoordinatorResponse::NodeSummaries { nodes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected node summaries"); + }; + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].id, NodeId::from("node-a")); + assert!(nodes[0].online); + assert!(!nodes[0].stale); + assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); + assert_eq!(nodes[0].capabilities.os, Os::Linux); + for (tenant, project, node, digest) in [ + ("tenant-a", "project-a", "node-a", "tenant-a-bytes"), + ("tenant-b", "project-b", "node-b", "tenant-b-bytes"), + ] { + service.artifact_registry.flush_metadata(ArtifactFlush { + id: ArtifactId::from("shared-artifact"), + tenant: TenantId::from(tenant), + project: ProjectId::from(project), + process: ProcessId::from("process-one"), + producer_task: TaskInstanceId::from("task-one"), + retaining_node: NodeId::from(node), + digest: Digest::sha256(digest), + size: digest.len() as u64, + }); + } + service.artifact_registry.flush_metadata(ArtifactFlush { + id: ArtifactId::from("second-artifact"), + tenant: TenantId::from("tenant-a"), + project: ProjectId::from("project-a"), + process: ProcessId::from("process-two"), + producer_task: TaskInstanceId::from("task-two"), + retaining_node: NodeId::from("node-a"), + digest: Digest::sha256("second"), + size: 6, + }); + + let CoordinatorResponse::Artifacts { + artifacts, + next_cursor: Some(cursor), + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListArtifacts { + process: None, + cursor: None, + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected first artifact page"); + }; + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact")); + assert_eq!(artifacts[0].availability, ArtifactAvailability::Available); + let CoordinatorResponse::Artifacts { + artifacts, + next_cursor: None, + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListArtifacts { + process: None, + cursor: Some(cursor), + limit: 1, + }, + }) + .unwrap() + else { + panic!("expected final artifact page"); + }; + assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact")); + assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes")); + + let cross_tenant = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "tenant-b-only".to_owned(), + }, + }) + .unwrap_err(); + assert!(cross_tenant.to_string().contains("does not exist")); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected tenant-b artifact"); + }; + assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes")); + + service.set_server_time(131); + let CoordinatorResponse::NodeSummaries { nodes, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListNodeSummaries { + cursor: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected stale node summary"); + }; + assert!(!nodes[0].online); + assert!(nodes[0].stale); + assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected offline artifact metadata"); + }; + assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline); + assert!(!artifact.downloadable_now); + + service + .artifact_registry + .sync_to_explicit_store( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + &ArtifactId::from("shared-artifact"), + "store://tenant-a/shared-artifact", + ) + .unwrap(); + let CoordinatorResponse::Artifact { artifact } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::GetArtifact { + artifact: "shared-artifact".to_owned(), + }, + }) + .unwrap() + else { + panic!("expected explicitly retained artifact metadata"); + }; + assert_eq!(artifact.availability, ArtifactAvailability::Available); + assert_eq!( + artifact.retention_state, + ArtifactRetentionState::ExplicitStorage + ); + assert!(artifact.downloadable_now); +} + +#[test] +fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() { + let mut service = CoordinatorService::new(7); + for (tenant, project, user, secret, node, process) in [ + ( + "tenant-a", + "project-a", + "user-a", + "session-a", + "node-a", + "process-shared", + ), + ( + "tenant-b", + "project-b", + "user-b", + "session-b", + "node-b", + "process-b", + ), + ] { + service + .issue_cli_session( + TenantId::from(tenant), + ProjectId::from(project), + UserId::from(user), + secret, + None, + ) + .unwrap(); + enroll_test_node( + &mut service, + tenant, + project, + node, + &test_node_public_key(node), + ); + service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: secret.to_owned(), + request: AuthenticatedCoordinatorRequest::StartProcess { + launch_attempt: None, + process: process.to_owned(), + restart: false, + }, + }) + .unwrap(); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { + tenant: tenant.to_owned(), + project: project.to_owned(), + node: node.to_owned(), + process: process.to_owned(), + epoch: 7, + }) + .unwrap(); + } + service.set_server_time(100); + let first = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 0, + source_bytes: 5, + text: "hello".to_owned(), + truncated: false, + }) + .unwrap(); + let CoordinatorResponse::TaskLogChunkRecorded { + sequence: Some(first_sequence), + next_offset: 5, + .. + } = first + else { + panic!("expected first live log sequence"); + }; + assert_eq!( + service.quota.used_log_bytes( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + 100, + ), + 5, + "live bytes must be charged when accepted" + ); + let retry = service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 0, + source_bytes: 5, + text: "hello".to_owned(), + truncated: false, + }) + .unwrap(); + assert!(matches!( + retry, + CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. } + )); + assert_eq!( + service.quota.used_log_bytes( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + 100, + ), + 5, + "a retried chunk must not be charged twice" + ); + service + .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { + tenant: "tenant-a".to_owned(), + project: "project-a".to_owned(), + process: "process-shared".to_owned(), + node: "node-a".to_owned(), + task: "task-one".to_owned(), + stream: TaskLogStream::Stdout, + offset: 8, + source_bytes: 2, + text: "ok".to_owned(), + truncated: false, + }) + .unwrap(); + assert_eq!( + service.quota.used_log_bytes( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + 100, + ), + 10, + "a gap and the delivered bytes must both count toward source-byte usage" + ); + + let CoordinatorResponse::RecentLogs { + entries, + next_sequence: Some(cursor), + history_truncated: false, + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected first recent-log page"); + }; + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].sequence, first_sequence); + assert_eq!(entries[0].text, "hello"); + assert!(entries[1].text.contains("3 bytes")); + assert!(entries[1].truncated); + let CoordinatorResponse::RecentLogs { entries, .. } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: Some(cursor), + limit: 2, + }, + }) + .unwrap() + else { + panic!("expected second recent-log page"); + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].text, "ok"); + + service + .handle_report_task_log( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + 12, + 0, + "hello???okZZ".to_owned(), + String::new(), + false, + false, + false, + ) + .unwrap(); + assert_eq!( + service.quota.used_log_bytes( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + 100, + ), + 12, + "the final summary must charge only source bytes not already charged live" + ); + assert_eq!( + service + .recent_logs + .get(&(TenantId::from("tenant-a"), ProjectId::from("project-a"))) + .unwrap() + .back() + .unwrap() + .text, + "ZZ", + "final-tail reconciliation must append only the nonduplicating suffix" + ); + service + .handle_report_task_log( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + 12, + 0, + "hello???okZZ".to_owned(), + String::new(), + false, + false, + false, + ) + .unwrap(); + assert_eq!( + service.quota.used_log_bytes( + &TenantId::from("tenant-a"), + &ProjectId::from("project-a"), + 100, + ), + 12, + "replayed final accounting must be idempotent" + ); + + let marker = service + .handle_report_task_log_chunk( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + TaskLogStream::Stdout, + 12, + 0, + "[log output truncated at node capture limit]".to_owned(), + true, + ) + .unwrap(); + assert!(matches!( + marker, + CoordinatorResponse::TaskLogChunkRecorded { + sequence: Some(_), + next_offset: 12, + .. + } + )); + let repeated_marker = service + .handle_report_task_log_chunk( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + TaskLogStream::Stdout, + 12, + 0, + "[log output truncated at node capture limit]".to_owned(), + true, + ) + .unwrap(); + assert!(matches!( + repeated_marker, + CoordinatorResponse::TaskLogChunkRecorded { + sequence: None, + next_offset: 12, + .. + } + )); + + let cross_tenant = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-b".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 10, + }, + }) + .unwrap_err(); + assert!(cross_tenant.to_string().contains("outside")); + let oversized = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 201, + }, + }) + .unwrap_err(); + assert!(oversized.to_string().contains("limit")); + assert!(oversized.to_string().contains("200")); + + service + .handle_report_task_log_chunk( + "tenant-b".to_owned(), + "project-b".to_owned(), + "process-b".to_owned(), + "node-b".to_owned(), + "task-b".to_owned(), + TaskLogStream::Stderr, + 0, + 1, + "b".to_owned(), + false, + ) + .unwrap(); + for offset in 10..310 { + service + .handle_report_task_log_chunk( + "tenant-a".to_owned(), + "project-a".to_owned(), + "process-shared".to_owned(), + "node-a".to_owned(), + "task-one".to_owned(), + TaskLogStream::Stdout, + offset, + 1, + "x".to_owned(), + false, + ) + .unwrap(); + } + let tenant_a_logs = + &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))]; + assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS); + let tenant_b_logs = + &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))]; + assert_eq!(tenant_b_logs.len(), 1); + assert_eq!(tenant_b_logs[0].text, "b"); + let CoordinatorResponse::RecentLogs { + entries, + history_truncated, + .. + } = service + .handle_request(CoordinatorRequest::Authenticated { + session_secret: "session-a".to_owned(), + request: AuthenticatedCoordinatorRequest::ListRecentLogs { + process: "process-shared".to_owned(), + task: None, + after_sequence: None, + limit: 200, + }, + }) + .unwrap() + else { + panic!("expected bounded recent-log response"); + }; + assert_eq!(entries.len(), 200); + assert!(history_truncated); +} diff --git a/crates/clusterflux-coordinator/src/service/wire_protocol.rs b/crates/clusterflux-coordinator/src/service/wire_protocol.rs index 07cd995..07aec2a 100644 --- a/crates/clusterflux-coordinator/src/service/wire_protocol.rs +++ b/crates/clusterflux-coordinator/src/service/wire_protocol.rs @@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest { impl CoordinatorWireRequest { pub fn into_request(self) -> Result { + self.into_parts().map(|(_, request)| request) + } + + pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { match self { - Self::Envelope(envelope) => envelope.into_request(), + Self::Envelope(envelope) => envelope.into_parts(), } } } @@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope { impl CoordinatorRequestEnvelope { pub fn into_request(self) -> Result { + self.into_parts().map(|(_, request)| request) + } + + pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE { return Err(format!( "unsupported coordinator wire request type {}; expected {}", @@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope { self.operation, payload_operation )); } - Ok(self.payload) + Ok((self.request_id, self.payload)) } } diff --git a/crates/clusterflux-core/src/api_error.rs b/crates/clusterflux-core/src/api_error.rs new file mode 100644 index 0000000..b6625cf --- /dev/null +++ b/crates/clusterflux-core/src/api_error.rs @@ -0,0 +1,296 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ApiErrorCode { + Unauthenticated, + SessionExpired, + AccountSuspended, + Forbidden, + ValidationError, + NotFound, + Conflict, + ActiveProcessExists, + NodeOffline, + NoCapableNode, + TaskNotRestartable, + ArtifactUnavailable, + ArtifactLimitExceeded, + QuotaExceeded, + TemporaryCapacity, + DebugEpochPartial, + InternalError, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ApiErrorCategory { + Authentication, + Authorization, + Validation, + State, + Availability, + Resource, + Internal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ApiError { + pub code: ApiErrorCode, + pub category: ApiErrorCategory, + pub message: String, + pub retryable: bool, + pub request_id: String, +} + +impl fmt::Display for ApiError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} (code {:?}, request {})", + self.message, self.code, self.request_id + ) + } +} + +impl std::error::Error for ApiError {} + +impl ApiError { + pub fn new( + code: ApiErrorCode, + category: ApiErrorCategory, + message: impl Into, + retryable: bool, + request_id: impl Into, + ) -> Self { + Self { + code, + category, + message: message.into(), + retryable, + request_id: request_id.into(), + } + } + + pub fn from_message(request_id: impl Into, message: impl Into) -> Self { + let request_id = request_id.into(); + let message = message.into(); + let normalized = message.to_ascii_lowercase(); + let (code, category, retryable) = if normalized.contains("session credential has expired") + || normalized.contains("session expired") + { + ( + ApiErrorCode::SessionExpired, + ApiErrorCategory::Authentication, + false, + ) + } else if normalized.contains("tenant is suspended") + || normalized.contains("account is suspended") + || normalized.contains("suspended by hosted") + { + ( + ApiErrorCode::AccountSuspended, + ApiErrorCategory::Authorization, + false, + ) + } else if normalized.contains("session credential") + || normalized.contains("no authenticated") + || normalized.contains("not authenticated") + || normalized.contains("credential is not") + { + ( + ApiErrorCode::Unauthenticated, + ApiErrorCategory::Authentication, + false, + ) + } else if normalized.contains("project already has active virtual process") { + ( + ApiErrorCode::ActiveProcessExists, + ApiErrorCategory::State, + false, + ) + } else if normalized.contains("no capable node") { + ( + ApiErrorCode::NoCapableNode, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("node offline") + || normalized.contains("node is not live") + || normalized.contains("source node is not connected") + || normalized.contains("direct connectivity unavailable") + { + ( + ApiErrorCode::NodeOffline, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("restart") + && (normalized.contains("not restartable") + || normalized.contains("requires whole") + || normalized.contains("clean boundary")) + { + ( + ApiErrorCode::TaskNotRestartable, + ApiErrorCategory::State, + false, + ) + } else if normalized.contains("artifact") + && (normalized.contains("exceeds download limit") + || normalized.contains("download session limit") + || normalized.contains("artifact limit")) + { + ( + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCategory::Resource, + false, + ) + } else if normalized.contains("artifact") + && (normalized.contains("does not exist") + || normalized.contains("not found") + || normalized.contains("unknown artifact")) + { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } else if normalized.contains("artifact") + && (normalized.contains("unavailable") || normalized.contains("retention")) + { + ( + ApiErrorCode::ArtifactUnavailable, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("resource limit") + || normalized.contains("quota") + || normalized.contains("limit exceeded") + { + ( + ApiErrorCode::QuotaExceeded, + ApiErrorCategory::Resource, + true, + ) + } else if normalized.contains("capacity") + || normalized.contains("temporarily full") + || normalized.contains("replay window is full") + { + ( + ApiErrorCode::TemporaryCapacity, + ApiErrorCategory::Availability, + true, + ) + } else if normalized.contains("partial debug epoch") + || normalized.contains("debug epoch is partially") + { + ( + ApiErrorCode::DebugEpochPartial, + ApiErrorCategory::State, + true, + ) + } else if normalized.contains("malformed") + || normalized.contains("invalid ") + || normalized.contains("protocol") + || normalized.contains("unknown field") + || normalized.contains("missing field") + || normalized.contains("must ") + { + ( + ApiErrorCode::ValidationError, + ApiErrorCategory::Validation, + false, + ) + } else if normalized.contains("outside") + || normalized.contains("unauthorized") + || normalized.contains("denied") + || normalized.contains("requires an authenticated") + || normalized.contains("may only") + { + ( + ApiErrorCode::Forbidden, + ApiErrorCategory::Authorization, + false, + ) + } else if normalized.contains("not found") + || normalized.contains("does not exist") + || normalized.contains("unknown ") + { + (ApiErrorCode::NotFound, ApiErrorCategory::State, false) + } else if normalized.contains("already") + || normalized.contains("conflict") + || normalized.contains("requires an active") + { + (ApiErrorCode::Conflict, ApiErrorCategory::State, false) + } else { + ( + ApiErrorCode::InternalError, + ApiErrorCategory::Internal, + false, + ) + }; + Self::new(code, category, message, retryable, request_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn required_machine_codes_are_stably_serialized() { + let required = [ + ApiErrorCode::Unauthenticated, + ApiErrorCode::SessionExpired, + ApiErrorCode::AccountSuspended, + ApiErrorCode::Forbidden, + ApiErrorCode::ValidationError, + ApiErrorCode::NotFound, + ApiErrorCode::Conflict, + ApiErrorCode::ActiveProcessExists, + ApiErrorCode::NodeOffline, + ApiErrorCode::NoCapableNode, + ApiErrorCode::TaskNotRestartable, + ApiErrorCode::ArtifactUnavailable, + ApiErrorCode::ArtifactLimitExceeded, + ApiErrorCode::QuotaExceeded, + ApiErrorCode::TemporaryCapacity, + ApiErrorCode::DebugEpochPartial, + ]; + let serialized = required + .into_iter() + .map(|code| serde_json::to_value(code).unwrap()) + .collect::>(); + assert_eq!( + serialized, + vec![ + "unauthenticated", + "session_expired", + "account_suspended", + "forbidden", + "validation_error", + "not_found", + "conflict", + "active_process_exists", + "node_offline", + "no_capable_node", + "task_not_restartable", + "artifact_unavailable", + "artifact_limit_exceeded", + "quota_exceeded", + "temporary_capacity", + "debug_epoch_partial", + ] + ); + } + + #[test] + fn message_classification_keeps_request_identity() { + let error = ApiError::from_message( + "request-17", + "CLI session credential has expired; run login again", + ); + assert_eq!(error.code, ApiErrorCode::SessionExpired); + assert_eq!(error.category, ApiErrorCategory::Authentication); + assert_eq!(error.request_id, "request-17"); + assert!(!error.retryable); + } +} diff --git a/crates/clusterflux-core/src/artifact.rs b/crates/clusterflux-core/src/artifact.rs index 240d5b4..d9c89a3 100644 --- a/crates/clusterflux-core/src/artifact.rs +++ b/crates/clusterflux-core/src/artifact.rs @@ -10,7 +10,7 @@ use crate::{ const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32; const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60; -const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256; +const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -184,54 +184,45 @@ impl ArtifactRegistry { &mut self, flush: ArtifactFlush, pinned: &BTreeSet, + ) -> Result { + self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new()) + } + + pub fn flush_metadata_with_protected_processes( + &mut self, + flush: ArtifactFlush, + pinned: &BTreeSet, + protected_processes: &BTreeSet, ) -> Result { let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id); - let replacing_existing = self.artifacts.contains_key(&key); - while !replacing_existing - && self - .artifacts - .values() - .filter(|metadata| { - metadata.tenant == flush.tenant - && metadata.project == flush.project - && metadata.process == flush.process - }) - .count() - >= MAX_ARTIFACT_METADATA_PER_PROCESS - { - let candidate = self - .artifacts - .values() - .filter(|metadata| { - metadata.tenant == flush.tenant - && metadata.project == flush.project - && metadata.process == flush.process - && !pinned.contains(&ArtifactScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - &metadata.id, - )) - && !self.issued_download_links.values().any(|issued| { - issued.link.tenant == metadata.tenant - && issued.link.project == metadata.project - && issued.link.artifact == metadata.id - }) - }) - .min_by_key(|metadata| metadata.flushed_epoch) - .map(|metadata| { - ArtifactScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - &metadata.id, - ) - }) + let replacing = self.artifacts.contains_key(&key); + let retained_for_project = self + .artifacts + .values() + .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project) + .count(); + let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT { + None + } else { + let mut protected_processes = protected_processes.clone(); + protected_processes.insert(flush.process.clone()); + Some( + self.project_metadata_eviction_candidate( + &flush.tenant, + &flush.project, + pinned, + &protected_processes, + ) .ok_or_else(|| { - "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download" - .to_owned() - })?; - self.artifacts.remove(&candidate); - } - self.next_epoch += 1; + format!( + "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \ + exhausted by active or retained artifacts" + ) + })?, + ) + }; + + self.next_epoch = self.next_epoch.saturating_add(1); let metadata = ArtifactMetadata { id: flush.id.clone(), tenant: flush.tenant, @@ -246,10 +237,75 @@ impl ArtifactRegistry { explicit_locations: Vec::new(), coordinator_has_large_bytes: false, }; + if let Some(eviction) = eviction { + self.artifacts.remove(&eviction); + } self.artifacts.insert(key, metadata.clone()); Ok(metadata) } + pub fn enforce_project_metadata_limit( + &mut self, + tenant: &TenantId, + project: &ProjectId, + pinned: &BTreeSet, + protected_processes: &BTreeSet, + ) -> usize { + let mut evicted = 0; + while self + .artifacts + .values() + .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) + .count() + > MAX_ARTIFACT_METADATA_PER_PROJECT + { + let candidate = self.project_metadata_eviction_candidate( + tenant, + project, + pinned, + protected_processes, + ); + let Some(candidate) = candidate else { + break; + }; + self.artifacts.remove(&candidate); + evicted += 1; + } + evicted + } + + fn project_metadata_eviction_candidate( + &self, + tenant: &TenantId, + project: &ProjectId, + pinned: &BTreeSet, + protected_processes: &BTreeSet, + ) -> Option { + self.artifacts + .values() + .filter(|metadata| { + &metadata.tenant == tenant + && &metadata.project == project + && !pinned.contains(&ArtifactScopeKey::from_refs( + &metadata.tenant, + &metadata.project, + &metadata.id, + )) + && !protected_processes.contains(&metadata.process) + && metadata.explicit_locations.is_empty() + && !self.issued_download_links.values().any(|issued| { + !issued.revoked + && issued.link.tenant == metadata.tenant + && issued.link.project == metadata.project + && issued.link.artifact == metadata.id + }) + }) + .min_by_key(|metadata| metadata.flushed_epoch) + .map(|metadata| { + ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id) + }) + } + pub fn sync_to_explicit_store( &mut self, tenant: &TenantId, @@ -304,6 +360,16 @@ impl ArtifactRegistry { .get(&ArtifactScopeKey::from_refs(tenant, project, artifact)) } + pub fn metadata_for_project<'a>( + &'a self, + tenant: &'a TenantId, + project: &'a ProjectId, + ) -> impl Iterator + 'a { + self.artifacts + .values() + .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project) + } + pub fn download_action( &self, context: &AuthContext, @@ -1340,8 +1406,10 @@ mod tests { } #[test] - fn artifact_metadata_is_bounded_without_evicting_pins() { + fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() { let mut registry = ArtifactRegistry::default(); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); registry.flush_metadata(ArtifactFlush { id: ArtifactId::from("artifact-0"), tenant: TenantId::from("other-tenant"), @@ -1352,72 +1420,92 @@ mod tests { digest: Digest::sha256("other-content"), size: 1, }); - let mut pinned = BTreeSet::new(); - for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS { + for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { let id = ArtifactId::new(format!("artifact-{index}")); - pinned.insert(ArtifactScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - id.clone(), - )); - registry - .flush_metadata_bounded( - ArtifactFlush { - id, - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - producer_task: TaskInstanceId::new(format!("task-{index}")), - retaining_node: NodeId::from("node"), - digest: Digest::sha256(format!("content-{index}")), - size: 1, - }, - &pinned, - ) - .unwrap(); + registry.flush_metadata(ArtifactFlush { + id, + tenant: tenant.clone(), + project: project.clone(), + process: ProcessId::new(if index == 3 { + "active-process-3".to_owned() + } else { + format!("completed-process-{index}") + }), + producer_task: TaskInstanceId::new(format!("task-{index}")), + retaining_node: NodeId::from("node"), + digest: Digest::sha256(format!("content-{index}")), + size: 1, + }); } assert_eq!( registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROCESS + 1 + MAX_ARTIFACT_METADATA_PER_PROJECT + 1 ); + let pinned = BTreeSet::from([ArtifactScopeKey::new( + tenant.clone(), + project.clone(), + ArtifactId::from("artifact-0"), + )]); + registry + .sync_to_explicit_store( + &tenant, + &project, + &ArtifactId::from("artifact-1"), + "store://retained-export", + ) + .unwrap(); + let context = AuthContext { + tenant: tenant.clone(), + project: project.clone(), + actor: Actor::User(UserId::from("user")), + }; + registry + .create_download_link( + &context, + &ArtifactId::from("artifact-2"), + &DownloadPolicy { max_bytes: 1 }, + "active-download", + 10, + 60, + ) + .unwrap(); + let protected_processes = BTreeSet::from([ + ProcessId::from("active-process-3"), + ProcessId::from("active-process"), + ]); let next = ArtifactFlush { id: ArtifactId::from("artifact-next"), - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), + tenant: tenant.clone(), + project: project.clone(), + process: ProcessId::from("active-process"), producer_task: TaskInstanceId::from("task-next"), retaining_node: NodeId::from("node"), digest: Digest::sha256("next"), size: 1, }; - assert!(registry - .flush_metadata_bounded(next.clone(), &pinned) - .unwrap_err() - .contains("pinned")); - - pinned.remove(&ArtifactScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - ArtifactId::from("artifact-0"), - )); - registry.flush_metadata_bounded(next, &pinned).unwrap(); + registry + .flush_metadata_with_protected_processes(next, &pinned, &protected_processes) + .unwrap(); assert_eq!( registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROCESS + 1 + MAX_ARTIFACT_METADATA_PER_PROJECT + 1 + ); + for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] { + assert!( + registry + .metadata(&tenant, &project, &ArtifactId::from(id)) + .is_some(), + "{id} was evicted despite being pinned, exported, downloaded, or live" + ); + } + assert!( + registry + .metadata(&tenant, &project, &ArtifactId::from("artifact-4")) + .is_none(), + "the oldest completed unprotected metadata should be evicted" ); assert!(registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact-0"), - ) - .is_none()); - assert!(registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact-next"), - ) + .metadata(&tenant, &project, &ArtifactId::from("artifact-next")) .is_some()); assert_eq!( registry @@ -1433,6 +1521,87 @@ mod tests { ); } + #[test] + fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() { + let mut registry = ArtifactRegistry::default(); + let tenant = TenantId::from("tenant"); + let project = ProjectId::from("project"); + let active_process = ProcessId::from("active-process"); + for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { + registry.flush_metadata(ArtifactFlush { + id: ArtifactId::new(format!("artifact-{index}")), + tenant: tenant.clone(), + project: project.clone(), + process: active_process.clone(), + producer_task: TaskInstanceId::new(format!("task-{index}")), + retaining_node: NodeId::from("node"), + digest: Digest::sha256(format!("content-{index}")), + size: 1, + }); + } + + let replacement = ArtifactFlush { + id: ArtifactId::from("artifact-0"), + tenant: tenant.clone(), + project: project.clone(), + process: active_process.clone(), + producer_task: TaskInstanceId::from("replacement-task"), + retaining_node: NodeId::from("node"), + digest: Digest::sha256("replacement"), + size: 2, + }; + registry + .flush_metadata_with_protected_processes( + replacement, + &BTreeSet::new(), + &BTreeSet::from([active_process.clone()]), + ) + .expect("replacement must remain possible at the metadata bound"); + assert_eq!( + registry + .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) + .unwrap() + .digest, + Digest::sha256("replacement") + ); + + let epoch_before_rejection = registry.next_epoch; + let result = registry.flush_metadata_with_protected_processes( + ArtifactFlush { + id: ArtifactId::from("artifact-rejected"), + tenant: tenant.clone(), + project: project.clone(), + process: active_process.clone(), + producer_task: TaskInstanceId::from("rejected-task"), + retaining_node: NodeId::from("node"), + digest: Digest::sha256("rejected"), + size: 3, + }, + &BTreeSet::new(), + &BTreeSet::from([active_process]), + ); + + assert!(result + .unwrap_err() + .contains("exhausted by active or retained artifacts")); + assert_eq!(registry.next_epoch, epoch_before_rejection); + assert_eq!( + registry.metadata_for_project(&tenant, &project).count(), + MAX_ARTIFACT_METADATA_PER_PROJECT + ); + assert!(registry + .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected")) + .is_none()); + assert_eq!( + registry + .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) + .unwrap() + .digest, + Digest::sha256("replacement"), + "rejection must not modify existing metadata" + ); + } + #[test] fn download_stream_accounts_usage_before_and_during_streaming() { let mut registry = registry_with_artifact(); diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index b2f9188..c9d5b77 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -1,3 +1,4 @@ +mod api_error; pub mod artifact; pub mod auth; pub mod bundle; @@ -18,6 +19,7 @@ pub mod transport; pub mod vfs; pub mod wire; +pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode}; pub use artifact::{ ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry, ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, diff --git a/crates/clusterflux-dap/src/adapter.rs b/crates/clusterflux-dap/src/adapter.rs index bbf9a28..4e144ab 100644 --- a/crates/clusterflux-dap/src/adapter.rs +++ b/crates/clusterflux-dap/src/adapter.rs @@ -1208,10 +1208,12 @@ fn emit_runtime_outcome( )?; } RuntimeContinuationOutcome::Terminal(record) => { + let exit_code = record.status_code.unwrap_or(1); apply_runtime_record_with_thread_events(writer, state, record)?; if state.last_task_failed { writer.output("stderr", format!("{}\n", state.command_status))?; } + writer.event("exited", json!({ "exitCode": exit_code }))?; writer.event("terminated", json!({}))?; } } diff --git a/crates/clusterflux-dap/src/main.rs b/crates/clusterflux-dap/src/main.rs index 32336b2..b0a9c14 100644 --- a/crates/clusterflux-dap/src/main.rs +++ b/crates/clusterflux-dap/src/main.rs @@ -22,7 +22,7 @@ use breakpoints::{ #[cfg(test)] use dap_protocol::{initialize_capabilities, read_message}; #[cfg(test)] -use runtime_client::{client_user_request, parse_task_restart_response}; +use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code}; #[cfg(test)] use variables::variables_response; #[cfg(test)] @@ -36,6 +36,26 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD}; use virtual_model::{process_id, RuntimeBackend}; fn main() -> Result<()> { + let raw_args = std::env::args().skip(1).collect::>(); + match raw_args.as_slice() { + [flag] if matches!(flag.as_str(), "--version" | "-V") => { + println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + [flag] if matches!(flag.as_str(), "--help" | "-h") => { + println!( + "Clusterflux Debug Adapter Protocol server.\n\n\ + Usage: clusterflux-debug-dap\n\n\ + The adapter communicates over standard input and output.\n\n\ + Options:\n \ + -h, --help\n \ + -V, --version" + ); + return Ok(()); + } + [] => {} + [argument, ..] => anyhow::bail!("unknown argument: {argument}"), + } adapter::run_adapter() } diff --git a/crates/clusterflux-dap/src/runtime_client.rs b/crates/clusterflux-dap/src/runtime_client.rs index 2cdefaf..9b483ac 100644 --- a/crates/clusterflux-dap/src/runtime_client.rs +++ b/crates/clusterflux-dap/src/runtime_client.rs @@ -742,14 +742,16 @@ fn fetch_current_process_status( client_user_request( state, json!({ - "type": "list_processes", + "type": "list_process_summaries", "tenant": state.tenant, "project": state.project_id, "actor_user": state.actor_user, + "cursor": null, + "limit": 100, }), ), )?; - let current = statuses + let summary = statuses .get("processes") .and_then(Value::as_array) .and_then(|processes| { @@ -758,6 +760,9 @@ fn fetch_current_process_status( }) }) .cloned(); + let current = merge_active_process_status(state, summary, |request| { + coordinator_request(coordinator, request) + })?; Ok((statuses, current)) } @@ -768,13 +773,15 @@ fn fetch_current_process_status_in( let statuses = session.request(client_user_request( state, json!({ - "type": "list_processes", + "type": "list_process_summaries", "tenant": state.tenant, "project": state.project_id, "actor_user": state.actor_user, + "cursor": null, + "limit": 100, }), ))?; - let current = statuses + let summary = statuses .get("processes") .and_then(Value::as_array) .and_then(|processes| { @@ -783,9 +790,51 @@ fn fetch_current_process_status_in( }) }) .cloned(); + let current = merge_active_process_status(state, summary, |request| session.request(request))?; Ok((statuses, current)) } +fn merge_active_process_status( + state: &AdapterState, + summary: Option, + mut request: F, +) -> Result> +where + F: FnMut(Value) -> Result, +{ + let Some(mut summary) = summary else { + return Ok(None); + }; + if summary.get("lifecycle").and_then(Value::as_str) != Some("active") { + return Ok(Some(summary)); + } + let active_statuses = request(client_user_request( + state, + json!({ + "type": "list_processes", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + }), + ))?; + let active = active_statuses + .get("processes") + .and_then(Value::as_array) + .and_then(|processes| { + processes.iter().find(|process| { + process.get("process").and_then(Value::as_str) == Some(state.process.as_str()) + }) + }); + if let (Some(summary), Some(active)) = + (summary.as_object_mut(), active.and_then(Value::as_object)) + { + for (key, value) in active { + summary.entry(key.clone()).or_insert_with(|| value.clone()); + } + } + Ok(Some(summary)) +} + pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result { let coordinator = crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); @@ -859,27 +908,7 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result snapshots, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime terminal snapshot observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - let process_status_request = - if inject_process_status_failure && !process_status_failure_injected { - process_status_failure_injected = true; - Err(anyhow!("injected process-status transport failure")) - } else { - fetch_current_process_status_in(current_session, state) - }; - let (process_statuses, process_status) = match process_status_request { - Ok(status) => status, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime terminal process observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if !has_current_runtime_task(&task_snapshots) { - if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { - record.node_report = json!({ - "terminal_event": record.node_report.get("terminal_event"), - "task_snapshots": task_snapshots, - "process_status": process_status, - "process_statuses": process_statuses, - }); - } - emit(outcome); - return Ok(()); - } - } let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected { snapshot_failure_injected = true; Err(anyhow!("injected task snapshot transport failure")) @@ -1219,6 +1190,22 @@ pub(crate) fn observe_services_runtime( } }; reconnect_delay = Duration::from_millis(100); + if !has_current_runtime_task(&task_snapshots) { + if let Some(mut outcome) = + terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) + { + if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { + record.node_report = json!({ + "terminal_event": record.node_report.get("terminal_event"), + "task_snapshots": task_snapshots, + "process_status": process_status, + "process_statuses": process_statuses, + }); + } + emit(outcome); + return Ok(()); + } + } if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) { let failed_task = failed_task.to_owned(); let failed_event = events @@ -1350,7 +1337,9 @@ pub(crate) fn observe_services_runtime( continue; } }; - if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) { + if let Some(mut outcome) = + terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) + { let task_snapshots = match fetch_task_snapshots_in(current_session, state) { Ok(snapshots) => snapshots, Err(snapshot_error) => { @@ -1543,68 +1532,111 @@ fn has_current_runtime_task(task_snapshots: &Value) -> bool { }) } +#[cfg(test)] +pub(crate) fn whole_process_status_code( + main_status_code: Option, + task_snapshots: &Value, +) -> Option { + let main_status_code = main_status_code?; + if main_status_code != 0 { + return Some(main_status_code); + } + + let snapshots = task_snapshots.get("snapshots").and_then(Value::as_array)?; + for snapshot in snapshots + .iter() + .filter(|snapshot| snapshot.get("current").and_then(Value::as_bool) == Some(true)) + { + match snapshot.get("state").and_then(Value::as_str) { + Some("completed") => {} + Some("failed" | "cancelled" | "failed_awaiting_action") => { + return Some( + snapshot + .get("status_code") + .and_then(Value::as_i64) + .and_then(|status| i32::try_from(status).ok()) + .filter(|status| *status != 0) + .unwrap_or(1), + ); + } + Some("queued" | "running") => return None, + _ => return Some(1), + } + } + Some(0) +} + fn terminal_runtime_outcome( coordinator: &str, - state: &AdapterState, + _state: &AdapterState, events: &Value, + process_status: Option<&Value>, ) -> Option { + let final_result = process_status? + .get("final_result") + .and_then(Value::as_str)?; + let status_code = match final_result { + "completed" => 0, + "failed" | "cancelled" => 1, + _ => return None, + }; let event = events .get("events") - .and_then(Value::as_array)? - .get(state.runtime_event_count..)? - .iter() - .rev() - .find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?; - let status_code = event - .get("status_code") - .and_then(Value::as_i64) - .map(|status| status as i32) - .or_else( - || match event.get("terminal_state").and_then(Value::as_str) { - Some("completed") => Some(0), - Some("failed" | "cancelled") => Some(1), - _ => None, - }, - ); + .and_then(Value::as_array) + .and_then(|events| { + events.iter().rev().find(|event| { + event.get("executor").and_then(Value::as_str) == Some("coordinator_main") + }) + }); Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord { coordinator: coordinator.to_owned(), node: event - .get("node") + .and_then(|event| event.get("node")) .and_then(Value::as_str) + .or_else(|| { + process_status + .and_then(|status| status.get("connected_nodes")) + .and_then(Value::as_array) + .and_then(|nodes| nodes.first()) + .and_then(Value::as_str) + }) .unwrap_or("coordinator-main") .to_owned(), - node_report: json!({ "terminal_event": event }), + node_report: json!({ + "terminal_event": event, + "process_status": process_status, + }), task_events: events.clone(), placed_task_launched: true, - status_code, + status_code: Some(status_code), stdout_bytes: event - .get("stdout_bytes") + .and_then(|event| event.get("stdout_bytes")) .and_then(Value::as_u64) .unwrap_or(0), stderr_bytes: event - .get("stderr_bytes") + .and_then(|event| event.get("stderr_bytes")) .and_then(Value::as_u64) .unwrap_or(0), stdout_tail: event - .get("stdout_tail") + .and_then(|event| event.get("stdout_tail")) .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stderr_tail: event - .get("stderr_tail") + .and_then(|event| event.get("stderr_tail")) .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stdout_truncated: event - .get("stdout_truncated") + .and_then(|event| event.get("stdout_truncated")) .and_then(Value::as_bool) .unwrap_or(false), stderr_truncated: event - .get("stderr_truncated") + .and_then(|event| event.get("stderr_truncated")) .and_then(Value::as_bool) .unwrap_or(false), artifact_path: event - .get("artifact_path") + .and_then(|event| event.get("artifact_path")) .and_then(Value::as_str) .map(str::to_owned), event_count: events @@ -1690,6 +1722,53 @@ mod transactional_launch_tests { assert!(error.contains("no virtual process was created")); } + #[test] + fn durable_process_summary_is_terminal_authority_after_event_rotation() { + let state = AdapterState { + runtime_event_count: 10_000, + ..AdapterState::default() + }; + let completed = terminal_runtime_outcome( + "127.0.0.1:1", + &state, + &json!({ "events": [] }), + Some(&json!({ + "process": state.process.as_str(), + "lifecycle": "recent_terminal", + "final_result": "completed", + "connected_nodes": [] + })), + ) + .expect("the durable summary should terminate observation"); + let RuntimeContinuationOutcome::Terminal(completed) = completed else { + panic!("expected terminal outcome"); + }; + assert_eq!(completed.status_code, Some(0)); + + let failed = terminal_runtime_outcome( + "127.0.0.1:1", + &state, + &json!({ + "events": [{ + "executor": "coordinator_main", + "terminal_state": "completed", + "status_code": 0 + }] + }), + Some(&json!({ + "process": state.process.as_str(), + "lifecycle": "recent_terminal", + "final_result": "failed", + "connected_nodes": [] + })), + ) + .expect("the aggregate summary should override a successful main event"); + let RuntimeContinuationOutcome::Terminal(failed) = failed else { + panic!("expected terminal outcome"); + }; + assert_eq!(failed.status_code, Some(1)); + } + #[test] fn failed_debug_launch_reconnects_and_aborts_the_process() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/clusterflux-dap/src/tests.rs b/crates/clusterflux-dap/src/tests.rs index d64ac6e..ed1c82f 100644 --- a/crates/clusterflux-dap/src/tests.rs +++ b/crates/clusterflux-dap/src/tests.rs @@ -704,11 +704,40 @@ fn terminal_record_without_snapshots_clears_active_threads() { #[test] fn source_locals_infer_clusterflux_api_values_from_runtime_state() { - let mut state = AdapterState::default(); - let project = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../tests/fixtures/runtime-conformance") - .canonicalize() + let project = std::env::temp_dir().join(format!( + "clusterflux-dap-source-locals-{}", + std::process::id() + )); + let src = project.join("src"); + fs::create_dir_all(&src).unwrap(); + fs::write( + src.join("lib.rs"), + r#"async fn run_build_workflow() { + let source = prepare_source_snapshot(); + let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) + .name("compile linux") + .env(linux_env()) + .start() + .await .unwrap(); + let linux_thread = linux.virtual_thread_id(); + let linux_artifact = linux.join().await.unwrap(); + let package = clusterflux::spawn::async_task_with_arg( + vec![linux_artifact.clone()], + package_release, + ) + .name("package artifacts") + .env(linux_env()) + .start() + .await + .unwrap(); + let package_artifact = package.join().await.unwrap(); +} +"#, + ) + .unwrap(); + + let mut state = AdapterState::default(); state.project = project.to_string_lossy().into_owned(); state.source_path = "src/lib.rs".to_owned(); let source = fs::read_to_string(project.join(&state.source_path)).unwrap(); @@ -744,6 +773,8 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() { variable["name"] == "unavailable-local-diagnostic" && variable["type"] == "unavailable-local" })); + + let _ = fs::remove_dir_all(project); } #[test] @@ -863,11 +894,20 @@ fn package_release(inputs: Vec) -> Artifact { #[test] fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { + let project = std::env::temp_dir().join(format!( + "clusterflux-dap-wasm-locals-{}", + std::process::id() + )); + let src = project.join("src"); + fs::create_dir_all(&src).unwrap(); + fs::write( + src.join("lib.rs"), + "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n input + 1\n}\n", + ) + .unwrap(); + let mut state = AdapterState::default(); - state.project = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../tests/fixtures/runtime-conformance") - .to_string_lossy() - .into_owned(); + state.project = project.to_string_lossy().into_owned(); state.source_path = "src/lib.rs".to_owned(); let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap(); state.threads.get_mut(&MAIN_THREAD).unwrap().line = source @@ -895,6 +935,8 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { assert!(!locals .iter() .any(|variable| variable["name"] == "wasm-local-diagnostic")); + + let _ = fs::remove_dir_all(project); } #[test] @@ -922,6 +964,98 @@ fn detects_current_failed_attempt_awaiting_operator_action() { ); } +#[test] +fn whole_process_terminal_status_covers_every_current_logical_task_attempt() { + let snapshots = |items: serde_json::Value| json!({ "snapshots": items }); + + assert_eq!( + whole_process_status_code( + Some(0), + &snapshots(json!([ + { + "task": "child", + "current": true, + "state": "completed", + "status_code": 0 + } + ])) + ), + Some(0), + "a successful main and final child must succeed" + ); + for (state, status_code) in [("failed", 23), ("cancelled", 1)] { + assert_eq!( + whole_process_status_code( + Some(0), + &snapshots(json!([ + { + "task": "child", + "current": true, + "state": state, + "status_code": status_code + } + ])) + ), + Some(status_code), + "a terminal child must determine the whole-process result" + ); + } + assert_eq!( + whole_process_status_code( + Some(0), + &snapshots(json!([ + { + "task": "accepted-failure", + "current": true, + "state": "failed", + "command_state": "failure_accepted", + "status_code": 17 + } + ])) + ), + Some(17), + "accepting an AwaitOperator failure releases the process but does not turn failure into success" + ); + assert_eq!( + whole_process_status_code( + Some(0), + &snapshots(json!([ + { + "task": "restarted", + "attempt_id": "attempt-1", + "current": false, + "state": "failed", + "status_code": 7 + }, + { + "task": "restarted", + "attempt_id": "attempt-2", + "current": true, + "state": "completed", + "status_code": 0 + } + ])) + ), + Some(0), + "a successful replacement attempt is authoritative over stale failed attempts" + ); + assert_eq!( + whole_process_status_code( + Some(9), + &snapshots(json!([ + { + "task": "child", + "current": true, + "state": "completed", + "status_code": 0 + } + ])) + ), + Some(9), + "a failed coordinator main cannot be masked by successful children" + ); +} + #[test] fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() { let mut state = AdapterState::default(); diff --git a/crates/clusterflux-dap/src/virtual_model.rs b/crates/clusterflux-dap/src/virtual_model.rs index 5dcba43..9ef76eb 100644 --- a/crates/clusterflux-dap/src/virtual_model.rs +++ b/crates/clusterflux-dap/src/virtual_model.rs @@ -114,7 +114,7 @@ impl Default for AdapterState { project, source_path: "src/lib.rs".to_owned(), runtime_backend: RuntimeBackend::Simulated, - coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(), + coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(), tenant: TenantId::from("tenant"), project_id: ProjectId::from("project"), actor_user: UserId::from("dap"), diff --git a/crates/clusterflux-node/src/assignment_runner.rs b/crates/clusterflux-node/src/assignment_runner.rs index 0e6ce6d..54f5a00 100644 --- a/crates/clusterflux-node/src/assignment_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Read; use std::path::PathBuf; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -46,6 +46,40 @@ use validation::{ resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot, }; +#[derive(Debug)] +struct AssignmentExecutionError { + message: String, + stdout_source_bytes: u64, + stderr_source_bytes: u64, +} + +impl std::fmt::Display for AssignmentExecutionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for AssignmentExecutionError {} + +fn execution_error_with_log_bytes( + message: impl Into, + stdout_source_bytes: &AtomicU64, + stderr_source_bytes: &AtomicU64, +) -> Box { + Box::new(AssignmentExecutionError { + message: message.into(), + stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed), + stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed), + }) +} + +pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) { + error + .downcast_ref::() + .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes)) + .unwrap_or((0, 0)) +} + pub(crate) fn run_verified_wasmtime_assignment( args: &Args, task: &RuntimeTask, @@ -95,6 +129,8 @@ pub(crate) fn run_verified_wasmtime_assignment( return Err("Wasm entrypoint assignment omitted its descriptor export".into()); } }; + let command_stdout_source_bytes = Arc::new(AtomicU64::new(0)); + let command_stderr_source_bytes = Arc::new(AtomicU64::new(0)); let (stdout, boundary_result) = match abi { WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => { let invocation = WasmTaskInvocation::new( @@ -102,18 +138,28 @@ pub(crate) fn run_verified_wasmtime_assignment( task_spec.task_instance.clone(), task_spec.args.clone(), ); - let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host( - &module, - expected_bundle_digest, - export, - &invocation, - Box::new(CoordinatorWasmTaskHost::new( - args, - task, - node_private_key, + let result = WasmtimeTaskRuntime::new()? + .run_task_export_verified_with_task_host( &module, - )?), - )?; + expected_bundle_digest, + export, + &invocation, + Box::new(CoordinatorWasmTaskHost::new( + args, + task, + node_private_key, + &module, + Arc::clone(&command_stdout_source_bytes), + Arc::clone(&command_stderr_source_bytes), + )?), + ) + .map_err(|error| { + execution_error_with_log_bytes( + error.to_string(), + &command_stdout_source_bytes, + &command_stderr_source_bytes, + ) + })?; if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() { eprintln!( "clusterflux debug control: Wasm assignment returned for task {}", @@ -122,17 +168,35 @@ pub(crate) fn run_verified_wasmtime_assignment( } match result.outcome { WasmTaskOutcome::Completed => { - let boundary = result.result.ok_or("completed Wasm task omitted result")?; + let boundary = result.result.ok_or_else(|| { + execution_error_with_log_bytes( + "completed Wasm task omitted result", + &command_stdout_source_bytes, + &command_stderr_source_bytes, + ) + })?; ( - format!("{}\n", serde_json::to_string(&boundary)?), + format!( + "{}\n", + serde_json::to_string(&boundary).map_err(|error| { + execution_error_with_log_bytes( + error.to_string(), + &command_stdout_source_bytes, + &command_stderr_source_bytes, + ) + })? + ), Some(boundary), ) } WasmTaskOutcome::Failed => { - return Err(result - .error - .unwrap_or_else(|| "Wasm task failed without an error".to_owned()) - .into()) + return Err(execution_error_with_log_bytes( + result + .error + .unwrap_or_else(|| "Wasm task failed without an error".to_owned()), + &command_stdout_source_bytes, + &command_stderr_source_bytes, + )) } } } @@ -140,12 +204,18 @@ pub(crate) fn run_verified_wasmtime_assignment( let task_id = TaskInstanceId::new(task.task.clone()); let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone())); let manifest = artifacts.flush(); + let stdout_source_bytes = command_stdout_source_bytes + .load(Ordering::Relaxed) + .saturating_add(stdout.len() as u64); + let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed); Ok(( CommandOutput { virtual_thread: task_id, status_code: Some(0), stdout, stderr: String::new(), + stdout_source_bytes, + stderr_source_bytes, stdout_truncated: false, stderr_truncated: false, log_backpressured: false, @@ -176,6 +246,8 @@ struct CoordinatorWasmTaskHost { next_handle_id: u64, handles: Arc>>, command_status: Arc>>, + command_stdout_source_bytes: Arc, + command_stderr_source_bytes: Arc, cancellation_requested: Arc, abort_requested: Arc, debug_control: Arc, @@ -188,6 +260,8 @@ impl CoordinatorWasmTaskHost { parent: &RuntimeTask, node_private_key: &str, module: &[u8], + command_stdout_source_bytes: Arc, + command_stderr_source_bytes: Arc, ) -> Result> { let task_spec = parent .task_spec @@ -268,6 +342,8 @@ impl CoordinatorWasmTaskHost { next_handle_id: 1, handles, command_status, + command_stdout_source_bytes, + command_stderr_source_bytes, cancellation_requested, abort_requested, debug_control, @@ -293,7 +369,16 @@ impl CoordinatorWasmTaskHost { runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?; let execution = run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key); - let (terminal_state, status_code, stdout, stderr, result, retained) = match execution { + let ( + terminal_state, + status_code, + stdout, + stderr, + stdout_source_bytes, + stderr_source_bytes, + result, + retained, + ) = match execution { Ok((output, _manifest, result)) => { let retained = retained_result_artifact( self.args.project_root.as_deref(), @@ -306,20 +391,40 @@ impl CoordinatorWasmTaskHost { output.status_code, output.stdout, output.stderr, + output.stdout_source_bytes, + output.stderr_source_bytes, result, retained, ), - Err(error) => ("failed", Some(1), String::new(), error, None, None), + Err(error) => ( + "failed", + Some(1), + String::new(), + error.clone(), + output.stdout_source_bytes, + output + .stderr_source_bytes + .saturating_add(error.len() as u64), + None, + None, + ), } } - Err(error) => ( - "failed", - Some(1), - String::new(), - error.to_string(), - None, - None, - ), + Err(error) => { + let (stdout_source_bytes, stderr_source_bytes) = + assignment_error_log_bytes(error.as_ref()); + let error = error.to_string(); + ( + "failed", + Some(1), + String::new(), + error.clone(), + stdout_source_bytes, + stderr_source_bytes.saturating_add(error.len() as u64), + None, + None, + ) + } }; let artifact_path = retained .as_ref() @@ -339,8 +444,8 @@ impl CoordinatorWasmTaskHost { "task": runtime_task.task, "terminal_state": terminal_state, "status_code": status_code, - "stdout_bytes": stdout.len(), - "stderr_bytes": stderr.len(), + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": stdout, "stderr_tail": stderr, "stdout_truncated": false, @@ -676,6 +781,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost { let mut runner = CoordinatorControlledProcessRunner::new( self, Duration::from_millis(request.timeout_ms), + configured_secrets.clone(), ); let output = LinuxRootlessPodmanBackend .execute_local_checkout_task( diff --git a/crates/clusterflux-node/src/assignment_runner/process_runner.rs b/crates/clusterflux-node/src/assignment_runner/process_runner.rs index 2b6dbd3..0e2da69 100644 --- a/crates/clusterflux-node/src/assignment_runner/process_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner/process_runner.rs @@ -1,4 +1,83 @@ use super::*; +use std::sync::mpsc::{self, Receiver, SyncSender}; + +struct LiveLogChunk { + stream: &'static str, + offset: u64, + source_bytes: u64, + bytes: Vec, + truncated: bool, +} + +fn append_bounded_tail(tail: &mut Vec, bytes: &[u8], maximum: usize) { + if maximum == 0 { + tail.clear(); + return; + } + if bytes.len() >= maximum { + tail.clear(); + tail.extend_from_slice(&bytes[bytes.len() - maximum..]); + return; + } + let overflow = tail + .len() + .saturating_add(bytes.len()) + .saturating_sub(maximum); + if overflow > 0 { + tail.drain(..overflow); + } + tail.extend_from_slice(bytes); +} + +fn redact_safe_live_log_prefix( + pending: &[u8], + configured_secrets: &[String], + final_chunk: bool, +) -> Option<(usize, String)> { + if pending.is_empty() { + return None; + } + let secret_bytes = configured_secrets + .iter() + .filter(|secret| secret.len() >= 4) + .map(String::as_bytes) + .collect::>(); + let maximum_secret_bytes = secret_bytes + .iter() + .map(|secret| secret.len()) + .max() + .unwrap_or(0); + if !final_chunk && pending.len() <= maximum_secret_bytes { + return None; + } + let mut consumed = if final_chunk { + pending.len() + } else { + pending.len() - maximum_secret_bytes + }; + loop { + let previous = consumed; + for secret in &secret_bytes { + for start in 0..=pending.len().saturating_sub(secret.len()) { + let end = start + secret.len(); + if start < consumed && end > consumed && &pending[start..end] == *secret { + consumed = end; + } + } + } + if consumed == previous { + break; + } + } + if consumed == 0 { + return None; + } + let text = redact_configured_values( + String::from_utf8_lossy(&pending[..consumed]).into_owned(), + configured_secrets, + ); + Some((consumed, text)) +} pub(super) struct CoordinatorControlledProcessRunner { pub(super) args: Args, @@ -7,13 +86,20 @@ pub(super) struct CoordinatorControlledProcessRunner { pub(super) node_private_key: String, pub(super) debug_control: Arc, pub(super) command_status: Arc>>, + pub(super) stdout_source_bytes: Arc, + pub(super) stderr_source_bytes: Arc, pub(super) timeout: Duration, + pub(super) configured_secrets: Vec, } impl CoordinatorControlledProcessRunner { const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1; - pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self { + pub(super) fn new( + host: &CoordinatorWasmTaskHost, + timeout: Duration, + configured_secrets: Vec, + ) -> Self { Self { args: host.args.clone(), process: host.process.clone(), @@ -21,7 +107,10 @@ impl CoordinatorControlledProcessRunner { node_private_key: host.node_private_key.clone(), debug_control: Arc::clone(&host.debug_control), command_status: Arc::clone(&host.command_status), + stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes), + stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes), timeout, + configured_secrets, } } @@ -201,10 +290,18 @@ impl CoordinatorControlledProcessRunner { fn drain_bounded( mut reader: impl Read + Send + 'static, maximum: usize, + stream: &'static str, + sender: SyncSender, + configured_secrets: Vec, + source_bytes_total: Arc, ) -> thread::JoinHandle, String>> { thread::spawn(move || { let mut captured = Vec::new(); let mut buffer = [0_u8; 16 * 1024]; + let stream_base = source_bytes_total.load(Ordering::Relaxed); + let mut source_bytes_read = 0_u64; + let mut pending_offset = stream_base; + let mut pending = Vec::new(); loop { let count = reader .read(&mut buffer) @@ -212,12 +309,128 @@ impl CoordinatorControlledProcessRunner { if count == 0 { break; } - let remaining = maximum.saturating_sub(captured.len()); - captured.extend_from_slice(&buffer[..count.min(remaining)]); + let _ = source_bytes_total.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |current| Some(current.saturating_add(count as u64)), + ); + source_bytes_read = source_bytes_read.saturating_add(count as u64); + append_bounded_tail(&mut captured, &buffer[..count], maximum); + pending.extend_from_slice(&buffer[..count]); + if let Some((consumed, text)) = + redact_safe_live_log_prefix(&pending, &configured_secrets, false) + { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: pending_offset, + source_bytes: consumed as u64, + bytes: text.into_bytes(), + truncated: false, + }); + pending.drain(..consumed); + pending_offset = pending_offset.saturating_add(consumed as u64); + } + } + if let Some((consumed, text)) = + redact_safe_live_log_prefix(&pending, &configured_secrets, true) + { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: pending_offset, + source_bytes: consumed as u64, + bytes: text.into_bytes(), + truncated: false, + }); + } + if source_bytes_read > maximum as u64 { + let _ = sender.try_send(LiveLogChunk { + stream, + offset: stream_base.saturating_add(source_bytes_read), + source_bytes: 0, + bytes: b"[log output truncated at node capture limit]".to_vec(), + truncated: true, + }); } Ok(captured) }) } + + fn spawn_live_log_reporter(&self, receiver: Receiver) -> thread::JoinHandle<()> { + let args = self.args.clone(); + let process = self.process.clone(); + let task = self.task.clone(); + let node_private_key = self.node_private_key.clone(); + let configured_secrets = self.configured_secrets.clone(); + let command_status = Arc::clone(&self.command_status); + thread::spawn(move || { + let mut log_session = None; + let mut delivery_available = true; + while let Ok(chunk) = receiver.recv() { + if !delivery_available { + continue; + } + let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned(); + text = redact_configured_values(text, &configured_secrets); + let mut delivered = false; + for _ in 0..2 { + if log_session.is_none() { + log_session = CoordinatorSession::connect_with_timeouts( + &args.coordinator, + Duration::from_millis(500), + Duration::from_millis(500), + ) + .ok(); + } + let Some(session) = log_session.as_mut() else { + continue; + }; + let request = signed_node_request_json( + &args, + &node_private_key, + "report_task_log_chunk", + serde_json::json!({ + "type": "report_task_log_chunk", + "tenant": &args.tenant, + "project": &args.project, + "process": &process, + "node": &args.node, + "task": &task, + "stream": chunk.stream, + "offset": chunk.offset, + "source_bytes": chunk.source_bytes, + "text": &text, + "truncated": chunk.truncated, + }), + ); + let result = match request { + Ok(request) => session + .request(request) + .map(|_| ()) + .map_err(|error| error.to_string()), + Err(error) => Err(error.to_string()), + }; + match result { + Ok(()) => { + delivered = true; + break; + } + Err(_) => { + log_session = None; + } + } + } + if !delivered { + delivery_available = false; + if let Ok(mut current) = command_status.lock() { + *current = Some( + "live log delivery was interrupted; final bounded output remains available" + .to_owned(), + ); + } + } + } + }) + } } impl ProcessRunner for CoordinatorControlledProcessRunner { @@ -245,12 +458,17 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { command.program, command.args.join(" ") )); + let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64); let stdout = Self::drain_bounded( child .stdout .take() .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, + "stdout", + live_log_sender.clone(), + self.configured_secrets.clone(), + Arc::clone(&self.stdout_source_bytes), ); let stderr = Self::drain_bounded( child @@ -258,11 +476,19 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .take() .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, + "stderr", + live_log_sender, + self.configured_secrets.clone(), + Arc::clone(&self.stderr_source_bytes), ); + let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver); let mut session = match CoordinatorSession::connect(&self.args.coordinator) { Ok(session) => session, Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); + let _ = stdout.join(); + let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "establish execution control channel: {error}" ))); @@ -277,6 +503,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Ok(None) => {} Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); + let _ = stdout.join(); + let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(error.to_string())); } } @@ -289,6 +518,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "native command exceeded wall-clock timeout of {} ms", self.timeout.as_millis() @@ -303,6 +533,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(BackendError::Cancelled( "coordinator requested cancellation or abort".to_owned(), )); @@ -312,6 +543,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); + let _ = live_log_reporter.join(); return Err(error); } } @@ -360,6 +592,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .join() .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))? .map_err(BackendError::Command)?; + live_log_reporter + .join() + .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?; self.set_command_status(format!( "native command exited with status {:?}", status.code() @@ -371,3 +606,52 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { }) } } + +#[cfg(test)] +mod tests { + use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner}; + use std::io::Cursor; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::{mpsc, Arc}; + + #[test] + fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() { + let secrets = vec!["correct-horse".to_owned()]; + let mut pending = b"prefix correct-".to_vec(); + let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap(); + pending.drain(..consumed); + pending.extend_from_slice(b"horse suffix"); + let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap(); + + let combined = format!("{first}{second}"); + assert_eq!(combined, "prefix [REDACTED] suffix"); + assert!(!combined.contains("correct-")); + assert!(!combined.contains("horse")); + } + + #[test] + fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() { + let source_bytes = Arc::new(AtomicU64::new(5)); + let (sender, receiver) = mpsc::sync_channel(8); + let reader = CoordinatorControlledProcessRunner::drain_bounded( + Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()), + 8, + "stdout", + sender, + Vec::new(), + Arc::clone(&source_bytes), + ); + let captured = reader.join().unwrap().unwrap(); + let chunks = receiver.into_iter().collect::>(); + + assert_eq!(captured, b"opqrstuv"); + assert_eq!(source_bytes.load(Ordering::Relaxed), 37); + assert_eq!( + chunks.iter().map(|chunk| chunk.source_bytes).sum::(), + 32 + ); + assert_eq!(chunks[0].offset, 5); + assert_eq!(chunks.last().unwrap().offset, 37); + assert!(chunks.iter().any(|chunk| chunk.truncated)); + } +} diff --git a/crates/clusterflux-node/src/assignment_runner/tests.rs b/crates/clusterflux-node/src/assignment_runner/tests.rs index 4d2d864..d654d1f 100644 --- a/crates/clusterflux-node/src/assignment_runner/tests.rs +++ b/crates/clusterflux-node/src/assignment_runner/tests.rs @@ -48,7 +48,10 @@ fn test_controlled_runner( ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), + stdout_source_bytes: Arc::new(AtomicU64::new(0)), + stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout, + configured_secrets: Vec::new(), } } @@ -89,7 +92,10 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() { ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), + stdout_source_bytes: Arc::new(AtomicU64::new(0)), + stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout: Duration::from_secs(30), + configured_secrets: Vec::new(), }; let started = Instant::now(); let error = runner diff --git a/crates/clusterflux-node/src/command_runner.rs b/crates/clusterflux-node/src/command_runner.rs index a1ebd33..790b838 100644 --- a/crates/clusterflux-node/src/command_runner.rs +++ b/crates/clusterflux-node/src/command_runner.rs @@ -1,6 +1,6 @@ use clusterflux_core::{ - CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, - VfsOverlay, VfsPath, + CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay, + VfsPath, }; use serde::{Deserialize, Serialize}; @@ -42,6 +42,8 @@ pub struct CommandOutput { pub status_code: Option, pub stdout: String, pub stderr: String, + pub stdout_source_bytes: u64, + pub stderr_source_bytes: u64, pub stdout_truncated: bool, pub stderr_truncated: bool, pub log_backpressured: bool, @@ -83,6 +85,8 @@ impl LocalCommandExecutor { &output.stderr, max_log_bytes, ); + let stdout_source_bytes = output.stdout.len() as u64; + let stderr_source_bytes = output.stderr.len() as u64; let staged_artifact = if let Some(path) = command.stage_stdout_as { Some(overlay.write( path, @@ -98,6 +102,8 @@ impl LocalCommandExecutor { status_code: output.status.code(), stdout: logs.stdout, stderr: logs.stderr, + stdout_source_bytes, + stderr_source_bytes, stdout_truncated: logs.stdout_truncated, stderr_truncated: logs.stderr_truncated, log_backpressured: logs.backpressured, @@ -107,24 +113,20 @@ impl LocalCommandExecutor { } pub(super) fn capture_command_logs( - task: &TaskInstanceId, + _task: &TaskInstanceId, stdout: &[u8], stderr: &[u8], max_log_bytes: usize, ) -> CapturedCommandLogs { - let mut logs = LogBuffer::new(max_log_bytes); - logs.push(task.clone(), stdout); - logs.push(task.clone(), stderr); - let records = logs.records(); - let stdout_record = &records[0]; - let stderr_record = &records[1]; - debug_assert_eq!(&stdout_record.task, task); - debug_assert_eq!(&stderr_record.task, task); + let stdout_truncated = stdout.len() > max_log_bytes; + let stderr_truncated = stderr.len() > max_log_bytes; + let stdout_start = stdout.len().saturating_sub(max_log_bytes); + let stderr_start = stderr.len().saturating_sub(max_log_bytes); CapturedCommandLogs { - stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(), - stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(), - stdout_truncated: stdout_record.truncated, - stderr_truncated: stderr_record.truncated, - backpressured: logs.backpressured(), + stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(), + stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(), + stdout_truncated, + stderr_truncated, + backpressured: stdout_truncated || stderr_truncated, } } diff --git a/crates/clusterflux-node/src/coordinator_session.rs b/crates/clusterflux-node/src/coordinator_session.rs index 997e0f0..13e64ec 100644 --- a/crates/clusterflux-node/src/coordinator_session.rs +++ b/crates/clusterflux-node/src/coordinator_session.rs @@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity; use clusterflux_control::ControlSession; use clusterflux_core::coordinator_wire_request; use serde_json::Value; +use std::time::Duration; pub(crate) struct CoordinatorSession { inner: ControlSession, @@ -15,6 +16,16 @@ impl CoordinatorSession { }) } + pub(crate) fn connect_with_timeouts( + addr: &str, + connect_timeout: Duration, + io_timeout: Duration, + ) -> Result> { + Ok(Self { + inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?, + }) + } + pub(crate) fn request(&mut self, value: Value) -> Result> { let request_id = format!("node-{}", self.inner.requests() + 1); let wire_request = coordinator_wire_request(request_id, value); diff --git a/crates/clusterflux-node/src/daemon.rs b/crates/clusterflux-node/src/daemon.rs index 513fc83..b223ebc 100644 --- a/crates/clusterflux-node/src/daemon.rs +++ b/crates/clusterflux-node/src/daemon.rs @@ -11,7 +11,7 @@ use clusterflux_core::{ }; use serde_json::{json, Value}; -use crate::assignment_runner::run_verified_wasmtime_assignment; +use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment}; #[cfg(test)] use crate::coordinator_session::control_endpoint_identity; use crate::coordinator_session::CoordinatorSession; @@ -464,6 +464,8 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, + 0, + 0, ); } @@ -498,9 +500,13 @@ fn run_runtime_task( debug_command, node_private_key, &error, + output.stdout_source_bytes, + output.stderr_source_bytes, ), }, Err(error) => { + let (stdout_source_bytes, stderr_source_bytes) = + assignment_error_log_bytes(error.as_ref()); let error = error.to_string(); if error.contains("task execution cancelled:") { record_cancelled_task( @@ -512,6 +518,8 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, + stdout_source_bytes, + stderr_source_bytes, ) } else { record_failed_task( @@ -524,6 +532,8 @@ fn run_runtime_task( debug_command, node_private_key, &error, + stdout_source_bytes, + stderr_source_bytes, ) } } @@ -634,13 +644,12 @@ mod tests { #[test] fn hosted_url_remains_an_https_control_endpoint() { assert_eq!( - control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(), - "https://clusterflux.michelpaulissen.com/api/v1/control" + control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(), + "https://clusterflux.lesstuff.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") - .unwrap(), - "https://clusterflux.michelpaulissen.com/api/v1/control" + control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), + "https://clusterflux.lesstuff.com/api/v1/control" ); assert_eq!( control_endpoint_identity("127.0.0.1:7999").unwrap(), diff --git a/crates/clusterflux-node/src/lib.rs b/crates/clusterflux-node/src/lib.rs index f35c3d6..d9db699 100644 --- a/crates/clusterflux-node/src/lib.rs +++ b/crates/clusterflux-node/src/lib.rs @@ -741,7 +741,7 @@ mod tests { } #[test] - fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() { + fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() { let invocation = CommandInvocation { program: "cargo".to_owned(), args: vec!["build".to_owned()], @@ -774,7 +774,7 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "abcd"); + assert_eq!(output.stdout, "cdef"); assert!(output.stdout_truncated); assert!(output.log_backpressured); assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6); @@ -992,7 +992,7 @@ mod tests { #[cfg(unix)] #[test] - fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() { + fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() { let executor = LocalCommandExecutor { node: clusterflux_core::NodeId::from("node"), hosted_control_plane: false, @@ -1021,15 +1021,15 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "abcd"); - assert_eq!(output.stderr, ""); + assert_eq!(output.stdout, "cdef"); + assert_eq!(output.stderr, "err"); assert!(output.stdout_truncated); - assert!(output.stderr_truncated); + assert!(!output.stderr_truncated); assert!(output.log_backpressured); } #[test] - fn public_node_crate_does_not_require_hosted_private_types() { + fn public_node_crate_does_not_require_hosted_service_types() { let _tenant = TenantId::from("tenant"); let _project = ProjectId::from("project"); let _backend = LinuxRootlessPodmanBackend; diff --git a/crates/clusterflux-node/src/main.rs b/crates/clusterflux-node/src/main.rs index 8f2960b..2ada582 100644 --- a/crates/clusterflux-node/src/main.rs +++ b/crates/clusterflux-node/src/main.rs @@ -8,5 +8,34 @@ mod task_artifacts; mod task_reports; fn main() -> Result<(), Box> { + let raw_args = std::env::args().skip(1).collect::>(); + match raw_args.as_slice() { + [flag] if matches!(flag.as_str(), "--version" | "-V") => { + println!("clusterflux-node {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + [flag] if matches!(flag.as_str(), "--help" | "-h") => { + println!( + "Clusterflux node worker.\n\n\ + Usage: clusterflux-node --coordinator [OPTIONS]\n\n\ + Options:\n \ + --coordinator \n \ + --tenant [default: tenant]\n \ + --project-id [default: project]\n \ + --node [default: node]\n \ + --project-root \n \ + --enrollment-grant \n \ + --public-key \n \ + --worker\n \ + --emit-ready\n \ + --control-poll-ms \n \ + --assignment-poll-ms [default: 500]\n \ + -h, --help\n \ + -V, --version" + ); + return Ok(()); + } + _ => {} + } daemon::run() } diff --git a/crates/clusterflux-node/src/task_reports.rs b/crates/clusterflux-node/src/task_reports.rs index a700e76..afaf0c8 100644 --- a/crates/clusterflux-node/src/task_reports.rs +++ b/crates/clusterflux-node/src/task_reports.rs @@ -47,8 +47,8 @@ pub(crate) fn record_completed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": output.stdout.len(), - "stderr_bytes": output.stderr.len(), + "stdout_bytes": output.stdout_source_bytes, + "stderr_bytes": output.stderr_source_bytes, "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -85,8 +85,8 @@ pub(crate) fn record_completed_task( "node": &args.node, "task": &task.task, "status_code": output.status_code, - "stdout_bytes": output.stdout.len(), - "stderr_bytes": output.stderr.len(), + "stdout_bytes": output.stdout_source_bytes, + "stderr_bytes": output.stderr_source_bytes, "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -123,8 +123,11 @@ pub(crate) fn record_failed_task( debug_command: Value, node_private_key: &str, error: &str, + stdout_source_bytes: u64, + command_stderr_source_bytes: u64, ) -> Result> { let error = bounded_runtime_error(error); + let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64); let log_event = session.request(signed_node_request_json( args, node_private_key, @@ -136,8 +139,8 @@ pub(crate) fn record_failed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": 0, - "stderr_bytes": error.len(), + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -175,8 +178,8 @@ pub(crate) fn record_failed_task( "task": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": 0, - "stderr_bytes": error.len(), + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -189,6 +192,8 @@ pub(crate) fn record_failed_task( Ok(failed_node_report( task, &error, + stdout_source_bytes, + stderr_source_bytes, registration, heartbeat, capability_report, @@ -210,6 +215,8 @@ pub(crate) fn record_cancelled_task( capability_report: Value, debug_command: Value, node_private_key: &str, + stdout_source_bytes: u64, + stderr_source_bytes: u64, ) -> Result> { let recorded = session.request(signed_node_request_json( args, @@ -224,12 +231,12 @@ pub(crate) fn record_cancelled_task( "task": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": 0, - "stderr_bytes": 0, + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": false, - "stderr_truncated": false, + "stdout_truncated": stdout_source_bytes > 0, + "stderr_truncated": stderr_source_bytes > 0, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, @@ -238,6 +245,8 @@ pub(crate) fn record_cancelled_task( )?)?; Ok(cancelled_node_report( task, + stdout_source_bytes, + stderr_source_bytes, registration, heartbeat, capability_report, @@ -281,8 +290,8 @@ pub(crate) fn completed_node_report( "virtual_thread": output.virtual_thread, "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" }, "status_code": output.status_code, - "stdout_bytes": output.stdout.len(), - "stderr_bytes": output.stderr.len(), + "stdout_bytes": output.stdout_source_bytes, + "stderr_bytes": output.stderr_source_bytes, "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -305,6 +314,8 @@ pub(crate) fn completed_node_report( #[allow(clippy::too_many_arguments)] pub(crate) fn cancelled_node_report( task: &RuntimeTask, + stdout_source_bytes: u64, + stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -318,12 +329,12 @@ pub(crate) fn cancelled_node_report( "virtual_thread": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": 0, - "stderr_bytes": 0, + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": false, - "stderr_truncated": false, + "stdout_truncated": stdout_source_bytes > 0, + "stderr_truncated": stderr_source_bytes > 0, "log_backpressured": false, "staged_artifact": null, "large_bytes_uploaded": false, @@ -343,6 +354,8 @@ pub(crate) fn cancelled_node_report( pub(crate) fn failed_node_report( task: &RuntimeTask, error: &str, + stdout_source_bytes: u64, + stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -357,8 +370,8 @@ pub(crate) fn failed_node_report( "virtual_thread": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": 0, - "stderr_bytes": error.len(), + "stdout_bytes": stdout_source_bytes, + "stderr_bytes": stderr_source_bytes, "stdout_tail": "", "stderr_tail": error, "stdout_truncated": false, @@ -377,3 +390,40 @@ pub(crate) fn failed_node_report( "coordinator_response": coordinator_response, }) } + +#[cfg(test)] +mod tests { + use super::*; + use clusterflux_core::TaskInstanceId; + + #[test] + fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() { + let report = completed_node_report( + CommandOutput { + virtual_thread: TaskInstanceId::from("task"), + status_code: Some(0), + stdout: "bounded tail".to_owned(), + stderr: String::new(), + stdout_source_bytes: 519, + stderr_source_bytes: 0, + stdout_truncated: true, + stderr_truncated: false, + log_backpressured: false, + staged_artifact: None, + }, + false, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + Value::Null, + 1, + ); + + assert_eq!(report["stdout_bytes"], 519); + assert_eq!(report["stdout_tail"], "bounded tail"); + } +} diff --git a/docs/artifacts.md b/docs/artifacts.md index 9320564..a1521a7 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context. The reverse stream counts framing, base64 expansion, failed bytes, and abandoned transfers. The CLI verifies the final digest. Hosted policy may impose per-project, per-tenant, and per-account size, concurrency, and period limits. -Operators may disable relay traffic as an emergency safety control; one tenant's -period usage does not consume a shared customer byte quota. +There is no hosted global circuit breaker, and one tenant's period usage does +not consume a shared customer byte quota. diff --git a/docs/contributing/releases.md b/docs/contributing/releases.md deleted file mode 100644 index 6dc422e..0000000 --- a/docs/contributing/releases.md +++ /dev/null @@ -1,60 +0,0 @@ -# Release candidates - -This is a contributor and release-engineering procedure, not an end-user setup -path. Publication is a three-stage transaction: - -1. `candidate` builds immutable archives and a manifest with paths relative to - the manifest directory. -2. `live-test` downloads that exact candidate in a clean job, deploys it, and - records the full named production-shaped acceptance result plus deployment, - runtime configuration, and proxy configuration identities. -3. `final` downloads the candidate and evidence in another clean job, verifies - every binding, and publishes without rebuilding any binary. - -Set `CLUSTERFLUX_RELEASE_STAGE=candidate` while creating the candidate and -`CLUSTERFLUX_RELEASE_STAGE=final` while finalizing it. The final stage requires -`CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST` and complete live evidence. There is no -incomplete-evidence publication override. - -The `clusterflux-release` runner must provide `CLUSTERFLUX_DEPLOY_COMMAND` as a -protected secret. The command runs locally with -`CLUSTERFLUX_CANDIDATE_ARCHIVE`, `CLUSTERFLUX_CANDIDATE_COORDINATOR`, and their -SHA-256 identities exported. It must deploy that executable and restart the -configured service. `scripts/deploy-release-candidate.sh` then independently -compares the running `/proc//exe` digest with the candidate and records -the service and proxy unit identities; a mismatch stops the release. - -Clusterflux release binaries are built once. The public client/node archive and -the private-source hosted-service archive are both digest-bound to the same -candidate. The hosted archive is deployed, the strict production-shaped batch -uses the public archive against it, and finalization copies both archives -without rebuilding. - -Create the candidate in a dedicated directory: - -~~~bash -CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/release-candidate \ -CLUSTERFLUX_RELEASE_STAGE=candidate \ -./scripts/prepare-public-release.js -~~~ - -Deploy `target/release-candidate/assets/clusterflux-public-binaries-*.tar.gz`. -Set `CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST` to the candidate manifest while -running the strict batch. Set `CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH` to the -JSON record from the private and public acceptance commands. The result records -the source commit, source-tree and -public-tree identities, candidate binary digests, deployment generation, and -configuration identity. - -Finalize into a different directory after the strict result passes: - -~~~bash -CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/public-release \ -CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST=target/release-candidate/public-release-manifest.json \ -CLUSTERFLUX_FINAL_RESULT_PATH=target/acceptance/cli-happy-path-live.json \ -./scripts/prepare-public-release.js -~~~ - -Finalization rejects a changed commit, source tree, public tree, candidate -archive, binary digest set, deployment binding, or strict result. It never runs -the release binary build when a candidate manifest is supplied. diff --git a/docs/getting-started.md b/docs/getting-started.md index 0b4f593..be09923 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,6 +11,9 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ +On NixOS or another system with Nix, the equivalent package is available with +`nix profile install .#clusterflux-tools`. + Install rootless Podman on each Linux node that will execute container-backed environments. @@ -52,7 +55,7 @@ same stored identity: ~~~bash clusterflux-node \ - --coordinator https://clusterflux.michelpaulissen.com \ + --coordinator https://clusterflux.lesstuff.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ diff --git a/docs/nodes.md b/docs/nodes.md index f6ffd19..3913859 100644 --- a/docs/nodes.md +++ b/docs/nodes.md @@ -23,7 +23,7 @@ key pair. ~~~bash clusterflux-node \ - --coordinator https://clusterflux.michelpaulissen.com \ + --coordinator https://clusterflux.lesstuff.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ diff --git a/flake.nix b/flake.nix index f99377a..d9ccf26 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,18 @@ forAllSystems = nixpkgs.lib.genAttrs systems; in { + packages = forAllSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + publicPackages = import ./packages.nix { inherit pkgs self; }; + privatePackages = + if builtins.pathExists ./web/packages.nix then + import ./web/packages.nix { inherit pkgs self; } + else + { }; + in + publicPackages // privatePackages); + devShells = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; diff --git a/packages.nix b/packages.nix new file mode 100644 index 0000000..1305b67 --- /dev/null +++ b/packages.nix @@ -0,0 +1,74 @@ +{ pkgs, self }: +let + clusterflux-tools = pkgs.rustPlatform.buildRustPackage { + pname = "clusterflux-tools"; + version = "0.1.0"; + src = self; + cargoLock.lockFile = ./Cargo.lock; + nativeBuildInputs = [ + pkgs.git + pkgs.lld + pkgs.makeWrapper + ]; + cargoBuildFlags = [ + "--package" + "clusterflux-cli" + "--package" + "clusterflux-node" + "--package" + "clusterflux-coordinator" + "--package" + "clusterflux-dap" + ]; + cargoTestFlags = [ + "--package" + "clusterflux-cli" + "--package" + "clusterflux-node" + "--package" + "clusterflux-coordinator" + "--package" + "clusterflux-dap" + ]; + NIX_BUILD_CORES = "2"; + RUST_MIN_STACK = "1073741824"; + postInstall = '' + test -x "$out/bin/clusterflux" + test -x "$out/bin/clusterflux-node" + test -x "$out/bin/clusterflux-coordinator" + test -x "$out/bin/clusterflux-debug-dap" + for command in \ + clusterflux \ + clusterflux-node \ + clusterflux-coordinator \ + clusterflux-debug-dap + do + ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --version >/dev/null + ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --help >/dev/null + done + ''; + postFixup = + let + runtimePath = pkgs.lib.makeBinPath [ + pkgs.cargo + pkgs.git + pkgs.lld + pkgs.rustc + ]; + in + '' + wrapProgram "$out/bin/clusterflux" --prefix PATH : ${runtimePath} + wrapProgram "$out/bin/clusterflux-node" --prefix PATH : ${runtimePath} + wrapProgram "$out/bin/clusterflux-debug-dap" --prefix PATH : ${runtimePath} + ''; + meta = { + description = "Clusterflux CLI, node, coordinator, and debugger adapter"; + mainProgram = "clusterflux"; + }; + }; +in +{ + inherit clusterflux-tools; + clusterflux = clusterflux-tools; + default = clusterflux-tools; +} diff --git a/scripts/acceptance-private.sh b/scripts/acceptance-private.sh deleted file mode 100755 index 5a54a15..0000000 --- a/scripts/acceptance-private.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo" - -if [[ ! -d private/hosted-policy ]]; then - exit 0 -fi - -scripts/check-old-name.sh -node scripts/check-docs.js -scripts/check-code-size.sh -cargo fmt --all --manifest-path private/hosted-policy/Cargo.toml --check -cargo clippy --manifest-path private/hosted-policy/Cargo.toml --all-targets -- -D warnings -cargo test --locked --manifest-path private/hosted-policy/Cargo.toml --all-targets -node private/hosted-policy/scripts/prepare-hosted-deployment.js -if command -v podman >/dev/null 2>&1; then - node private/hosted-policy/scripts/postgres-durable-smoke.js -elif command -v nix >/dev/null 2>&1; then - nix shell nixpkgs#podman --command node private/hosted-policy/scripts/postgres-durable-smoke.js -else - node private/hosted-policy/scripts/postgres-durable-smoke.js -fi -if [[ -n "${CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR:-}" ]]; then - node private/hosted-policy/scripts/hosted-service-live-check.js -fi diff --git a/scripts/acceptance-public.sh b/scripts/acceptance-public.sh deleted file mode 100755 index 96a1363..0000000 --- a/scripts/acceptance-public.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo" - -scripts/check-old-name.sh -node scripts/check-docs.js -scripts/check-code-size.sh -scripts/release-source-scan.sh -cargo fmt --all --check -cargo clippy --workspace --all-targets -- -D warnings -cargo test --workspace --all-targets -cargo build --workspace --all-targets -cargo build -p runtime-conformance --target wasm32-unknown-unknown -cargo build -p hello-build --target wasm32-unknown-unknown -cargo build -p recovery-build --target wasm32-unknown-unknown -node scripts/resource-metering-contract-smoke.js -node scripts/hostile-input-contract-smoke.js -node scripts/tenant-isolation-contract-smoke.js -node scripts/self-hosted-coordinator-smoke.js -node scripts/public-local-demo-matrix-smoke.js -node scripts/cli-output-mode-smoke.js -node scripts/cli-login-smoke.js -node scripts/cli-error-exit-smoke.js -node scripts/cli-browser-login-flow-smoke.js -node scripts/cli-install-smoke.js -node scripts/user-session-token-boundary-smoke.js -node scripts/sdk-spawn-runtime-smoke.js -node scripts/node-lifecycle-contract-smoke.js -node scripts/wasmtime-node-smoke.js -node scripts/wasmtime-assignment-smoke.js -if command -v podman >/dev/null 2>&1; then - node scripts/podman-backend-smoke.js -elif command -v nix >/dev/null 2>&1; then - nix shell nixpkgs#podman --command node scripts/podman-backend-smoke.js -else - node scripts/podman-backend-smoke.js -fi -node scripts/vscode-extension-smoke.js -node scripts/vscode-f5-smoke.js -node scripts/node-attach-smoke.js -node scripts/cli-local-run-smoke.js -node scripts/artifact-download-smoke.js -node scripts/artifact-export-smoke.js -node scripts/operator-panel-smoke.js -node scripts/source-preparation-smoke.js -node scripts/scheduler-placement-smoke.js -node scripts/windows-best-effort-smoke.js -node scripts/quic-smoke.js -node scripts/dap-smoke.js -node scripts/recovery-build-smoke.js -node scripts/flagship-demo-smoke.js -scripts/verify-public-split.sh diff --git a/scripts/agent-signing.js b/scripts/agent-signing.js deleted file mode 100644 index bfdc461..0000000 --- a/scripts/agent-signing.js +++ /dev/null @@ -1,99 +0,0 @@ -const crypto = require("crypto"); - -const { nodeIdentity, signedRequestPayloadDigest } = require("./node-signing"); - -function agentIdentity(seedPrefix, agent) { - const identity = nodeIdentity(seedPrefix, agent); - return { - ...identity, - publicKeyFingerprint: `sha256:${crypto - .createHash("sha256") - .update(identity.publicKey) - .digest("hex")}`, - }; -} - -function agentWorkflowSignatureMessage({ - tenant, - project, - agent, - requestKind, - process: processId, - task = "", - payloadDigest, - nonce, - issuedAtEpochSeconds, -}) { - const parts = [ - "clusterflux-agent-workflow-signature:v2", - tenant, - project, - agent, - requestKind, - processId, - task, - payloadDigest, - nonce, - String(issuedAtEpochSeconds), - ]; - return Buffer.concat( - parts.flatMap((part) => [ - Buffer.from(`${Buffer.byteLength(part)}:`), - Buffer.from(part), - Buffer.from("\n"), - ]) - ); -} - -function signedAgentWorkflowProof(identity, request, options = {}) { - const nonce = - options.nonce ?? - `${request.type}-${process.pid}-${Date.now()}-${crypto - .randomBytes(8) - .toString("hex")}`; - const issuedAtEpochSeconds = - options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000); - const processId = - request.type === "launch_task" ? request.task_spec?.process : request.process; - const task = - request.type === "launch_task" ? request.task_spec?.task_instance : request.task || ""; - const signature = crypto.sign( - null, - agentWorkflowSignatureMessage({ - tenant: request.tenant, - project: request.project, - agent: request.actor_agent, - requestKind: request.type, - process: processId, - task, - payloadDigest: signedRequestPayloadDigest(request), - nonce, - issuedAtEpochSeconds, - }), - identity.privateKeyObject - ); - return { - nonce, - issued_at_epoch_seconds: issuedAtEpochSeconds, - signature: `ed25519:${signature.toString("base64")}`, - }; -} - -function signedAgentWorkflowRequest(identity, request, options = {}) { - const unsignedRequest = { - ...request, - agent_public_key_fingerprint: - options.publicKeyFingerprint || identity.publicKeyFingerprint, - }; - return { - ...unsignedRequest, - agent_signature: signedAgentWorkflowProof(identity, unsignedRequest, options), - }; -} - -module.exports = { - agentIdentity, - agentWorkflowSignatureMessage, - signedAgentWorkflowProof, - signedAgentWorkflowRequest, -}; diff --git a/scripts/artifact-download-smoke.js b/scripts/artifact-download-smoke.js deleted file mode 100755 index f9e3c55..0000000 --- a/scripts/artifact-download-smoke.js +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); -const { - ensureRootlessPodman, - flagshipNodeCapabilities, - launchFlagship, - repo, - runFlagshipWorker, - send, - waitForJsonLine, -} = require("./real-flagship-harness"); - -const downloadNode = "node-download"; -const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode); - -const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); - -async function downloadRetainedBytes(addr, link, artifact, expectedSize) { - const chunks = []; - let offset = 0; - for (let attempt = 0; attempt < 500; attempt += 1) { - const response = await send(addr, { - type: "open_artifact_download_stream", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - token_digest: link.link.scoped_token_digest, - chunk_bytes: 256 * 1024, - }); - assert.strictEqual(response.type, "artifact_download_stream"); - if (!response.content_bytes_available) { - assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending"); - await delay(10); - continue; - } - assert.strictEqual(response.content_source, "retaining_node_reverse_stream"); - assert.strictEqual(response.content_offset, offset); - const bytes = Buffer.from(response.content_base64, "base64"); - assert.strictEqual(response.streamed_bytes, bytes.length); - chunks.push(bytes); - offset += bytes.length; - if (response.content_eof) { - const content = Buffer.concat(chunks); - assert.strictEqual(content.length, expectedSize); - return { content, response }; - } - } - throw new Error("timed out waiting for retained artifact reverse stream"); -} - -function downloadNodeCapabilities() { - return flagshipNodeCapabilities(); -} - -(async () => { - ensureRootlessPodman(); - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback", - ], - { cwd: repo } - ); - - let worker; - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity); - const workerReady = await worker.ready; - assert.strictEqual(workerReady.node_status, "ready"); - assert.strictEqual(workerReady.mode, "worker"); - const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr); - assert.strictEqual(compileEvent.status_code, 0); - assert.strictEqual(packageEvent.status_code, 0); - assert.deepStrictEqual(compileEvent.result, { - Artifact: { - id: compileEvent.artifact_path.slice("/vfs/artifacts/".length), - digest: compileEvent.artifact_digest, - size_bytes: compileEvent.artifact_size_bytes, - }, - }); - assert.deepStrictEqual(packageEvent.result, { - Artifact: { - id: packageEvent.artifact_path.slice("/vfs/artifacts/".length), - digest: packageEvent.artifact_digest, - size_bytes: packageEvent.artifact_size_bytes, - }, - }); - assert.ok(compileEvent.artifact_size_bytes > 0); - assert.strictEqual(packageEvent.artifact_size_bytes, compileEvent.artifact_size_bytes); - assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); - assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); - const artifactPath = packageEvent.artifact_path; - assert.match( - artifactPath, - /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ - ); - const artifact = artifactPath.slice("/vfs/artifacts/".length); - - const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: downloadNode, - capabilities: downloadNodeCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [artifact], - direct_connectivity: false, - online: true, - })); - assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded"); - - const disconnectedLink = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - ttl_seconds: 60, - }); - assert.strictEqual(disconnectedLink.type, "artifact_download_link"); - - const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: downloadNode, - capabilities: downloadNodeCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [artifact], - direct_connectivity: true, - online: true, - })); - assert.strictEqual(connectedReport.type, "node_capabilities_recorded"); - - const link = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - ttl_seconds: 60, - }); - assert.strictEqual(link.type, "artifact_download_link"); - assert.strictEqual(link.link.tenant, "tenant"); - assert.strictEqual(link.link.project, "project"); - assert.strictEqual(link.link.process, virtualProcess); - assert.deepStrictEqual(link.link.actor, { User: "user" }); - assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/); - assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); - assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60); - assert.ok( - link.link.url_path.endsWith( - `/artifacts/tenant/project/${virtualProcess}/${artifact}` - ) - ); - assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" }); - - const crossTenant = await send(addr, { - type: "create_artifact_download_link", - tenant: "other", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - ttl_seconds: 60, - }); - assert.strictEqual(crossTenant.type, "error"); - assert.match(crossTenant.message, /artifact does not exist/); - - const crossProject = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "other-project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - ttl_seconds: 60, - }); - assert.strictEqual(crossProject.type, "error"); - assert.match(crossProject.message, /artifact does not exist/); - - const crossTenantOpen = await send(addr, { - type: "open_artifact_download_stream", - tenant: "other", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - token_digest: link.link.scoped_token_digest, - chunk_bytes: 1, - }); - assert.strictEqual(crossTenantOpen.type, "error"); - assert.match(crossTenantOpen.message, /artifact does not exist/); - - const crossProjectOpen = await send(addr, { - type: "open_artifact_download_stream", - tenant: "tenant", - project: "other-project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - token_digest: link.link.scoped_token_digest, - chunk_bytes: 1, - }); - assert.strictEqual(crossProjectOpen.type, "error"); - assert.match(crossProjectOpen.message, /artifact does not exist/); - - const guessed = await send(addr, { - type: "open_artifact_download_stream", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - token_digest: "sha256:guessed", - chunk_bytes: 1, - }); - assert.strictEqual(guessed.type, "error"); - assert.match(guessed.message, /token is invalid/); - - const crossActorOpen = await send(addr, { - type: "open_artifact_download_stream", - tenant: "tenant", - project: "project", - actor_user: "other-user", - artifact, - max_bytes: 1024 * 1024, - token_digest: link.link.scoped_token_digest, - chunk_bytes: 1, - }); - assert.strictEqual(crossActorOpen.type, "error"); - assert.match(crossActorOpen.message, /token is invalid/); - - const downloaded = await downloadRetainedBytes( - addr, - link, - artifact, - packageEvent.artifact_size_bytes, - ); - const downloadedDigest = `sha256:${crypto - .createHash("sha256") - .update(downloaded.content) - .digest("hex")}`; - assert.strictEqual(downloadedDigest, packageEvent.artifact_digest); - const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-")); - try { - const executable = path.join(inspect, "hello-clusterflux"); - fs.writeFileSync(executable, downloaded.content); - fs.chmodSync(executable, 0o755); - assert.strictEqual( - cp.execFileSync(executable, { encoding: "utf8" }), - "hello from a real Clusterflux build\n" - ); - } finally { - fs.rmSync(inspect, { recursive: true, force: true }); - } - const cliDownloadDirectory = fs.mkdtempSync( - path.join(os.tmpdir(), "clusterflux-cli-download-") - ); - try { - const cliDownloadPath = path.join(cliDownloadDirectory, "hello-clusterflux"); - const cliDownload = JSON.parse( - cp.execFileSync( - "cargo", - [ - "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", - "artifact", "download", artifact, - "--to", cliDownloadPath, - "--max-bytes", "1048576", - "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, - "--tenant", "tenant", - "--project-id", "project", - "--user", "user", - "--json", - ], - { cwd: repo, env: process.env, encoding: "utf8" } - ) - ); - assert.strictEqual(cliDownload.command, "artifact download"); - assert.strictEqual(cliDownload.local_download.status, "local_bytes_written"); - assert.strictEqual( - cliDownload.local_download.verified_digest, - packageEvent.artifact_digest - ); - assert.strictEqual( - `sha256:${crypto - .createHash("sha256") - .update(fs.readFileSync(cliDownloadPath)) - .digest("hex")}`, - packageEvent.artifact_digest - ); - } finally { - fs.rmSync(cliDownloadDirectory, { recursive: true, force: true }); - } - assert.strictEqual(downloaded.response.content_eof, true); - assert.strictEqual( - downloaded.response.charged_download_bytes, - packageEvent.artifact_size_bytes - ); - assert.strictEqual(downloaded.response.link.artifact, artifact); - - const crossActorRevoke = await send(addr, { - type: "revoke_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "other-user", - artifact, - token_digest: link.link.scoped_token_digest, - }); - assert.strictEqual(crossActorRevoke.type, "error"); - assert.match(crossActorRevoke.message, /token is invalid/); - - const revoked = await send(addr, { - type: "revoke_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - token_digest: link.link.scoped_token_digest, - }); - assert.strictEqual(revoked.type, "artifact_download_link_revoked"); - assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest); - - const revokedOpen = await send(addr, { - type: "open_artifact_download_stream", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - token_digest: link.link.scoped_token_digest, - chunk_bytes: 1, - }); - assert.strictEqual(revokedOpen.type, "error"); - assert.match(revokedOpen.message, /revoked/); - - const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: downloadNode, - capabilities: downloadNodeCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: false, - online: true, - })); - assert.strictEqual(gcReport.type, "node_capabilities_recorded"); - - const collectedLink = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - max_bytes: 1024 * 1024, - ttl_seconds: 60, - }); - assert.strictEqual(collectedLink.type, "error"); - assert.match(collectedLink.message, /unavailable from current retention/); - } finally { - worker?.child.kill("SIGTERM"); - coordinator.kill("SIGTERM"); - } - - console.log("Artifact download smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/artifact-export-smoke.js b/scripts/artifact-export-smoke.js deleted file mode 100644 index f33b0ba..0000000 --- a/scripts/artifact-export-smoke.js +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); -const { - ensureRootlessPodman, - flagshipNodeCapabilities, - launchFlagship, - repo, - runFlagshipWorker, - send, - waitForJsonLine, - waitForNodeStatus, -} = require("./real-flagship-harness"); - -const sourceNode = "node-export-source"; -const sourceIdentity = nodeIdentity("artifact-export-smoke", sourceNode); - -function nodeCapabilities() { - return flagshipNodeCapabilities(); -} - -function runJson(command, args, options = {}) { - return new Promise((resolve, reject) => { - const child = cp.spawn(command, args, { cwd: repo, ...options }); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - child.once("error", reject); - child.once("exit", (code) => { - if (code !== 0) { - reject( - new Error( - `${command} ${args.join(" ")} failed with code ${code}\n${stderr}\n${stdout}` - ) - ); - return; - } - try { - resolve(JSON.parse(stdout)); - } catch (error) { - reject(new Error(`${command} did not return JSON\n${stdout}\n${error.message}`)); - } - }); - }); -} - -async function reportNode( - addr, - node, - identity, - { directConnectivity = true, online = true, artifacts = [] } = {} -) { - const response = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node, - capabilities: nodeCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: artifacts, - direct_connectivity: directConnectivity, - online, - })); - assert.strictEqual(response.type, "node_capabilities_recorded"); - assert.strictEqual(response.node, node); -} - -(async () => { - ensureRootlessPodman(); - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback", - ], - { - cwd: repo, - env: { ...process.env, CLUSTERFLUX_NODE_STALE_AFTER_SECONDS: "1" }, - } - ); - - let worker; - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - worker = await runFlagshipWorker(addr, sourceNode, sourceIdentity); - const workerReady = await worker.ready; - assert.strictEqual(workerReady.node_status, "ready"); - assert.strictEqual(workerReady.mode, "worker"); - const firstNodeTaskCompletion = waitForNodeStatus(worker.child, "completed"); - const { compileEvent, process: virtualProcess } = await launchFlagship(addr); - const workerCompletion = await firstNodeTaskCompletion; - assert.strictEqual(workerCompletion.node_status, "completed"); - assert.strictEqual( - workerCompletion.task_assignment_response.task_spec.task_definition, - "snapshot_current_project" - ); - assert.strictEqual( - workerCompletion.virtual_thread, - workerCompletion.task_assignment_response.task_spec.task_instance - ); - assert.strictEqual(compileEvent.status_code, 0); - assert.match( - compileEvent.artifact_path, - /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ - ); - const artifact = compileEvent.artifact_path.slice("/vfs/artifacts/".length); - - await reportNode(addr, sourceNode, sourceIdentity, { - artifacts: [artifact], - }); - - const receiverIdentity = nodeIdentity("artifact-export-smoke", "node-export-receiver"); - const attachedReceiver = await send(addr, { - type: "attach_node", - tenant: "tenant", - project: "project", - node: "node-export-receiver", - public_key: receiverIdentity.publicKey, - }); - assert.strictEqual(attachedReceiver.type, "node_attached"); - await reportNode(addr, "node-export-receiver", receiverIdentity); - - const exportPlan = await send(addr, { - type: "export_artifact_to_node", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - receiver_node: "node-export-receiver", - direct_connectivity: true, - failure_reason: "", - }); - assert.strictEqual(exportPlan.type, "artifact_export_plan"); - assert.strictEqual(exportPlan.source_node, "node-export-source"); - assert.strictEqual(exportPlan.receiver_node, "node-export-receiver"); - assert.strictEqual(exportPlan.plan.transport, "NativeQuic"); - assert.strictEqual(exportPlan.plan.scope.tenant, "tenant"); - assert.strictEqual(exportPlan.plan.scope.project, "project"); - assert.strictEqual(exportPlan.plan.scope.process, virtualProcess); - assert.deepStrictEqual(exportPlan.plan.scope.object, { Artifact: artifact }); - assert.strictEqual(exportPlan.plan.source.node, "node-export-source"); - assert.strictEqual(exportPlan.plan.destination.node, "node-export-receiver"); - assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true); - assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false); - assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/); - assert.strictEqual(exportPlan.artifact_size_bytes, compileEvent.artifact_size_bytes); - - const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-artifact-export-")); - const exportPath = path.join(temp, "hello-clusterflux"); - cp.execFileSync( - "cargo", - ["build", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux"], - { cwd: repo, stdio: "inherit" } - ); - const cliBinary = path.join( - path.resolve(repo, process.env.CARGO_TARGET_DIR || "target"), - "debug", - process.platform === "win32" ? "clusterflux.exe" : "clusterflux" - ); - await reportNode(addr, sourceNode, sourceIdentity, { - artifacts: [artifact], - }); - const cliExport = await runJson(cliBinary, [ - "artifact", - "export", - "--coordinator", - `${addr.host}:${addr.port}`, - "--tenant", - "tenant", - "--project-id", - "project", - "--user", - "user", - "--json", - artifact, - "--receiver-node", - "node-export-receiver", - "--to", - exportPath, - ]); - assert.strictEqual(cliExport.command, "artifact export"); - assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true); - assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written"); - assert.strictEqual( - cliExport.export_plan.bytes_written, - compileEvent.artifact_size_bytes - ); - assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false); - assert.strictEqual( - cliExport.local_export.verified_digest, - compileEvent.artifact_digest - ); - fs.chmodSync(exportPath, 0o755); - assert.strictEqual( - cp.execFileSync(exportPath, { encoding: "utf8" }), - "hello from a real Clusterflux build\n" - ); - - const crossTenant = await send(addr, { - type: "export_artifact_to_node", - tenant: "other", - project: "project", - actor_user: "user", - artifact, - receiver_node: "node-export-receiver", - direct_connectivity: true, - failure_reason: "", - }); - assert.strictEqual(crossTenant.type, "error"); - assert.match(crossTenant.message, /artifact does not exist/); - - await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); - const failedDirect = await send(addr, { - type: "export_artifact_to_node", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - receiver_node: "node-export-receiver", - direct_connectivity: false, - failure_reason: "nat traversal failed", - }); - assert.strictEqual(failedDirect.type, "error"); - assert.match(failedDirect.message, /nat traversal failed/); - assert.match(failedDirect.message, /coordinator bulk relay is disabled/); - - await reportNode(addr, "node-export-receiver", receiverIdentity, { online: false }); - await new Promise((resolve) => setTimeout(resolve, 2100)); - await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); - const offlineReceiver = await send(addr, { - type: "export_artifact_to_node", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact, - receiver_node: "node-export-receiver", - direct_connectivity: true, - failure_reason: "", - }); - assert.strictEqual(offlineReceiver.type, "error"); - assert.match(offlineReceiver.message, /offline/); - } finally { - worker?.child.kill("SIGTERM"); - coordinator.kill("SIGTERM"); - } - - console.log("Artifact export smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/check-code-size.sh b/scripts/check-code-size.sh deleted file mode 100755 index 96e54d3..0000000 --- a/scripts/check-code-size.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo_root" - -maximum_lines=3000 -failed=0 -source_roots=(crates) -if [[ -d private/hosted-policy/src ]]; then - source_roots+=(private/hosted-policy/src) -fi -while IFS= read -r -d '' file; do - case "$file" in - */tests.rs|*/tests/*) continue ;; - esac - lines="$(wc -l < "$file")" - if ((lines > maximum_lines)); then - printf '%s has %s lines; production file limit is %s\n' "$file" "$lines" "$maximum_lines" >&2 - failed=1 - fi -done < <(find "${source_roots[@]}" -type f -name '*.rs' -print0) - -if ((failed)); then - exit 1 -fi -printf 'production source file size guard passed (%s lines maximum)\n' "$maximum_lines" diff --git a/scripts/check-docs.js b/scripts/check-docs.js deleted file mode 100644 index dce6c9e..0000000 --- a/scripts/check-docs.js +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env node -const fs = require("node:fs"); -const path = require("node:path"); - -const root = path.resolve(__dirname, ".."); -const publicDocs = [ - "README.md", - "SECURITY.md", - "docs/getting-started.md", - "docs/architecture.md", - "docs/nodes.md", - "docs/environments.md", - "docs/artifacts.md", - "docs/debugging.md", - "docs/task-abi.md", - "docs/self-hosting.md", - "docs/security.md", -]; -const contributorDocs = ["docs/contributing/releases.md"]; -const privateDocs = [ - "private/docs/hosted-deployment.md", - "private/docs/authentik.md", - "private/docs/community-policy.md", - "private/docs/bandwidth-and-cost-controls.md", - "private/docs/publishing.md", -]; -const internalDocs = ["internal/finish_mvp_2.md"]; -const filteredPublicTree = - process.env.CLUSTERFLUX_FILTERED_PUBLIC_TREE === "1" || - fs.existsSync(path.join(root, "CLUSTERFLUX_PUBLIC_TREE.json")); - -const failures = []; -const expectExactMarkdownSet = (directory, expected) => { - const actual = fs - .readdirSync(path.join(root, directory), { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) - .map((entry) => path.posix.join(directory, entry.name)) - .sort(); - const wanted = [...expected].sort(); - if (JSON.stringify(actual) !== JSON.stringify(wanted)) { - failures.push(`${directory} markdown set is ${actual.join(", ")}; expected ${wanted.join(", ")}`); - } -}; - -const requiredDocs = filteredPublicTree - ? [...publicDocs, ...contributorDocs] - : [...publicDocs, ...contributorDocs, ...privateDocs, ...internalDocs]; -for (const file of requiredDocs) { - if (!fs.existsSync(path.join(root, file))) failures.push(`missing canonical document: ${file}`); -} -expectExactMarkdownSet("docs", publicDocs.filter((file) => file.startsWith("docs/"))); -expectExactMarkdownSet("docs/contributing", contributorDocs); -if (filteredPublicTree) { - for (const directory of ["private", "internal"]) { - if (fs.existsSync(path.join(root, directory))) { - failures.push(`${directory}/ must not exist in the filtered public tree`); - } - } -} - -const forbidden = [ - [/\bmvp\b/i, "internal milestone term"], - [/acceptance criteria/i, "internal gate language"], - [/release verification/i, "internal release language"], - [/public\/private source split/i, "source split narrative"], - [/founder|business decisions/i, "business planning narrative"], - [/hacker news|\bHN\b/, "launch-channel narrative"], - [/\busers can\b/i, "indirect reader wording"], - [/node\s+scripts\//i, "developer script instruction"], - [/scripts\/[^\s)]*smoke/i, "smoke script instruction"], - [/internal\/[^\s)]*/i, "internal tooling reference"], - [/private\/[^\s)]*/i, "private source reference"], -]; -const topLevelCommands = new Set([ - "doctor", "login", "logout", "auth", "agent", "key", "project", "inspect", - "build", "bundle", "run", "node", "process", "task", "logs", "artifact", - "dap", "debug", "quota", "admin", -]); - -for (const file of publicDocs) { - const absolute = path.join(root, file); - const content = fs.readFileSync(absolute, "utf8"); - for (const [pattern, description] of forbidden) { - if (pattern.test(content)) failures.push(`${file}: contains ${description}`); - } - for (const match of content.matchAll(/\]\(([^)#]+)(?:#[^)]+)?\)/g)) { - const target = match[1]; - if (/^(?:https?:|mailto:)/.test(target)) continue; - const resolved = path.resolve(path.dirname(absolute), target); - if (!fs.existsSync(resolved)) failures.push(`${file}: broken link ${target}`); - } - for (const line of content.split(/\r?\n/)) { - const command = line.trim().match(/^clusterflux\s+([a-z][a-z-]*)\b/); - if (command && !topLevelCommands.has(command[1])) { - failures.push(`${file}: unknown top-level CLI command ${command[1]}`); - } - } -} - -const rootMarkdown = fs - .readdirSync(root, { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) - .map((entry) => entry.name) - .sort(); -if (JSON.stringify(rootMarkdown) !== JSON.stringify(["README.md", "SECURITY.md"])) { - failures.push(`top-level markdown set is ${rootMarkdown.join(", ")}; expected README.md, SECURITY.md`); -} - -if (failures.length) { - for (const failure of failures) console.error(failure); - process.exit(1); -} -console.log("documentation checks passed"); diff --git a/scripts/check-old-name.sh b/scripts/check-old-name.sh deleted file mode 100755 index 9a871de..0000000 --- a/scripts/check-old-name.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo_root" - -legacy_lower="$(printf '%s%s' 'disa' 'smer')" -legacy_title="$(printf '%s%s' 'Disa' 'smer')" -legacy_upper="$(printf '%s%s' 'DISA' 'SMER')" -pattern="${legacy_lower}|${legacy_title}|${legacy_upper}" - -allowed_content='^\./scripts/migrate-clusterflux-state\.sh:' -matches="$( - rg -n --hidden \ - --glob '!**/.git/**' \ - --glob '!**/target/**' \ - --glob '!**/node_modules/**' \ - --glob '!**/vendor/**' \ - --glob '!**/.direnv/**' \ - --glob '!**/.cache/**' \ - --glob '!**/dist/**' \ - --glob '!**/out/**' \ - "$pattern" . 2>/dev/null | rg -v "$allowed_content" || true -)" - -paths="$( - find . \ - \( -type d \( -name .git -o -name target -o -name node_modules -o -name vendor -o -name .direnv -o -name .cache -o -name dist -o -name out \) -prune \) -o \ - -iname "*${legacy_lower}*" -print | sort || true -)" - -if [[ -n "$matches" || -n "$paths" ]]; then - [[ -z "$matches" ]] || printf '%s\n' "$matches" >&2 - [[ -z "$paths" ]] || printf '%s\n' "$paths" >&2 - printf 'unexpected legacy product name remains\n' >&2 - exit 1 -fi - -printf 'old-name guard passed\n' diff --git a/scripts/cli-browser-login-flow-smoke.js b/scripts/cli-browser-login-flow-smoke.js deleted file mode 100644 index 0c6341b..0000000 --- a/scripts/cli-browser-login-flow-smoke.js +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const net = require("net"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const tmp = path.join(repo, "target", "acceptance", "tmp", "cli-browser-login-flow"); -const project = path.join(tmp, "project"); -fs.rmSync(tmp, { recursive: true, force: true }); -fs.mkdirSync(project, { recursive: true }); - -function writeOpener() { - const opener = path.join(tmp, "browser-opener.js"); - const trace = path.join(tmp, "browser-opener.log"); - fs.writeFileSync( - opener, - `#!/usr/bin/env node -const fs = require("fs"); -const trace = ${JSON.stringify(trace)}; -const loginUrl = new URL(process.argv[2]); -const state = loginUrl.searchParams.get("state"); -const nonce = loginUrl.searchParams.get("nonce"); -const challenge = loginUrl.searchParams.get("code_challenge"); -if (loginUrl.protocol !== "https:" || !state || !nonce || !challenge) { - fs.appendFileSync(trace, "missing server-owned OIDC parameters\\n"); - process.exit(1); -} -fs.appendFileSync(trace, "server-owned browser transaction\\n"); -// Model a real browser/opener that remains alive after the CLI transaction. -// Its descriptors must not keep the invoking CLI process open. -setTimeout(() => process.exit(0), 5000); -` - ); - fs.chmodSync(opener, 0o755); - return opener; -} - -function startCoordinator() { - return new Promise((resolve, reject) => { - const requests = []; - const server = net.createServer((socket) => { - let buffered = ""; - socket.on("data", (chunk) => { - buffered += chunk.toString("utf8"); - while (buffered.includes("\n")) { - const newline = buffered.indexOf("\n"); - const line = buffered.slice(0, newline); - buffered = buffered.slice(newline + 1); - const envelope = JSON.parse(line); - requests.push(envelope); - assert.strictEqual(envelope.type, "coordinator_request"); - assert.strictEqual(envelope.protocol_version, 1); - assert.strictEqual(envelope.authentication.kind, "none"); - const request = envelope.payload; - - if (requests.length === 1) { - assert.strictEqual(envelope.request_id, "cli-1"); - assert.strictEqual(envelope.operation, "begin_oidc_browser_login"); - assert.deepStrictEqual(request, { - type: "begin_oidc_browser_login", - }); - socket.write( - `${JSON.stringify({ - type: "oidc_browser_login_started", - transaction_id: "login-transaction", - polling_secret: "opaque-polling-secret", - authorization_url: - "https://auth.michelpaulissen.com/application/o/authorize/?state=server-state&nonce=server-nonce&code_challenge=server-pkce&code_challenge_method=S256&redirect_uri=https%3A%2F%2Fclusterflux.michelpaulissen.com%2Fauth%2Fcallback", - expires_at_epoch_seconds: 1800000000, - })}\n` - ); - continue; - } - - assert.strictEqual(envelope.request_id, "cli-2"); - assert.strictEqual(envelope.operation, "poll_oidc_browser_login"); - assert.deepStrictEqual(request, { - type: "poll_oidc_browser_login", - transaction_id: "login-transaction", - polling_secret: "opaque-polling-secret", - }); - socket.end( - `${JSON.stringify({ - type: "oidc_browser_session", - session: { - tenant: "tenant-smoke", - project: "project-smoke", - user: "user-smoke", - cli_session_credential_kind: "CliDeviceSession", - cli_session_secret: "scoped-cli-session-secret", - expires_at_epoch_seconds: 1800000000, - provider_tokens_sent_to_nodes: false, - }, - })}\n` - ); - server.close(); - } - }); - }); - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - resolve({ - url: `${address.address}:${address.port}`, - requests, - close: () => - new Promise((closeResolve) => { - if (!server.listening) closeResolve(); - else server.close(() => closeResolve()); - }), - }); - }); - }); -} - -function runClusterflux(args, env) { - return new Promise((resolve, reject) => { - const child = cp.spawn( - "cargo", - [ - "run", - "-q", - "--manifest-path", - path.join(repo, "Cargo.toml"), - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - ...args, - ], - { cwd: project, env, stdio: ["ignore", "pipe", "pipe"] } - ); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => (stdout += chunk.toString("utf8"))); - child.stderr.on("data", (chunk) => (stderr += chunk.toString("utf8"))); - child.once("error", reject); - child.once("close", (code) => { - if (code === 0) resolve(stdout); - else reject(new Error(`clusterflux exited ${code}\n${stderr}\n${stdout}`)); - }); - }); -} - -(async () => { - const opener = writeOpener(); - const coordinator = await startCoordinator(); - try { - const loginStarted = Date.now(); - const report = JSON.parse( - await runClusterflux( - [ - "login", - "--browser", - "--json", - "--coordinator", - coordinator.url, - "--project-id", - "project-smoke", - ], - { - ...process.env, - CLUSTERFLUX_BROWSER_OPEN_COMMAND: opener, - CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS: "5", - } - ) - ); - assert( - Date.now() - loginStarted < 3000, - "a long-lived browser opener must not keep CLI output pipes or login completion open" - ); - assert.strictEqual(report.plan.coordinator, coordinator.url); - assert.strictEqual(report.boundary.cli_contacted_coordinator, true); - assert.strictEqual(report.boundary.scoped_cli_session_received, true); - assert.strictEqual(report.boundary.local_cli_session_file_written, true); - assert.strictEqual(report.boundary.provider_tokens_persisted_locally, false); - assert.strictEqual(report.boundary.provider_tokens_exposed_to_cli, false); - assert.strictEqual(report.boundary.provider_tokens_sent_to_nodes, false); - assert.strictEqual(report.boundary.coordinator_session_requests, 2); - assert.strictEqual(coordinator.requests.length, 2); - - const sessionFile = path.join(project, ".clusterflux", "session.json"); - const sessionText = fs.readFileSync(sessionFile, "utf8"); - const session = JSON.parse(sessionText); - assert.strictEqual(session.kind, "human"); - assert.strictEqual(session.coordinator, coordinator.url); - assert.strictEqual(session.tenant, "tenant-smoke"); - assert.strictEqual(session.project, "project-smoke"); - assert.strictEqual(session.user, "user-smoke"); - assert.strictEqual(session.cli_session_credential_kind, "CliDeviceSession"); - assert.strictEqual(session.token_expiry_posture, "expires_at"); - assert.strictEqual(session.expires_at, "1800000000"); - assert.strictEqual(session.provider_tokens_exposed_to_cli, false); - assert.strictEqual(session.provider_tokens_sent_to_nodes, false); - assert.doesNotMatch( - sessionText, - /access_token|refresh_token|id_token|provider-secret|authorization_code|Bearer/ - ); - - const authStatus = JSON.parse( - await runClusterflux(["auth", "status", "--json"], process.env) - ); - assert.strictEqual(authStatus.active_coordinator, coordinator.url); - assert.strictEqual(authStatus.principal, "user-smoke"); - assert.strictEqual(authStatus.tenant, "tenant-smoke"); - assert.strictEqual(authStatus.project, "project-smoke"); - assert.strictEqual(authStatus.session.kind, "human"); - assert.strictEqual(authStatus.session.source, "session_file"); - assert.strictEqual(authStatus.session.provider_tokens_exposed_to_cli, false); - assert.strictEqual(authStatus.session.provider_tokens_exposed_to_nodes, false); - } finally { - await coordinator.close(); - } - console.log("CLI browser login flow smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/cli-error-exit-smoke.js b/scripts/cli-error-exit-smoke.js deleted file mode 100755 index 68ad258..0000000 --- a/scripts/cli-error-exit-smoke.js +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const http = require("http"); -const os = require("os"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const project = path.join(repo, "tests/fixtures/runtime-conformance"); -const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-error-")); - -function runClusterflux(args) { - return new Promise((resolve) => { - const child = cp.spawn( - "cargo", - ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], - { - cwd: repo, - stdio: ["ignore", "pipe", "pipe"], - } - ); - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk) => { - stderr += chunk; - }); - child.on("close", (code, signal) => { - resolve({ code, signal, stdout, stderr }); - }); - }); -} - -async function runWithOneCoordinatorResponse(buildArgs, response) { - let request = ""; - const server = http.createServer((incoming, outgoing) => { - incoming.setEncoding("utf8"); - incoming.on("data", (chunk) => { - request += chunk; - }); - incoming.on("end", () => { - outgoing.writeHead(200, { "content-type": "application/json" }); - outgoing.end(JSON.stringify(response)); - server.close(); - }); - }); - - const address = await new Promise((resolve) => { - server.listen(0, "127.0.0.1", () => resolve(server.address())); - }); - const coordinator = `http://${address.address}:${address.port}`; - const result = await runClusterflux(buildArgs(coordinator)); - return { request, result }; -} - -async function main() { - const environmentProject = path.join(tempRoot, "missing-env-project"); - fs.mkdirSync(path.join(environmentProject, "src"), { recursive: true }); - fs.writeFileSync( - path.join(environmentProject, "Cargo.toml"), - "[package]\nname = \"missing-env-project\"\nversion = \"0.1.0\"\nedition = \"2021\"\n" - ); - fs.writeFileSync( - path.join(environmentProject, "src", "main.rs"), - "fn main() { let _target = env!(\"linux\"); }\n" - ); - const environmentFailure = await runClusterflux([ - "build", - "--project", - environmentProject, - "--json", - ]); - assert.strictEqual(environmentFailure.signal, null, environmentFailure.stderr); - assert.strictEqual(environmentFailure.code, 26, environmentFailure.stderr); - const environmentReport = JSON.parse(environmentFailure.stdout); - assert.strictEqual(environmentReport.status, "blocked_before_schedule"); - assert.strictEqual(environmentReport.scheduled_work, false); - assert.strictEqual(environmentReport.machine_error.category, "environment"); - assert.strictEqual( - environmentReport.machine_error.process_exit_code_applied, - true - ); - assert( - environmentReport.machine_error.next_actions.includes("clusterflux inspect") - ); - assert.strictEqual(environmentReport.diagnostics[0].code, "missing_environment"); - - const nonInteractive = await runClusterflux([ - "run", - "build", - "--project", - project, - "--non-interactive", - "--json", - ]); - assert.strictEqual(nonInteractive.signal, null, nonInteractive.stderr); - assert.strictEqual(nonInteractive.code, 20, nonInteractive.stderr); - assert.doesNotMatch(nonInteractive.stderr, /Opening Clusterflux browser login/); - const nonInteractiveReport = JSON.parse(nonInteractive.stdout); - assert.strictEqual(nonInteractiveReport.status, "authentication_required"); - assert.strictEqual(nonInteractiveReport.non_interactive, true); - assert.strictEqual(nonInteractiveReport.browser_opened, false); - assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); - assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); - assert.strictEqual( - nonInteractiveReport.machine_error.process_exit_code_applied, - true - ); - assert( - nonInteractiveReport.machine_error.next_actions.includes( - "pass --local to run against local services" - ) - ); - - let request = ""; - const server = http.createServer((incoming, outgoing) => { - incoming.setEncoding("utf8"); - incoming.on("data", (chunk) => { - request += chunk; - }); - incoming.on("end", () => { - outgoing.writeHead(200, { "content-type": "application/json" }); - outgoing.end( - JSON.stringify({ - type: "error", - message: "quota unavailable: resource limit exceeded for api_calls", - }) - ); - server.close(); - }); - }); - - const address = await new Promise((resolve) => { - server.listen(0, "127.0.0.1", () => resolve(server.address())); - }); - const coordinator = `http://${address.address}:${address.port}`; - - const result = await runClusterflux([ - "run", - "build", - "--project", - project, - "--coordinator", - coordinator, - "--json", - ]); - - assert.strictEqual(result.signal, null, result.stderr); - assert.strictEqual(result.code, 22, result.stderr); - assert.match(request, /"type":"start_process"/); - const report = JSON.parse(result.stdout); - assert.strictEqual(report.status, "coordinator_rejected"); - assert.strictEqual(report.run_start.machine_error.category, "quota"); - assert.strictEqual(report.run_start.machine_error.resource_category, "api_calls"); - assert.strictEqual(report.run_start.machine_error.community_tier_language, true); - assert.strictEqual( - report.run_start.machine_error.community_tier_label, - "community tier" - ); - assert.doesNotMatch(result.stdout, new RegExp(["free", "tier"].join(" "), "i")); - assert.strictEqual( - report.run_start.machine_error.private_abuse_heuristics_exposed, - false - ); - assert.strictEqual(report.run_start.machine_error.stable_exit_code, 22); - assert.strictEqual( - report.run_start.machine_error.process_exit_code_applied, - true - ); - assert( - report.run_start.machine_error.next_actions.includes("clusterflux quota status") - ); - - const capabilityFailure = await runWithOneCoordinatorResponse( - (coordinator) => [ - "run", - "build", - "--project", - project, - "--coordinator", - coordinator, - "--json", - ], - { - type: "error", - message: - "scheduler placement failed: no capable node for placement: missing capability Command", - } - ); - assert.strictEqual(capabilityFailure.result.signal, null, capabilityFailure.result.stderr); - assert.strictEqual(capabilityFailure.result.code, 24, capabilityFailure.result.stderr); - assert.match(capabilityFailure.request, /"type":"start_process"/); - const capabilityReport = JSON.parse(capabilityFailure.result.stdout); - assert.strictEqual( - capabilityReport.run_start.machine_error.category, - "capability" - ); - assert.strictEqual( - capabilityReport.run_start.machine_error.process_exit_code_applied, - true - ); - assert( - capabilityReport.run_start.machine_error.next_actions.includes( - "attach a node with the required capabilities" - ) - ); - - const nodePolicyFailure = await runWithOneCoordinatorResponse( - (coordinator) => [ - "run", - "build", - "--project", - project, - "--coordinator", - coordinator, - "--json", - ], - { - type: "error", - message: "node policy denied native command execution", - } - ); - assert.strictEqual(nodePolicyFailure.result.signal, null, nodePolicyFailure.result.stderr); - assert.strictEqual(nodePolicyFailure.result.code, 23, nodePolicyFailure.result.stderr); - assert.match(nodePolicyFailure.request, /"type":"start_process"/); - const nodePolicyReport = JSON.parse(nodePolicyFailure.result.stdout); - assert.strictEqual( - nodePolicyReport.run_start.machine_error.category, - "policy" - ); - assert.strictEqual( - nodePolicyReport.run_start.machine_error.process_exit_code_applied, - true - ); - assert( - nodePolicyReport.run_start.machine_error.next_actions.includes( - "check coordinator policy for this action" - ) - ); - - const programFailure = await runWithOneCoordinatorResponse( - (coordinator) => [ - "task", - "list", - "--coordinator", - coordinator, - "--json", - ], - { - type: "task_events", - events: [ - { - process: "vp-current", - task: "compile", - terminal_state: "failed", - environment: "linux", - node: "node-linux", - status_code: 1, - stderr_tail: "task exited with status 1", - }, - ], - } - ); - assert.strictEqual(programFailure.result.signal, null, programFailure.result.stderr); - assert.strictEqual(programFailure.result.code, 0, programFailure.result.stderr); - assert.match(programFailure.request, /"type":"list_task_events"/); - const programReport = JSON.parse(programFailure.result.stdout); - assert.strictEqual(programReport.tasks[0].machine_error.category, "program"); - assert.strictEqual(programReport.tasks[0].machine_error.stable_exit_code, 27); - assert( - programReport.tasks[0].machine_error.next_actions.includes("clusterflux logs") - ); - - const artifactDownload = await runWithOneCoordinatorResponse( - (coordinator) => [ - "artifact", - "download", - "app.txt", - "--coordinator", - coordinator, - "--json", - ], - { - type: "artifact_download_denied", - message: "artifact download unauthorized for project", - } - ); - assert.strictEqual(artifactDownload.result.signal, null, artifactDownload.result.stderr); - assert.strictEqual(artifactDownload.result.code, 21, artifactDownload.result.stderr); - assert.match(artifactDownload.request, /"type":"create_artifact_download_link"/); - const artifactDownloadReport = JSON.parse(artifactDownload.result.stdout); - assert.strictEqual( - artifactDownloadReport.download_session.machine_error.category, - "authorization" - ); - assert.strictEqual( - artifactDownloadReport.download_session.machine_error.process_exit_code_applied, - true - ); - - const artifactExport = await runWithOneCoordinatorResponse( - (coordinator) => [ - "artifact", - "export", - "app.txt", - "--to", - path.join(project, "target", "blocked-artifact.txt"), - "--coordinator", - coordinator, - "--json", - ], - { - type: "artifact_export_unavailable", - message: "direct connectivity unavailable for artifact export", - } - ); - assert.strictEqual(artifactExport.result.signal, null, artifactExport.result.stderr); - assert.strictEqual(artifactExport.result.code, 25, artifactExport.result.stderr); - assert.match(artifactExport.request, /"type":"export_artifact_to_node"/); - const artifactExportReport = JSON.parse(artifactExport.result.stdout); - assert.strictEqual( - artifactExportReport.export_plan.machine_error.category, - "connectivity" - ); - assert.strictEqual( - artifactExportReport.export_plan.machine_error.process_exit_code_applied, - true - ); - - const confirmation = await runClusterflux([ - "process", - "cancel", - "--coordinator", - "127.0.0.1:9", - "--json", - ]); - assert.strictEqual(confirmation.signal, null, confirmation.stderr); - assert.strictEqual(confirmation.code, 23, confirmation.stderr); - const confirmationReport = JSON.parse(confirmation.stdout); - assert.strictEqual(confirmationReport.status, "confirmation_required"); - assert.strictEqual(confirmationReport.coordinator_request_sent, false); - assert.strictEqual(confirmationReport.machine_error.category, "policy"); - assert.strictEqual( - confirmationReport.machine_error.process_exit_code_applied, - true - ); - assert( - confirmationReport.next_actions.some((action) => action.includes("--yes")) - ); -} - -main() - .then(() => { - console.log("CLI error exit smoke passed"); - }) - .catch((error) => { - console.error(error); - process.exit(1); - }); diff --git a/scripts/cli-happy-path-live-smoke.js b/scripts/cli-happy-path-live-smoke.js deleted file mode 100644 index 9439f75..0000000 --- a/scripts/cli-happy-path-live-smoke.js +++ /dev/null @@ -1,7128 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const https = require("https"); -const os = require("os"); -const path = require("path"); -const { DapClient } = require("./dap-client"); -const { agentIdentity, signedAgentWorkflowRequest } = require("./agent-signing"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { - nodeIdentity, - nodeIdentityFromPrivateKey, - signedNodeHeartbeat, - signedNodeRequest, -} = require("./node-signing"); -const { configurePodmanTestEnvironment } = require("./podman-test-env"); - -const repo = path.resolve(__dirname, ".."); -configurePodmanTestEnvironment(repo); -if ( - !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && - cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && - cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 -) { - cp.execFileSync( - "nix", - ["shell", "nixpkgs#podman", "--command", "node", __filename], - { - cwd: repo, - env: { ...process.env, CLUSTERFLUX_PODMAN_NIX_SHELL: "1" }, - stdio: "inherit", - } - ); - process.exit(0); -} - -const releaseRoot = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/public-release") -); -const acceptanceRoot = path.join(repo, "target/acceptance"); -const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); -const reportPath = path.join(acceptanceRoot, "cli-happy-path-live.json"); -const serviceEndpoint = "https://clusterflux.michelpaulissen.com"; -const serviceAddr = - process.env.CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR || - "clusterflux.michelpaulissen.com:443"; -const browserOpenCommand = - process.env.CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND; -const reuseSessionFile = - process.env.CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE; -const qualityGateEvidencePath = - process.env.CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH; -const enabled = process.env.CLUSTERFLUX_CLI_HAPPY_PATH_LIVE === "1"; -const strictFullRelease = process.env.CLUSTERFLUX_STRICT_FULL_RELEASE === "1"; -const strictVpsRestart = process.env.CLUSTERFLUX_STRICT_VPS_RESTART === "1"; -const strictVpsHost = process.env.CLUSTERFLUX_STRICT_VPS_HOST; -const strictVpsIdentity = process.env.CLUSTERFLUX_STRICT_VPS_IDENTITY; -const strictServiceUnit = - process.env.CLUSTERFLUX_STRICT_SERVICE_UNIT || - "clusterflux-hosted.service"; -const strictSoakSeconds = Number( - process.env.CLUSTERFLUX_STRICT_SOAK_SECONDS || "300" -); -const commands = []; - -function requireEnabled() { - if (!enabled) { - throw new Error( - "CLUSTERFLUX_CLI_HAPPY_PATH_LIVE=1 is required because this runs the live CLI happy path" - ); - } - if (!browserOpenCommand && !reuseSessionFile) { - throw new Error( - "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND or CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE is required" - ); - } - const hasSecondTenantSession = Boolean( - process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE - ); - if ( - strictFullRelease && - !hasSecondTenantSession - ) { - throw new Error( - "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE so the compiled two-tenant Node and artifact collision can run" - ); - } - if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) { - throw new Error( - "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE" - ); - } - if (strictFullRelease && !qualityGateEvidencePath) { - throw new Error( - "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH" - ); - } - if (strictFullRelease && !strictVpsRestart) { - throw new Error( - "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_STRICT_VPS_RESTART=1" - ); - } - if (strictVpsRestart && (!strictVpsHost || !strictVpsIdentity)) { - throw new Error( - "strict VPS restart requires CLUSTERFLUX_STRICT_VPS_HOST and CLUSTERFLUX_STRICT_VPS_IDENTITY" - ); - } - if ( - !Number.isInteger(strictSoakSeconds) || - strictSoakSeconds < 120 || - strictSoakSeconds > 1800 - ) { - throw new Error("CLUSTERFLUX_STRICT_SOAK_SECONDS must be an integer from 120 to 1800"); - } -} - -function readJson(file) { - return JSON.parse(fs.readFileSync(file, "utf8")); -} - -function strictQualityGateEvidence(manifest) { - if (!strictFullRelease) return null; - const evidence = readJson(path.resolve(qualityGateEvidencePath)); - assert.strictEqual(evidence.source_commit, manifest.source_commit); - assert.strictEqual(evidence.source_tree_digest, manifest.source_tree_digest); - for (const gate of [evidence.private, evidence.public]) { - assert.strictEqual(gate.status, "passed"); - assert(Number.isFinite(gate.duration_ms) && gate.duration_ms > 0); - } - assert.strictEqual(evidence.public.independent_filtered_tree, true); - for (const id of [ - "formatting", - "clippy_warnings_denied", - "public_workspace_tests", - "private_hosted_policy_locked_tests", - "process_lifecycle_regressions", - "wasm_example_builds", - "filtered_public_tree_build_and_tests", - "vscode_extension_candidate", - ]) { - const check = evidence.checks?.[id]; - assert(check, `quality evidence omitted ${id}`); - assert.strictEqual(check.status, "passed"); - assert(Number.isFinite(check.duration_ms) && check.duration_ms > 0); - } - assert.strictEqual( - evidence.checks.vscode_extension_candidate.candidate_vsix.sha256, - manifest.release_candidate.extension_sha256 - ); - return evidence; -} - -function ensureDir(dir) { - fs.mkdirSync(dir, { recursive: true }); -} - -function sha256(bytes) { - return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`; -} - -function nonInteractiveGitEnv(extra = {}) { - return { - ...process.env, - GIT_TERMINAL_PROMPT: "0", - GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", - SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", - GIT_SSH_COMMAND: - process.env.GIT_SSH_COMMAND || - "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", - ...extra, - }; -} - -function commandEnv(command, env) { - if (command !== "git") return env; - return nonInteractiveGitEnv(env); -} - -function recordCommand(command, args) { - const redacted = []; - let hideNext = false; - for (const argument of args) { - if (hideNext) { - redacted.push(""); - hideNext = false; - } else { - redacted.push(argument); - hideNext = ["--enrollment-grant", "--admin-token"].includes(argument); - } - } - commands.push([command, ...redacted].join(" ")); -} - -function run(command, args, options = {}) { - recordCommand(command, args); - return cp.execFileSync(command, args, { - cwd: repo, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 15 * 60 * 1000, - ...options, - env: commandEnv(command, options.env), - }); -} - -function parseJsonOutput(output) { - const trimmed = output.trim(); - if (!trimmed) throw new Error("expected JSON output, got empty stdout"); - try { - return JSON.parse(trimmed); - } catch (_) { - // Commands may print progress before their final JSON report. - } - const lines = trimmed.split(/\r?\n/).filter(Boolean); - for (let index = lines.length - 1; index >= 0; index -= 1) { - try { - return JSON.parse(lines.slice(index).join("\n")); - } catch (_) { - // Keep scanning for the trailing JSON value. - } - } - throw new Error(`could not parse JSON output:\n${trimmed}`); -} - -function runJson(command, args, options = {}) { - return parseJsonOutput(run(command, args, options)); -} - -function authenticatedRequest(sessionSecret, request) { - return { - type: "authenticated", - session_secret: sessionSecret, - request, - }; -} - -function sendHostedControl(payload) { - return sendHostedControlEnvelope( - coordinatorWireRequest(payload, "strict-live") - ); -} - -function sendHostedControlEnvelope(envelope) { - const body = Buffer.from( - JSON.stringify(envelope) - ); - const url = new URL("/api/v1/control", serviceEndpoint); - return new Promise((resolve, reject) => { - const request = https.request( - url, - { - method: "POST", - headers: { - "content-type": "application/json", - "content-length": body.length, - }, - timeout: 30_000, - }, - (response) => { - const chunks = []; - response.on("data", (chunk) => chunks.push(chunk)); - response.on("end", () => { - const responseBody = Buffer.concat(chunks).toString("utf8"); - if (response.statusCode < 200 || response.statusCode >= 300) { - reject( - new Error( - `hosted control HTTP ${response.statusCode}: ${responseBody}` - ) - ); - return; - } - try { - resolve(JSON.parse(responseBody)); - } catch (error) { - reject( - new Error(`hosted control returned invalid JSON: ${error.message}`) - ); - } - }); - } - ); - request.on("timeout", () => - request.destroy(new Error("hosted control request timed out")) - ); - request.on("error", reject); - request.end(body); - }); -} - -async function runMalformedIdentifierSuite({ - clusterflux, - projectDir, - scope, - sessionSecret, - tenant, - project, - user, - suffix, - securityNode, - bundle, - releaseArtifact, -}) { - const scenarioStartedAt = Date.now(); - const forms = [ - { name: "empty", value: "" }, - { name: "whitespace", value: " \t" }, - { name: "control", value: "hostile\u0000id" }, - { name: "invalid_format", value: "hostile id!" }, - { name: "oversized", value: "x".repeat(256) }, - ]; - const tokenForms = forms - .filter((form) => form.name !== "invalid_format") - .map((form) => - form.name === "oversized" - ? { ...form, value: "x".repeat(257) } - : form - ); - assert( - Buffer.byteLength( - tokenForms.find((form) => form.name === "oversized").value - ) > 256, - "hostile token suite must exceed the signed-nonce and hosted-login byte limit" - ); - const agent = `hostile-id-agent-${suffix}`; - const agentIdentityRecord = agentIdentity("strict-hostile-id-agent", agent); - const processId = `vp-hostile-identifiers-${suffix}`; - const launchAttempt = `hostile-launch-attempt-${suffix}`; - const mainTask = `ti:${processId}:main`; - const taskSpec = { - tenant, - project, - process: processId, - task_definition: bundle.entryStableId, - task_instance: mainTask, - dispatch: { - kind: "coordinator_node_wasm", - export: bundle.entryExport, - abi: "entrypoint_v1", - }, - environment_id: null, - environment: null, - environment_digest: null, - required_capabilities: [], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [], - vfs_epoch: 1, - failure_policy: "fail_fast", - bundle_digest: bundle.digest, - }; - const rejected = []; - const elapsedMs = (started) => - Number(process.hrtime.bigint() - started) / 1_000_000; - const rejectMalformed = async ({ - principal, - requestVariant, - fieldPath, - form, - send, - token = false, - }) => { - const started = process.hrtime.bigint(); - const response = await send(form.value); - assert.strictEqual( - response.type, - "error", - `${principal} ${fieldPath} ${form.name} reached the real ${requestVariant} variant but was not rejected: ${JSON.stringify(response)}` - ); - assert.match( - response.message, - token - ? /malformed external token|token.*invalid|control characters|byte limit|empty|whitespace/i - : /malformed external identifier|(?:Agent|Artifact|Node|Process|Project|TaskDefinition|TaskInstance|Tenant|User|LaunchAttempt)Id is invalid/i, - `${principal} ${fieldPath} ${form.name} was not rejected for identifier validation: ${JSON.stringify(response)}` - ); - assert.doesNotMatch( - response.message, - /unknown variant|unknown field|missing field/i, - `${principal} ${fieldPath} ${form.name} only proved generic deserialization failure` - ); - const health = await sendHostedControl({ type: "ping" }); - assert.strictEqual( - health.type, - "pong", - `valid traffic failed after hostile ${principal} ${fieldPath} ${form.name}` - ); - rejected.push({ - principal, - request_variant: requestVariant, - field_path: fieldPath, - fault: form.name, - expected: token - ? "structured malformed external token error" - : "structured malformed external identifier error", - observed: response.message, - health_after: health.type, - duration_ms: elapsedMs(started), - }); - }; - - let agentRegistered = false; - let processStarted = false; - try { - const added = runJson( - clusterflux, - [ - "key", - "add", - ...scope, - "--agent", - agent, - "--public-key", - agentIdentityRecord.publicKey, - ], - { cwd: projectDir } - ); - assert.strictEqual(added.command, "key add"); - agentRegistered = true; - - const agentStartCases = [ - ["tenant", "tenant"], - ["project", "project"], - ["agent", "actor_agent"], - ["process", "process"], - ["launch_attempt", "launch_attempt"], - ]; - for (const [principal, field] of agentStartCases) { - for (const form of forms) { - await rejectMalformed({ - principal: `agent_signed_${principal}`, - requestVariant: "start_process", - fieldPath: field, - form, - send: async (value) => { - const body = { - type: "start_process", - tenant, - project, - actor_agent: agent, - process: processId, - launch_attempt: launchAttempt, - restart: false, - }; - body[field] = value; - return sendHostedControl( - signedAgentWorkflowRequest(agentIdentityRecord, body, { - nonce: `hostile-agent-${principal}-${form.name}-${Date.now()}`, - }) - ); - }, - }); - } - } - for (const form of tokenForms) { - await rejectMalformed({ - principal: "agent_signed_nonce", - requestVariant: "start_process", - fieldPath: "agent_signature.nonce", - form, - token: true, - send: async (value) => { - const request = signedAgentWorkflowRequest( - agentIdentityRecord, - { - type: "start_process", - tenant, - project, - actor_agent: agent, - process: processId, - launch_attempt: launchAttempt, - restart: false, - }, - { nonce: value } - ); - assert.strictEqual( - request.agent_signature.nonce, - value, - "hostile Agent nonce was not preserved on the wire" - ); - return sendHostedControl(request); - }, - }); - } - for (const form of forms) { - await rejectMalformed({ - principal: "node_signed_heartbeat_node", - requestVariant: "node_heartbeat", - fieldPath: "node", - form, - send: async (value) => - sendHostedControl({ - type: "node_heartbeat", - tenant, - project, - node: value, - node_signature: signedNodeHeartbeat( - tenant, - project, - value, - securityNode.identity, - { - nonce: `hostile-heartbeat-node-${form.name}-${Date.now()}`, - } - ), - }), - }); - } - - const started = await sendHostedControl( - signedAgentWorkflowRequest( - agentIdentityRecord, - { - type: "start_process", - tenant, - project, - actor_agent: agent, - process: processId, - launch_attempt: launchAttempt, - restart: false, - }, - { nonce: `hostile-agent-valid-start-${suffix}` } - ) - ); - assert.strictEqual(started.type, "process_started", JSON.stringify(started)); - processStarted = true; - taskSpec.vfs_epoch = started.epoch; - - const taskSpecCases = [ - ["tenant", "task_spec.tenant"], - ["project", "task_spec.project"], - ["process", "task_spec.process"], - ["task_definition", "task_spec.task_definition"], - ["task_instance", "task_spec.task_instance"], - ]; - for (const [field, fieldPath] of taskSpecCases) { - for (const form of forms) { - await rejectMalformed({ - principal: `agent_signed_${field}`, - requestVariant: "launch_task", - fieldPath, - form, - send: async (value) => { - const malformedTaskSpec = structuredClone(taskSpec); - malformedTaskSpec[field] = value; - const body = { - type: "launch_task", - tenant, - project, - actor_agent: agent, - task_spec: malformedTaskSpec, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, - wasm_module_base64: bundle.moduleBase64, - }; - return sendHostedControl( - signedAgentWorkflowRequest(agentIdentityRecord, body, { - nonce: `hostile-task-spec-${field}-${form.name}-${Date.now()}`, - }) - ); - }, - }); - } - } - - for (const form of forms) { - await rejectMalformed({ - principal: "agent_signed_artifact_array", - requestVariant: "launch_task", - fieldPath: "task_spec.required_artifacts[0]", - form, - send: async (value) => { - const malformedTaskSpec = structuredClone(taskSpec); - malformedTaskSpec.required_artifacts = [releaseArtifact.artifact]; - malformedTaskSpec.args = [ - { - Artifact: { - id: releaseArtifact.artifact, - digest: releaseArtifact.digest, - size_bytes: Number(releaseArtifact.size_bytes || 1), - }, - }, - ]; - malformedTaskSpec.required_artifacts = [value]; - const body = { - type: "launch_task", - tenant, - project, - actor_agent: agent, - task_spec: malformedTaskSpec, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${mainTask}-artifact-output.txt`, - wasm_module_base64: bundle.moduleBase64, - }; - return sendHostedControl( - signedAgentWorkflowRequest(agentIdentityRecord, body, { - nonce: `hostile-artifact-array-${form.name}-${Date.now()}`, - }) - ); - }, - }); - } - - const mainLaunch = await sendHostedControl( - signedAgentWorkflowRequest( - agentIdentityRecord, - { - type: "launch_task", - tenant, - project, - actor_agent: agent, - task_spec: taskSpec, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, - wasm_module_base64: bundle.moduleBase64, - }, - { nonce: `hostile-agent-valid-main-${suffix}` } - ) - ); - assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch)); - - const authenticatedCases = [ - { - principal: "authenticated_cli_project", - requestVariant: "select_project", - fieldPath: "request.project", - payload: (value) => ({ type: "select_project", project: value }), - }, - { - principal: "authenticated_cli_process", - requestVariant: "list_task_events", - fieldPath: "request.process", - payload: (value) => ({ type: "list_task_events", process: value }), - }, - { - principal: "authenticated_cli_task", - requestVariant: "join_task", - fieldPath: "request.task", - payload: (value) => ({ - type: "join_task", - process: processId, - task: value, - }), - }, - { - principal: "authenticated_cli_artifact", - requestVariant: "create_artifact_download_link", - fieldPath: "request.artifact", - payload: (value) => ({ - type: "create_artifact_download_link", - artifact: value, - max_bytes: Number(releaseArtifact.size_bytes || 1), - ttl_seconds: 60, - }), - }, - { - principal: "authenticated_cli_launch_attempt", - requestVariant: "abort_process", - fieldPath: "request.launch_attempt", - payload: (value) => ({ - type: "abort_process", - process: processId, - launch_attempt: value, - }), - }, - ]; - for (const testCase of authenticatedCases) { - for (const form of forms) { - await rejectMalformed({ - ...testCase, - form, - send: async (value) => - sendHostedControl( - authenticatedRequest(sessionSecret, testCase.payload(value)) - ), - }); - } - } - - const nodeCases = [ - { - principal: "node_signed_wrapper_node", - requestVariant: "poll_task_assignment", - fieldPath: "node", - send: (value, form) => - sendHostedControl( - signedNodeRequest( - value, - securityNode.identity, - "poll_task_assignment", - { - type: "poll_task_assignment", - tenant, - project, - node: securityNode.node, - }, - { nonce: `hostile-node-wrapper-${form.name}-${Date.now()}` } - ) - ), - }, - { - principal: "node_signed_poll_tenant", - requestVariant: "poll_task_assignment", - fieldPath: "request.tenant", - send: (value, form) => - sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "poll_task_assignment", - { - type: "poll_task_assignment", - tenant: value, - project, - node: securityNode.node, - }, - { nonce: `hostile-node-tenant-${form.name}-${Date.now()}` } - ) - ), - }, - { - principal: "node_signed_poll_project", - requestVariant: "poll_task_assignment", - fieldPath: "request.project", - send: (value, form) => - sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "poll_task_assignment", - { - type: "poll_task_assignment", - tenant, - project: value, - node: securityNode.node, - }, - { nonce: `hostile-node-project-${form.name}-${Date.now()}` } - ) - ), - }, - { - principal: "node_signed_poll_node", - requestVariant: "poll_task_assignment", - fieldPath: "request.node", - send: (value, form) => - sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "poll_task_assignment", - { - type: "poll_task_assignment", - tenant, - project, - node: value, - }, - { nonce: `hostile-node-inner-${form.name}-${Date.now()}` } - ) - ), - }, - { - principal: "node_signed_artifact_array", - requestVariant: "report_node_capabilities", - fieldPath: "request.artifact_locations[0]", - send: (value, form) => - sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "report_node_capabilities", - { - ...securityNode.capability_body, - artifact_locations: [value], - }, - { nonce: `hostile-node-artifact-${form.name}-${Date.now()}` } - ) - ), - }, - ]; - for (const testCase of nodeCases) { - for (const form of forms) { - await rejectMalformed({ - principal: testCase.principal, - requestVariant: testCase.requestVariant, - fieldPath: testCase.fieldPath, - form, - send: (value) => testCase.send(value, form), - }); - } - } - for (const form of tokenForms) { - await rejectMalformed({ - principal: "node_signed_nonce", - requestVariant: "node_heartbeat", - fieldPath: "node_signature.nonce", - form, - token: true, - send: async (value) => { - const request = { - type: "node_heartbeat", - tenant, - project, - node: securityNode.node, - node_signature: signedNodeHeartbeat( - tenant, - project, - securityNode.node, - securityNode.identity, - { nonce: value } - ), - }; - assert.strictEqual( - request.node_signature.nonce, - value, - "hostile Node nonce was not preserved on the wire" - ); - return sendHostedControl(request); - }, - }); - } - - const endpointFingerprint = sha256( - Buffer.from(securityNode.identity.publicKey) - ); - const rendezvous = { - type: "request_rendezvous", - scope: { - tenant, - project, - process: processId, - object: { Artifact: releaseArtifact.artifact }, - authorization_subject: `${securityNode.node}-self-transfer`, - }, - source: { - node: securityNode.node, - advertised_addr: "127.0.0.1:4433", - public_key_fingerprint: endpointFingerprint, - }, - destination: { - node: securityNode.node, - advertised_addr: "127.0.0.1:4433", - public_key_fingerprint: endpointFingerprint, - }, - direct_connectivity: true, - failure_reason: "", - }; - const rendezvousCases = [ - ["scope.tenant", (request, value) => (request.scope.tenant = value)], - ["scope.project", (request, value) => (request.scope.project = value)], - ["scope.process", (request, value) => (request.scope.process = value)], - [ - "scope.object.artifact", - (request, value) => (request.scope.object.Artifact = value), - ], - ["source.node", (request, value) => (request.source.node = value)], - [ - "destination.node", - (request, value) => (request.destination.node = value), - ], - ]; - for (const [fieldPath, mutate] of rendezvousCases) { - for (const form of forms) { - await rejectMalformed({ - principal: "node_signed_rendezvous", - requestVariant: "request_rendezvous", - fieldPath: `request.${fieldPath}`, - form, - send: async (value) => { - const body = structuredClone(rendezvous); - mutate(body, value); - return sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "request_rendezvous", - body, - { - nonce: `hostile-rendezvous-${fieldPath}-${form.name}-${Date.now()}`, - } - ) - ); - }, - }); - } - } - const validRendezvous = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "request_rendezvous", - rendezvous, - { nonce: `hostile-rendezvous-valid-${suffix}` } - ) - ); - assert.strictEqual( - validRendezvous.type, - "rendezvous_plan", - JSON.stringify(validRendezvous) - ); - - const login = await sendHostedLogin({ type: "begin_oidc_browser_login" }); - assert.strictEqual(login.type, "oidc_browser_login_started", JSON.stringify(login)); - const loginCases = [ - ["transaction_id", login.transaction_id, login.polling_secret], - ["polling_secret", login.transaction_id, login.polling_secret], - ]; - for (const [field, validTransaction, validSecret] of loginCases) { - for (const form of tokenForms) { - await rejectMalformed({ - principal: "private_hosted_login", - requestVariant: "poll_oidc_browser_login", - fieldPath: field, - form, - token: true, - send: async (value) => - sendHostedLogin({ - type: "poll_oidc_browser_login", - transaction_id: - field === "transaction_id" ? value : validTransaction, - polling_secret: field === "polling_secret" ? value : validSecret, - }), - }); - const validPoll = await sendHostedLogin({ - type: "poll_oidc_browser_login", - transaction_id: login.transaction_id, - polling_secret: login.polling_secret, - }); - assert.strictEqual( - validPoll.type, - "oidc_browser_login_pending", - `valid hosted login polling failed after malformed ${field}` - ); - } - } - - const operatorControlCases = [ - ["payload.tenant", "tenant"], - ["payload.project", "project"], - ["payload.process", "process"], - ]; - for (const [fieldPath, field] of operatorControlCases) { - for (const form of forms) { - await rejectMalformed({ - principal: "private_hosted_operator_control", - requestVariant: "stop_hosted_process", - fieldPath, - form, - send: async (value) => { - const payload = { - type: "stop_hosted_process", - tenant, - project, - process: processId, - }; - payload[field] = value; - return sendHostedControlEnvelope({ - type: "hosted_operator_request", - protocol_version: 1, - request_id: `hostile-operator-${field}-${form.name}-${Date.now()}`, - operation: "stop_hosted_process", - operator_proof: sha256(Buffer.from("invalid operator proof")), - operator_nonce: `hostile-operator-nonce-${field}-${form.name}-${Date.now()}`, - issued_at_epoch_seconds: Math.floor(Date.now() / 1000), - payload, - }); - }, - }); - } - } - - const authenticatedHealth = await sendHostedControl( - authenticatedRequest(sessionSecret, { type: "auth_status" }) - ); - assert.strictEqual(authenticatedHealth.type, "auth_status"); - assert.strictEqual(authenticatedHealth.tenant, tenant); - assert.strictEqual(authenticatedHealth.project, project); - assert.strictEqual(authenticatedHealth.actor, user); - } finally { - if (processStarted) { - try { - runJson( - clusterflux, - [ - "process", - "abort", - ...scope, - "--process", - processId, - "--yes", - ], - { cwd: projectDir } - ); - } catch (_) { - // Preserve the primary validation failure while still attempting cleanup. - } - } - if (agentRegistered) { - try { - runJson( - clusterflux, - ["key", "revoke", ...scope, "--agent", agent, "--yes"], - { cwd: projectDir } - ); - } catch (_) { - // Preserve the primary validation failure while still attempting cleanup. - } - } - } - - const coveredPrincipals = [...new Set(rejected.map((entry) => entry.principal))]; - const coveredVariants = [...new Set(rejected.map((entry) => entry.request_variant))]; - return { - principals: coveredPrincipals, - request_variants: coveredVariants, - forms: forms.map((form) => form.name), - rejected_requests: rejected, - all_rejected_for_intended_reason: rejected.every( - (entry) => - /identifier|token|Id is invalid|control characters|byte limit|empty|whitespace/i.test( - entry.observed - ) - ), - valid_after_every_rejection: rejected.every( - (entry) => entry.health_after === "pong" - ), - valid_authenticated_action_after_suite: true, - valid_signed_rendezvous_after_suite: true, - valid_private_login_poll_after_every_private_rejection: true, - duration_ms: Date.now() - scenarioStartedAt, - }; -} - -function sendHostedControlDroppingResponse(payload) { - const body = Buffer.from( - JSON.stringify(coordinatorWireRequest(payload, "strict-live-dropped-response")) - ); - const url = new URL("/api/v1/control", serviceEndpoint); - return new Promise((resolve, reject) => { - let settled = false; - const request = https.request( - url, - { - method: "POST", - headers: { - "content-type": "application/json", - "content-length": body.length, - }, - timeout: 30_000, - }, - (response) => { - settled = true; - const statusCode = response.statusCode; - response.destroy(); - resolve({ status_code: statusCode, response_body_consumed: false }); - } - ); - request.on("timeout", () => - request.destroy(new Error("hosted dropped-response request timed out")) - ); - request.on("error", (error) => { - if (!settled) reject(error); - }); - request.end(body); - }); -} - -function sendHostedLoginStatus(extraHeaders = {}) { - const body = Buffer.from( - JSON.stringify( - coordinatorWireRequest( - { type: "begin_oidc_browser_login" }, - `strict-live-login-${Date.now()}-${Math.random()}` - ) - ) - ); - const url = new URL("/api/v1/login", serviceEndpoint); - return new Promise((resolve, reject) => { - const request = https.request( - url, - { - method: "POST", - headers: { - "content-type": "application/json", - "content-length": body.length, - ...extraHeaders, - }, - timeout: 30_000, - }, - (response) => { - const chunks = []; - response.on("data", (chunk) => chunks.push(chunk)); - response.on("end", () => - resolve({ - status_code: response.statusCode, - body: Buffer.concat(chunks).toString("utf8"), - }) - ); - } - ); - request.on("timeout", () => - request.destroy(new Error("hosted login request timed out")) - ); - request.on("error", reject); - request.end(body); - }); -} - -function sendHostedLogin(payload) { - const body = Buffer.from( - JSON.stringify( - coordinatorWireRequest( - payload, - `strict-live-login-protocol-${Date.now()}-${Math.random()}` - ) - ) - ); - const url = new URL("/api/v1/login", serviceEndpoint); - return new Promise((resolve, reject) => { - const request = https.request( - url, - { - method: "POST", - headers: { - "content-type": "application/json", - "content-length": body.length, - }, - timeout: 30_000, - }, - (response) => { - const chunks = []; - response.on("data", (chunk) => chunks.push(chunk)); - response.on("end", () => { - const responseBody = Buffer.concat(chunks).toString("utf8"); - if (response.statusCode < 200 || response.statusCode >= 300) { - reject( - new Error( - `hosted login HTTP ${response.statusCode}: ${responseBody}` - ) - ); - return; - } - try { - resolve(JSON.parse(responseBody)); - } catch (error) { - reject( - new Error(`hosted login returned invalid JSON: ${error.message}`) - ); - } - }); - } - ); - request.on("timeout", () => - request.destroy(new Error("hosted login request timed out")) - ); - request.on("error", reject); - request.end(body); - }); -} - -async function runHostedLoginIsolationBeforeRestart() { - const scenarioStartedAt = Date.now(); - const controlBefore = await sendHostedControl({ type: "ping" }); - assert.strictEqual(controlBefore.type, "pong"); - const statuses = []; - let limited; - for (let index = 0; index < 128; index += 1) { - const response = await sendHostedLoginStatus(); - statuses.push(response.status_code); - if (response.status_code === 429) { - limited = response; - break; - } - assert.strictEqual(response.status_code, 200, response.body); - } - assert(limited, `hosted login route was not rate limited: ${statuses.join(",")}`); - const spoofed = await sendHostedLoginStatus({ - forwarded: "for=203.0.113.99", - "x-forwarded-for": "203.0.113.99", - "x-real-ip": "203.0.113.99", - }); - assert.strictEqual( - spoofed.status_code, - 429, - "caller-supplied forwarding headers bypassed the client rate limit" - ); - - const remoteBody = JSON.stringify( - coordinatorWireRequest( - { type: "begin_oidc_browser_login" }, - `strict-vps-login-${Date.now()}` - ) - ); - const remoteStatus = Number( - run( - "ssh", - sshArgs( - "curl", - "--silent", - "--show-error", - "--output", - "/dev/null", - "--write-out=%{http_code}", - "-HContent-Type:application/json", - `--data-binary=${remoteBody}`, - `${serviceEndpoint}/api/v1/login` - ) - ).trim() - ); - assert.strictEqual( - remoteStatus, - 200, - "one client exhausted the browser-login allowance for a different client" - ); - const controlAfter = await sendHostedControl({ type: "ping" }); - assert.strictEqual(controlAfter.type, "pong"); - return { - scenario_started_at_ms: scenarioStartedAt, - local_statuses: statuses, - local_limited_status: limited.status_code, - forwarding_spoof_status: spoofed.status_code, - independent_vps_client_status: remoteStatus, - control_route_before: controlBefore.type, - control_route_after: controlAfter.type, - }; -} - -async function finishHostedLoginIsolationAfterRestart(evidence) { - const control = await sendHostedControl({ type: "ping" }); - assert.strictEqual(control.type, "pong"); - const response = await sendHostedLoginStatus(); - assert( - [200, 429].includes(response.status_code), - `login route did not recover after service restart: ${response.status_code} ${response.body}` - ); - evidence.after_service_restart = { - control_route: control.type, - login_route_status: response.status_code, - }; - evidence.duration_ms = Date.now() - evidence.scenario_started_at_ms; - delete evidence.scenario_started_at_ms; - return evidence; -} - -function assertDenied(response, pattern, label) { - assert.strictEqual(response.type, "error", `${label}: ${JSON.stringify(response)}`); - assert.match(response.message, pattern, `${label}: ${JSON.stringify(response)}`); - return { - denied: true, - category: response.error?.category || null, - message: response.message, - }; -} - -function assertDeniedOrEmptyCollection(response, pattern, label) { - const emptyCollectionField = { - process_statuses: "processes", - task_events: "events", - }[response.type]; - if (!emptyCollectionField) { - return assertDenied(response, pattern, label); - } - assert.deepStrictEqual( - Object.keys(response).sort(), - ["actor", emptyCollectionField, "type"] - .filter((key) => key !== "actor" || Object.hasOwn(response, key)) - .sort(), - `${label}: ${JSON.stringify(response)}` - ); - assert.deepStrictEqual( - response[emptyCollectionField], - [], - `${label}: ${JSON.stringify(response)}` - ); - return { - denied: false, - scoped_empty_result: true, - response_type: response.type, - }; -} - -function readNodeCredential(projectDir, node) { - const directory = path.join(projectDir, ".clusterflux", "nodes"); - for (const file of fs.readdirSync(directory)) { - if (!file.endsWith(".json")) continue; - const absolute = path.join(directory, file); - const credential = readJson(absolute); - if (credential.node === node) return { absolute, credential }; - } - throw new Error(`persisted credential for ${node} was not found`); -} - -function directoryStats(root) { - if (!fs.existsSync(root)) return { files: 0, bytes: 0 }; - let files = 0; - let bytes = 0; - const visit = (directory) => { - for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { - const absolute = path.join(directory, entry.name); - if (entry.isDirectory()) visit(absolute); - else if (entry.isFile()) { - files += 1; - bytes += fs.statSync(absolute).size; - } - } - }; - visit(root); - return { files, bytes }; -} - -function shellQuote(argument) { - return `'${String(argument).replaceAll("'", "'\"'\"'")}'`; -} - -function sshArgs(...remoteArgs) { - assert(strictVpsHost && strictVpsIdentity); - return [ - "-i", - path.resolve(strictVpsIdentity.replace(/^~(?=\/)/, os.homedir())), - "-o", - "BatchMode=yes", - strictVpsHost, - remoteArgs.map(shellQuote).join(" "), - ]; -} - -function strictInfrastructureSample(projectDir) { - const memory = Object.fromEntries( - run( - "ssh", - sshArgs( - "systemctl", - "show", - strictServiceUnit, - "--property=MemoryCurrent", - "--property=MemoryPeak" - ) - ) - .trim() - .split(/\r?\n/) - .map((line) => line.split("=")) - ); - const serviceDisk = Number( - run( - "ssh", - sshArgs("du", "-sb", "/var/lib/clusterflux-public-release") - ) - .trim() - .split(/\s+/)[0] - ); - const database = run( - "ssh", - sshArgs( - "runuser", - "-u", - "postgres", - "--", - "psql", - "-d", - "clusterflux", - "-AtF," - ), - { - input: - "SELECT pg_database_size('clusterflux'), (SELECT count(*) FROM clusterflux_tenants), (SELECT count(*) FROM clusterflux_projects), (SELECT count(*) FROM clusterflux_node_identities), (SELECT count(*) FROM clusterflux_cli_sessions);\n", - stdio: ["pipe", "pipe", "pipe"], - } - ) - .trim() - .split(",") - .map(Number); - assert(Number.isFinite(Number(memory.MemoryCurrent))); - assert(Number.isFinite(Number(memory.MemoryPeak))); - assert.strictEqual(database.length, 5); - return { - sampled_at_epoch_ms: Date.now(), - service_memory_current_bytes: Number(memory.MemoryCurrent), - service_memory_peak_bytes: Number(memory.MemoryPeak), - service_state_disk_bytes: serviceDisk, - postgres_database_bytes: database[0], - durable_rows: { - tenants: database[1], - projects: database[2], - node_identities: database[3], - cli_sessions: database[4], - }, - local_node_state: directoryStats(path.join(projectDir, ".clusterflux")), - }; -} - -function executable(root, name) { - return path.join(root, "bin", process.platform === "win32" ? `${name}.exe` : name); -} - -function binaryArchive(manifest) { - const platform = `${os.platform()}-${os.arch()}`; - const asset = manifest.assets.find( - (candidate) => - candidate.name.startsWith("clusterflux-public-binaries-") && - candidate.name.includes(platform) - ); - assert(asset, `missing released binary archive for ${platform}`); - assert(fs.existsSync(asset.file), `missing released binary archive ${asset.file}`); - return asset.file; -} - -function publicRepoUrl(manifest) { - const url = - process.env.CLUSTERFLUX_PUBLIC_REPO_URL || - process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || - manifest.public_repo_url || - manifest.public_repo_remote; - assert(url, "public repository URL is required"); - assert(url.includes("git.michelpaulissen.com")); - return url; -} - -function stagePublicCheckout(manifest, checkout) { - const candidateTree = process.env.CLUSTERFLUX_LIVE_PUBLIC_TREE; - if (candidateTree) { - const source = path.resolve(candidateTree); - assert.strictEqual( - source, - path.resolve(manifest.public_tree), - "explicit live candidate tree must be the tree recorded by the release manifest" - ); - fs.cpSync(source, checkout, { recursive: true }); - return { - kind: "local_filtered_release_candidate", - path: source, - published: false, - }; - } - run("git", ["clone", "--depth", "1", publicRepoUrl(manifest), checkout]); - return { - kind: "published_forgejo_checkout", - url: publicRepoUrl(manifest), - published: true, - }; -} - -function delay(milliseconds) { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -async function waitForCli(label, action, predicate, timeoutMs = 10 * 60 * 1000) { - const deadline = Date.now() + timeoutMs; - let lastValue; - let lastError; - while (Date.now() < deadline) { - try { - lastValue = action(); - if (predicate(lastValue)) return lastValue; - lastError = undefined; - } catch (error) { - lastError = error; - } - await delay(250); - } - throw new Error( - `${label} timed out; last value=${JSON.stringify(lastValue)}${ - lastError ? `; last error=${lastError.message}` : "" - }` - ); -} - -function spawnJsonLines(command, args, options) { - recordCommand(command, args); - const child = cp.spawn(command, args, { - ...options, - stdio: ["ignore", "pipe", "pipe"], - }); - let stdout = ""; - let stderr = ""; - const values = []; - const waiters = []; - - function deliver(value) { - values.push(value); - for (let index = waiters.length - 1; index >= 0; index -= 1) { - const waiter = waiters[index]; - if (waiter.predicate(value)) { - waiters.splice(index, 1); - clearTimeout(waiter.timer); - waiter.resolve(value); - } - } - } - - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - while (stdout.includes("\n")) { - const newline = stdout.indexOf("\n"); - const line = stdout.slice(0, newline).trim(); - stdout = stdout.slice(newline + 1); - if (!line) continue; - try { - deliver(JSON.parse(line)); - } catch (_) { - // Non-JSON progress stays out of the evidence report. - } - } - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - - function waitFor(predicate, label, timeoutMs = 120000) { - const existing = values.find(predicate); - if (existing) return Promise.resolve(existing); - return new Promise((resolve, reject) => { - const waiter = { predicate, resolve, reject, timer: undefined }; - waiter.timer = setTimeout(() => { - const index = waiters.indexOf(waiter); - if (index >= 0) waiters.splice(index, 1); - reject(new Error(`${label} timed out\n${stderr}`)); - }, timeoutMs); - waiters.push(waiter); - }); - } - - child.once("exit", (code) => { - for (const waiter of waiters.splice(0)) { - clearTimeout(waiter.timer); - waiter.reject(new Error(`${waiter.label || "child"} exited with ${code}\n${stderr}`)); - } - }); - return { child, values, waitFor, stderr: () => stderr }; -} - -async function stopChild(child) { - if (!child || child.exitCode !== null) return; - child.kill("SIGTERM"); - const exited = new Promise((resolve) => child.once("exit", resolve)); - const forced = delay(5000).then(() => { - if (child.exitCode === null) child.kill("SIGKILL"); - }); - await Promise.race([exited, forced]); - if (child.exitCode === null) await exited; -} - -function rawTaskEvents(report) { - return report?.events?.response?.events || []; -} - -function nodeDescriptor(status, node) { - return status?.response?.descriptors?.find( - (descriptor) => descriptor.id === node || descriptor.node === node - ); -} - -function completedFlagship(events) { - const completedDefinitions = new Set( - events - .filter((event) => event.terminal_state === "completed") - .map((event) => event.task_definition) - ); - return ( - completedDefinitions.has("prepare_source") && - completedDefinitions.has("compile_linux") && - completedDefinitions.has("package_release") && - events.some( - (event) => - event.executor === "coordinator_main" && event.terminal_state === "completed" - ) - ); -} - -function contentAddressedArtifactId(event, logicalName) { - const digest = event?.artifact_digest; - if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) { - return null; - } - const artifactId = `${logicalName}-${digest.slice("sha256:".length)}`; - return event.artifact_path?.endsWith(`/vfs/artifacts/${artifactId}`) - ? artifactId - : null; -} - -async function prepareLiveNodeCredentialSecurity({ - clusterflux, - projectDir, - scope, - tenant, - project, - suffix, -}) { - const startedAt = Date.now(); - const node = `security-node-${suffix}`; - const grant = runJson(clusterflux, ["node", "enroll", ...scope], { - cwd: projectDir, - }); - const attach = runJson( - clusterflux, - [ - "node", - "attach", - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--node", - node, - "--enrollment-grant", - grant.enrollment_grant.grant, - "--json", - ], - { cwd: projectDir } - ); - assert.strictEqual(attach.boundary.used_enrollment_exchange, true); - const stored = readNodeCredential(projectDir, node); - const identity = nodeIdentityFromPrivateKey(stored.credential.private_key); - assert.strictEqual(identity.publicKey, stored.credential.public_key); - - const heartbeat = { - type: "node_heartbeat", - tenant, - project, - node, - node_signature: signedNodeHeartbeat(tenant, project, node, identity, { - nonce: `strict-valid-${suffix}`, - }), - }; - const valid = await sendHostedControl(heartbeat); - assert.strictEqual(valid.type, "node_heartbeat"); - const replay = assertDenied( - await sendHostedControl(heartbeat), - /nonce.*already.*used|replay/i, - "replayed node credential" - ); - - const expired = assertDenied( - await sendHostedControl({ - type: "node_heartbeat", - tenant, - project, - node, - node_signature: signedNodeHeartbeat(tenant, project, node, identity, { - nonce: `strict-expired-${suffix}`, - issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, - }), - }), - /expired|clock skew/i, - "expired node credential" - ); - - const forgedIdentity = nodeIdentity("strict-forged-node", node); - const forged = assertDenied( - await sendHostedControl({ - type: "node_heartbeat", - tenant, - project, - node, - node_signature: signedNodeHeartbeat(tenant, project, node, forgedIdentity, { - nonce: `strict-forged-${suffix}`, - }), - }), - /signature|public key/i, - "forged node credential" - ); - - const originalCapabilityBody = { - type: "report_node_capabilities", - tenant, - project, - node, - capabilities: { - os: "Linux", - arch: process.arch === "x64" ? "x86_64" : process.arch, - capabilities: [], - environment_backends: [], - source_providers: [], - }, - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: false, - online: true, - }; - const modifiedEnvelope = signedNodeRequest( - node, - identity, - "report_node_capabilities", - originalCapabilityBody, - { nonce: `strict-modified-${suffix}` } - ); - modifiedEnvelope.request.online = false; - const bodyModified = assertDenied( - await sendHostedControl(modifiedEnvelope), - /signature/i, - "body-modified node credential" - ); - - return { - duration_ms: Date.now() - startedAt, - node, - identity, - credential_path: stored.absolute, - credential_digest: sha256(fs.readFileSync(stored.absolute)), - capability_body: originalCapabilityBody, - evidence: { - valid_request: valid.type, - replay, - expired, - forged, - body_modified: bodyModified, - }, - }; -} - -async function runLiveAgentCredentialSecurity({ - clusterfluxDap, - clusterflux, - projectDir, - scope, - sessionSecret, - tenant, - project, - suffix, - bundle, - securityNode, - workerRuntime, -}) { - const scenarioStartedAt = Date.now(); - const agent = `security-agent-${suffix}`; - const identity = agentIdentity("strict-live-agent", agent); - const added = runJson( - clusterflux, - ["key", "add", ...scope, "--agent", agent, "--public-key", identity.publicKey], - { cwd: projectDir } - ); - assert.strictEqual(added.command, "key add"); - - const processId = `vp-agent-security-${suffix}`; - const unsigned = { - type: "start_process", - tenant, - project, - actor_agent: agent, - process: processId, - restart: false, - }; - const validRequest = signedAgentWorkflowRequest(identity, unsigned, { - nonce: `strict-agent-valid-${suffix}`, - }); - const valid = await sendHostedControl(validRequest); - assert.strictEqual(valid.type, "process_started", JSON.stringify(valid)); - const replay = assertDenied( - await sendHostedControl(validRequest), - /nonce.*already.*used|replay/i, - "replayed agent credential" - ); - - const expired = assertDenied( - await sendHostedControl( - signedAgentWorkflowRequest( - identity, - { ...unsigned, process: `${processId}-expired` }, - { - nonce: `strict-agent-expired-${suffix}`, - issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, - } - ) - ), - /expired|clock skew/i, - "expired agent credential" - ); - - const modifiedRequest = signedAgentWorkflowRequest( - identity, - { ...unsigned, process: `${processId}-modified` }, - { nonce: `strict-agent-modified-${suffix}` } - ); - modifiedRequest.restart = true; - const bodyModified = assertDenied( - await sendHostedControl(modifiedRequest), - /signature/i, - "body-modified agent credential" - ); - - const forgedIdentity = agentIdentity("strict-live-forged-agent", agent); - const forged = assertDenied( - await sendHostedControl( - signedAgentWorkflowRequest( - forgedIdentity, - { ...unsigned, process: `${processId}-forged` }, - { - nonce: `strict-agent-forged-${suffix}`, - publicKeyFingerprint: identity.publicKeyFingerprint, - } - ) - ), - /signature|public key/i, - "forged agent credential" - ); - - const fakeEnvironmentDigest = sha256( - Buffer.from(`strict-debug-missing-participant-${suffix}`) - ); - const fakeCapabilityBody = { - ...securityNode.capability_body, - cached_environment_digests: [fakeEnvironmentDigest], - online: true, - }; - const mainTask = `ti:${processId}:main`; - const fakeTask = `missing-debug-participant-${suffix}`; - const mainLaunchBody = { - type: "launch_task", - tenant, - project, - actor_agent: agent, - task_spec: { - tenant, - project, - process: processId, - task_definition: bundle.entryStableId, - task_instance: mainTask, - dispatch: { - kind: "coordinator_node_wasm", - export: bundle.entryExport, - abi: "entrypoint_v1", - }, - environment_id: null, - environment: null, - environment_digest: null, - required_capabilities: [], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [], - vfs_epoch: valid.epoch, - bundle_digest: bundle.digest, - }, - wait_for_node: true, - artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, - wasm_module_base64: bundle.moduleBase64, - }; - let mainLaunch; - let directTaskV1; - let fakeLaunch; - let liveParentTask; - let liveParentAssignment; - let realDebugLaunch; - let preconfiguredBreakpoint; - let attachSourcePath; - let attachBreakpointLine; - const realDebugTask = `real-debug-participant-${suffix}`; - let workerPaused = false; - const baselineContainers = new Set( - run("podman", ["ps", "-q"]) - .trim() - .split(/\s+/) - .filter(Boolean) - ); - try { - mainLaunch = await sendHostedControl( - signedAgentWorkflowRequest(identity, mainLaunchBody, { - nonce: `strict-agent-main-${suffix}`, - }) - ); - assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch)); - assert.strictEqual(mainLaunch.task_instance, mainTask); - assert.strictEqual(mainLaunch.state, "running"); - assert.strictEqual(mainLaunch.actor.kind, "agent"); - assert.strictEqual(mainLaunch.actor.authenticated_without_browser, true); - - liveParentAssignment = await workerRuntime.worker.waitFor( - (value) => { - const taskSpec = value.task_assignment_response?.task_spec; - return ( - value.node_status === "assignment_started" && - value.node === workerRuntime.node && - value.process === processId && - taskSpec?.dispatch?.abi === "task_v1" && - taskSpec.task_definition === "compile_linux" && - taskSpec.environment_id === "linux" && - typeof taskSpec.environment_digest === "string" && - typeof taskSpec.source_snapshot === "string" - ); - }, - "agent main to dispatch an environment-bound live child to the real worker", - 120000 - ); - liveParentTask = liveParentAssignment.virtual_thread; - assert.strictEqual(typeof liveParentTask, "string"); - assert.notStrictEqual(liveParentTask, ""); - assert.strictEqual( - workerRuntime.child.kill("SIGSTOP"), - true, - "failed to pause the real worker after observing its live child" - ); - workerPaused = true; - - const fakeCapabilities = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "report_node_capabilities", - fakeCapabilityBody, - { nonce: `strict-agent-fake-capabilities-${suffix}` } - ) - ); - assert.strictEqual( - fakeCapabilities.type, - "node_capabilities_recorded", - JSON.stringify(fakeCapabilities) - ); - - const fakeTaskSpec = { - tenant, - project, - process: processId, - task_definition: "task_add_one", - task_instance: fakeTask, - dispatch: { - kind: "coordinator_node_wasm", - export: null, - abi: "task_v1", - }, - environment_id: "strict-debug-missing-participant", - environment: null, - environment_digest: fakeEnvironmentDigest, - required_capabilities: [], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [{ SmallJson: 41 }], - vfs_epoch: valid.epoch, - bundle_digest: bundle.digest, - }; - directTaskV1 = assertDenied( - await sendHostedControl( - signedAgentWorkflowRequest( - identity, - { - type: "launch_task", - tenant, - project, - actor_agent: agent, - task_spec: fakeTaskSpec, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${fakeTask}.txt`, - wasm_module_base64: bundle.moduleBase64, - }, - { nonce: `strict-agent-direct-task-v1-${suffix}` } - ) - ), - /external callers.*EntrypointV1|TaskV1 requires an authenticated live parent/i, - "external direct TaskV1 launch" - ); - - const liveParentSpec = - liveParentAssignment.task_assignment_response.task_spec; - attachSourcePath = path.join(projectDir, "src/lib.rs"); - const attachSourceLines = fs - .readFileSync(attachSourcePath, "utf8") - .split(/\r?\n/); - attachBreakpointLine = - attachSourceLines.findIndex((line) => line.includes("fn abort_probe(")) + 1; - assert( - attachBreakpointLine > 0, - "agent-security fixture omitted the abort_probe debug probe" - ); - preconfiguredBreakpoint = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "set_debug_breakpoints", - process: processId, - revision: 1, - probe_symbols: ["clusterflux.probe.abort_probe"], - }) - ); - assert.strictEqual( - preconfiguredBreakpoint.type, - "debug_breakpoints", - JSON.stringify(preconfiguredBreakpoint) - ); - assert.deepStrictEqual(preconfiguredBreakpoint.probe_symbols, [ - "clusterflux.probe.abort_probe", - ]); - realDebugLaunch = await sendHostedControl( - signedNodeRequest( - workerRuntime.node, - workerRuntime.identity, - "launch_child_task", - { - type: "launch_child_task", - tenant, - project, - process: processId, - node: workerRuntime.node, - parent_task: liveParentTask, - task_spec: { - tenant, - project, - process: processId, - task_definition: "abort_probe", - task_instance: realDebugTask, - dispatch: { - kind: "coordinator_node_wasm", - export: null, - abi: "task_v1", - }, - environment_id: liveParentSpec.environment_id, - environment: liveParentSpec.environment, - environment_digest: liveParentSpec.environment_digest, - required_capabilities: ["Command"], - dependency_cache: null, - source_snapshot: liveParentSpec.source_snapshot, - required_artifacts: [], - args: liveParentSpec.args, - vfs_epoch: valid.epoch, - bundle_digest: bundle.digest, - }, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${realDebugTask}.txt`, - wasm_module_base64: bundle.moduleBase64, - }, - { nonce: `strict-real-debug-participant-${suffix}` } - ) - ); - assert.strictEqual( - realDebugLaunch.type, - "task_launched", - JSON.stringify(realDebugLaunch) - ); - - const fakeLaunchBody = { - type: "launch_child_task", - tenant, - project, - process: processId, - node: workerRuntime.node, - parent_task: liveParentTask, - task_spec: fakeTaskSpec, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${fakeTask}.txt`, - wasm_module_base64: bundle.moduleBase64, - }; - fakeLaunch = await sendHostedControl( - signedNodeRequest( - workerRuntime.node, - workerRuntime.identity, - "launch_child_task", - fakeLaunchBody, - { nonce: `strict-parent-runtime-fake-task-${suffix}` } - ) - ); - assert.strictEqual(fakeLaunch.type, "task_launched", JSON.stringify(fakeLaunch)); - assert.strictEqual(fakeLaunch.placement.node, securityNode.node); - } finally { - if (workerPaused) { - assert.strictEqual( - workerRuntime.child.kill("SIGCONT"), - true, - "failed to resume the real worker after authenticated child launch" - ); - } - } - await workerRuntime.worker.waitFor( - (value) => - value.node_status === "assignment_started" && - value.virtual_thread === realDebugTask, - "real Podman debug participant to start", - 120_000 - ); - let realDebugContainerIds = new Set(); - - const debugClient = new DapClient({ - cwd: projectDir, - command: clusterfluxDap, - args: [], - env: { ...process.env }, - }); - const initialize = debugClient.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await debugClient.response(initialize, "initialize"); - const attachRequest = debugClient.send("attach", { - entry: "build", - project: projectDir, - processId, - runtimeBackend: "live-services", - coordinatorEndpoint: serviceEndpoint, - }); - await debugClient.response(attachRequest, "attach"); - await debugClient.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - const attachBreakpointRequest = debugClient.send("setBreakpoints", { - source: { path: attachSourcePath }, - breakpoints: [{ line: attachBreakpointLine }], - }); - const attachBreakpointResponse = await debugClient.response( - attachBreakpointRequest, - "setBreakpoints" - ); - assert.strictEqual( - attachBreakpointResponse.body.breakpoints[0].verified, - false, - "attach breakpoint was verified before runtime installation" - ); - const configurationDone = debugClient.send("configurationDone"); - await debugClient.response(configurationDone, "configurationDone"); - const attachBreakpointInstalled = await debugClient.waitFor( - (message) => - message.type === "event" && - message.event === "breakpoint" && - message.body?.breakpoint?.verified === true, - 120_000 - ); - assert.strictEqual( - attachBreakpointInstalled.body.breakpoint.line, - attachBreakpointLine - ); - const attachBreakpointStop = await debugClient.waitFor( - (message) => - message.seq > attachBreakpointInstalled.seq && - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint", - 120_000 - ); - const attachStackRequest = debugClient.send("stackTrace", { - threadId: attachBreakpointStop.body.threadId, - startFrame: 0, - levels: 1, - }); - const attachStackFrames = ( - await debugClient.response(attachStackRequest, "stackTrace") - ).body.stackFrames; - assert.strictEqual( - attachStackFrames[0].line, - attachBreakpointLine, - "attach stopped somewhere other than the installed configured breakpoint" - ); - const attachContinue = debugClient.send("continue", { - threadId: attachBreakpointStop.body.threadId, - }); - const attachContinueResponse = await debugClient.response( - attachContinue, - "continue" - ); - const realDebugContainerDeadline = Date.now() + 120_000; - let lastRealDebugPodmanStates = []; - while (Date.now() < realDebugContainerDeadline) { - const containers = runJson("podman", ["ps", "--format", "json"]); - lastRealDebugPodmanStates = containers; - realDebugContainerIds = new Set( - containers - .map((container) => container.Id || container.ID || container.IdHex || "") - .filter((id) => id && !baselineContainers.has(id)) - ); - if (realDebugContainerIds.size > 0) break; - await delay(100); - } - assert( - realDebugContainerIds.size > 0, - `continued real debug participant did not start a Podman container: ${JSON.stringify( - lastRealDebugPodmanStates - )}` - ); - const attachDeadline = Date.now() + 120_000; - let attachedThreads = []; - while (Date.now() < attachDeadline) { - const threadsRequest = debugClient.send("threads"); - attachedThreads = ( - await debugClient.response(threadsRequest, "threads") - ).body.threads; - if (attachedThreads.length > 0) break; - await delay(100); - } - assert(attachedThreads.length > 0, "DAP attach reported no live threads"); - - const freezeStartedAt = Date.now(); - const pauseRequest = debugClient.send("pause", { - threadId: attachedThreads[0].id, - }); - await debugClient.response(pauseRequest, "pause"); - const dapPartialStop = await debugClient.waitFor( - (message) => - message.type === "event" && - message.seq > attachContinueResponse.seq && - message.event === "stopped" && - message.body.reason === "pause", - 30_000 - ); - assert.strictEqual(dapPartialStop.body.allThreadsStopped, false); - const epochRequest = debugClient.send("evaluate", { - expression: "debug_epoch", - context: "watch", - }); - const freeze = { - epoch: Number( - (await debugClient.response(epochRequest, "evaluate")).body.result - ), - }; - assert(Number.isSafeInteger(freeze.epoch) && freeze.epoch > 0); - const partialFreeze = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "inspect_debug_epoch", - process: processId, - epoch: freeze.epoch, - }) - ); - assert.strictEqual(partialFreeze.type, "debug_epoch_status"); - assert.strictEqual(partialFreeze.partially_frozen, true, JSON.stringify(partialFreeze)); - assert.strictEqual(partialFreeze.fully_frozen, false); - assert.strictEqual(partialFreeze.failed, true); - assert.match( - partialFreeze.failure_messages.join("; "), - /did not acknowledge frozen state within \d+ ms/ - ); - const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]); - const pausedContainers = podmanStates.filter((container) => { - const id = container.Id || container.ID || container.IdHex || ""; - const state = String(container.State || container.Status || "").toLowerCase(); - return realDebugContainerIds.has(id) && state.includes("paused"); - }); - assert( - pausedContainers.length > 0, - `partial Debug Epoch did not pause a real Podman container: ${JSON.stringify( - podmanStates - )}` - ); - const pausedContainerIds = pausedContainers.map( - (container) => container.Id || container.ID || container.IdHex - ); - - const mismatchedDigest = sha256(Buffer.from(`strict-nested-mismatch-${suffix}`)); - const mismatchedChild = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "launch_child_task", - { - type: "launch_child_task", - tenant, - project, - process: processId, - node: securityNode.node, - parent_task: fakeTask, - task_spec: { - tenant, - project, - process: processId, - task_definition: "task_add_one", - task_instance: `${fakeTask}:child:mismatched-environment`, - dispatch: { - kind: "coordinator_node_wasm", - export: null, - abi: "task_v1", - }, - environment_id: "strict-nested-mismatch", - environment: null, - environment_digest: mismatchedDigest, - required_capabilities: [], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [{ SmallJson: 41 }], - vfs_epoch: valid.epoch, - bundle_digest: bundle.digest, - }, - wait_for_node: false, - artifact_path: `/vfs/artifacts/${fakeTask}-mismatched-child.txt`, - wasm_module_base64: bundle.moduleBase64, - }, - { nonce: `strict-nested-mismatch-${suffix}` } - ) - ); - const nestedEnvironmentMismatch = assertDenied( - mismatchedChild, - /environment|digest|compatible|placement|no node/i, - "nested environment mismatch" - ); - - const continueStartedAt = Date.now(); - const continueRequest = debugClient.send("continue", { - threadId: dapPartialStop.body.threadId, - }); - await debugClient.response(continueRequest, "continue"); - const continueResponseMs = Date.now() - continueStartedAt; - assert( - continueResponseMs < 2_000, - `partial-freeze continue blocked for ${continueResponseMs} ms` - ); - await delay(1000); - const resumed = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "inspect_debug_epoch", - process: processId, - epoch: freeze.epoch, - }) - ); - assert.strictEqual(resumed.type, "debug_epoch_status"); - const resumedAcknowledgements = resumed.acknowledgements.filter( - (acknowledgement) => acknowledgement.state === "running" - ); - assert(resumedAcknowledgements.length > 0, JSON.stringify(resumed)); - const resumedPodmanStates = runJson("podman", ["ps", "--format", "json"]); - assert( - !resumedPodmanStates.some((container) => { - const id = container.Id || container.ID || container.IdHex || ""; - const state = String(container.State || container.Status || "").toLowerCase(); - return pausedContainerIds.includes(id) && state.includes("paused"); - }), - `DAP continue left a Clusterflux container paused: ${JSON.stringify( - resumedPodmanStates - )}` - ); - await debugClient.close(); - - const mainBeforeChildStartedAt = Date.now(); - const completed = await waitForCli( - "agent-authenticated coordinator main and child workflow completion", - () => - runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { - cwd: projectDir, - }), - (tasks) => completedFlagship(rawTaskEvents(tasks)), - 120000 - ); - const completedEvents = rawTaskEvents(completed); - const concurrentCompileTasks = [ - ...new Set( - completedEvents - .filter( - (event) => - event.task_definition === "compile_linux" && - event.terminal_state === "completed" - ) - .map((event) => event.task) - ), - ]; - assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents)); - const mainCompletion = completedEvents.find( - (event) => - event.executor === "coordinator_main" && - event.terminal_state === "completed" - ); - assert(mainCompletion, "real coordinator main did not complete"); - assert( - !completedEvents.some( - (event) => - event.task === fakeTask && typeof event.terminal_state === "string" - ), - "the deliberately delayed final child completed before the coordinator main" - ); - const activeAfterMain = runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert.notStrictEqual(activeAfterMain.state, "not_active"); - assert.equal(activeAfterMain.live_process.main_state, null); - const debugStateAfterMain = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "inspect_debug_breakpoints", - process: processId, - }) - ); - assert.strictEqual( - debugStateAfterMain.type, - "debug_breakpoints", - `main completion cleared active-child debug state: ${JSON.stringify( - debugStateAfterMain - )}` - ); - const finalChildCompletion = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "task_completed", - { - type: "task_completed", - tenant, - project, - process: processId, - node: securityNode.node, - task: fakeTask, - terminal_state: "completed", - status_code: 0, - stdout_bytes: 2, - stderr_bytes: 0, - stdout_tail: "42", - stderr_tail: "", - stdout_truncated: false, - stderr_truncated: false, - artifact_path: null, - artifact_digest: null, - artifact_size_bytes: null, - result: { SmallJson: 42 }, - }, - { nonce: `strict-main-before-child-complete-${suffix}` } - ) - ); - assert.strictEqual( - finalChildCompletion.type, - "task_recorded", - JSON.stringify(finalChildCompletion) - ); - const releasedAfterFinalChild = await waitForCli( - "final delayed child to release the active process slot", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - const retainedEvents = runJson( - clusterflux, - ["task", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert( - rawTaskEvents(retainedEvents).some( - (event) => - event.task === fakeTask && event.terminal_state === "completed" - ), - "final delayed child history was not retained" - ); - const retainedArtifacts = runJson( - clusterflux, - ["artifact", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - const retainedArtifact = retainedArtifacts.artifacts.find( - (artifact) => - typeof artifact.digest === "string" && - /^sha256:[0-9a-f]{64}$/.test(artifact.digest) && - artifact.artifact.startsWith("release.tar-") - ); - assert(retainedArtifact, "real main artifact metadata was not retained"); - const subsequentRun = runJson( - clusterflux, - ["run", "park-wake", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(subsequentRun.status, "main_launched"); - const subsequentAbort = runJson( - clusterflux, - ["process", "abort", ...scope, "--process", subsequentRun.process, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(subsequentAbort.abort_request.accepted, true); - await waitForCli( - "subsequent run cleanup after final-child slot release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", subsequentRun.process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - const mainBeforeChildLifecycle = { - request: { - process: processId, - main: bundle.entryStableId, - delayed_child: fakeTask, - fault_injection: - "withhold the assigned final child's signed completion until the real coordinator main completes", - }, - expected: - "process and debug state remain until the final child completes; final cleanup retains history and real artifact metadata; a subsequent run starts", - observed: { - main_terminal_state: mainCompletion.terminal_state, - active_after_main: activeAfterMain.state !== "not_active", - debug_state_after_main: debugStateAfterMain.type, - child_terminal_state: "completed", - final_process_state: releasedAfterFinalChild.state, - retained_event_count: rawTaskEvents(retainedEvents).length, - retained_artifact: retainedArtifact, - subsequent_run_status: subsequentRun.status, - }, - duration_ms: Math.max(1, Date.now() - mainBeforeChildStartedAt), - }; - - const revoked = runJson( - clusterflux, - ["key", "revoke", ...scope, "--agent", agent, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(revoked.command, "key revoke"); - const revokedUse = assertDenied( - await sendHostedControl( - signedAgentWorkflowRequest( - identity, - { ...unsigned, process: `${processId}-revoked` }, - { nonce: `strict-agent-revoked-${suffix}` } - ) - ), - /revoked/i, - "revoked agent credential" - ); - return { - duration_ms: Date.now() - scenarioStartedAt, - agent, - valid_request: valid.type, - replay, - expired, - forged, - body_modified: bodyModified, - revoked: revokedUse, - workflow: { - process: processId, - main_launch: mainLaunch.type, - external_direct_task_v1: directTaskV1, - child_launch: fakeLaunch.type, - child_parent: liveParentTask, - real_debug_child_launch: realDebugLaunch.type, - real_debug_child: realDebugTask, - actor_kind: mainLaunch.actor.kind, - authenticated_without_browser: mainLaunch.actor.authenticated_without_browser, - flagship_completed: completedFlagship(completedEvents), - concurrent_compile_tasks: concurrentCompileTasks, - }, - main_before_child_lifecycle: mainBeforeChildLifecycle, - attach_with_preconfigured_breakpoint: { - request: { - process: processId, - source: attachSourcePath, - line: attachBreakpointLine, - }, - expected: - "preconfigured before attach, unverified until adapter installation, then a real breakpoint stop at the configured line", - observed: { - preconfigured_revision: preconfiguredBreakpoint.revision, - initially_verified: - attachBreakpointResponse.body.breakpoints[0].verified, - installation_event_verified: - attachBreakpointInstalled.body.breakpoint.verified, - stop_reason: attachBreakpointStop.body.reason, - stopped_line: attachStackFrames[0].line, - }, - duration_ms: Date.now() - scenarioStartedAt, - }, - partial_freeze: { - epoch: freeze.epoch, - elapsed_ms: Date.now() - freezeStartedAt, - partially_frozen: partialFreeze.partially_frozen, - fully_frozen: partialFreeze.fully_frozen, - dap_all_threads_stopped: dapPartialStop.body.allThreadsStopped, - dap_continue_response_ms: continueResponseMs, - podman_paused_container_ids: pausedContainerIds, - warning: partialFreeze.failure_messages, - resumed_participants: resumedAcknowledgements.map( - (acknowledgement) => acknowledgement.task - ), - }, - nested_environment_mismatch: nestedEnvironmentMismatch, - }; -} - -async function runSecondTenantIsolation({ - clusterflux, - clusterfluxNode, - firstScope, - firstHelloBuild, - firstHelloProjectDir, - workRoot, - firstSessionSecret, - firstTenant, - firstProject, - firstProcess, - firstNode, - firstArtifact, - manifest, -}) { - const startedAt = Date.now(); - const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE; - const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE; - const targetTenant = process.env.CLUSTERFLUX_ISOLATION_TARGET_TENANT; - const targetProject = process.env.CLUSTERFLUX_ISOLATION_TARGET_PROJECT; - if (!file && evidenceFile) { - const evidenceBytes = fs.readFileSync(path.resolve(evidenceFile)); - const previous = JSON.parse(evidenceBytes); - const isolation = previous.tenant_isolation; - assert.strictEqual(isolation?.executed, true); - assert.notStrictEqual(isolation.second_tenant, firstTenant); - assert.strictEqual(isolation.project?.denied, true); - assert.strictEqual(isolation.process_hidden, true); - assert.strictEqual(isolation.node_hidden, true); - assert.strictEqual(isolation.tasks_and_logs?.denied, true); - assert.strictEqual(isolation.debug?.denied, true); - assert.strictEqual(isolation.debug?.audited, true); - assert.strictEqual(isolation.debug?.charged_debug_read_bytes, 0); - assert.strictEqual(isolation.artifact_and_download?.denied, true); - assert.strictEqual(isolation.process_control?.denied, true); - const previousDeployment = previous.release_binding?.deployment; - const currentDeployment = deploymentProvenance(); - assert.strictEqual( - previousDeployment?.hosted_service_sha256, - currentDeployment.hosted_service_sha256, - "reused isolation evidence must target the exact deployed hosted binary" - ); - assert.deepStrictEqual( - previous.release_binding?.binary_digests, - manifest.binary_digests, - "reused isolation evidence must target the exact public binaries" - ); - return { - evidence: { - ...isolation, - duration_ms: Date.now() - startedAt, - reused_from_immutable_evidence: { - report_sha256: sha256(evidenceBytes), - source_commit: previous.source_commit, - hosted_service_sha256: currentDeployment.hosted_service_sha256, - public_binary_digests: manifest.binary_digests, - }, - }, - runtime: null, - }; - } - if (!file && targetTenant && targetProject) { - assert.notStrictEqual( - targetTenant, - firstTenant, - "single-account isolation target must name a different hosted tenant" - ); - assert.notStrictEqual( - targetProject, - firstProject, - "single-account isolation target must name a different hosted project" - ); - const call = (request) => - sendHostedControl(authenticatedRequest(firstSessionSecret, request)); - const project = assertDenied( - await call({ type: "select_project", project: targetProject }), - /outside|not visible|tenant|project|permission|unauthorized/i, - "single-account cross-tenant project selection" - ); - const processes = await call({ type: "list_processes" }); - assert.strictEqual( - processes.type, - "process_statuses", - JSON.stringify(processes) - ); - assert(!JSON.stringify(processes).includes(targetTenant)); - assert(!JSON.stringify(processes).includes(targetProject)); - const nodes = await call({ type: "list_node_descriptors" }); - assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes)); - assert(!JSON.stringify(nodes).includes(targetTenant)); - assert(!JSON.stringify(nodes).includes(targetProject)); - const foreignProcess = `vp-foreign-isolation-${Date.now()}`; - const foreignArtifact = `artifact-foreign-isolation-${Date.now()}`; - const tasksAndLogsResponse = await call({ - type: "list_task_events", - process: foreignProcess, - }); - let tasksAndLogs; - if (tasksAndLogsResponse.type === "error") { - tasksAndLogs = { - ...assertDenied( - tasksAndLogsResponse, - /outside|scope|tenant|project|not active|requires an active|unknown/i, - "single-account foreign task and log listing" - ), - enforcement: "explicit_error", - }; - } else { - assert.strictEqual( - tasksAndLogsResponse.type, - "task_events", - JSON.stringify(tasksAndLogsResponse) - ); - assert.deepStrictEqual( - tasksAndLogsResponse.events, - [], - "single-account foreign-looking task listing must disclose no events" - ); - assert(!JSON.stringify(tasksAndLogsResponse).includes(targetTenant)); - assert(!JSON.stringify(tasksAndLogsResponse).includes(targetProject)); - tasksAndLogs = { - denied: true, - reason: "authenticated scope returned no visible task or log events", - enforcement: "scoped_empty_result", - response_type: tasksAndLogsResponse.type, - event_count: tasksAndLogsResponse.events.length, - }; - } - const debugResponse = await call({ - type: "debug_attach", - process: foreignProcess, - }); - assert.strictEqual( - debugResponse.type, - "debug_attach", - JSON.stringify(debugResponse) - ); - assert.strictEqual(debugResponse.authorization?.allowed, false); - assert.strictEqual(debugResponse.audit_event?.allowed, false); - assert.strictEqual(debugResponse.charged_debug_read_bytes, 0); - const artifact = assertDenied( - await call({ - type: "create_artifact_download_link", - artifact: foreignArtifact, - max_bytes: 1024, - ttl_seconds: 60, - }), - /outside|scope|tenant|project|not found|does not exist|unavailable/i, - "single-account foreign artifact download" - ); - const control = assertDenied( - await call({ type: "abort_process", process: foreignProcess }), - /outside|scope|tenant|project|not active|requires an active|unknown/i, - "single-account foreign process control" - ); - return { - evidence: { - executed: true, - mode: "single_authenticated_account_foreign_project", - second_tenant: targetTenant, - target_project: targetProject, - project, - process_hidden: true, - node_hidden: true, - tasks_and_logs: tasksAndLogs, - debug: { - denied: true, - reason: debugResponse.authorization.reason, - audited: true, - charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, - }, - artifact_and_download: artifact, - process_control: control, - duration_ms: Date.now() - startedAt, - }, - runtime: null, - }; - } - if (!file) { - return { - evidence: { - executed: false, - reason: "second tenant session not supplied", - duration_ms: Math.max(1, Date.now() - startedAt), - }, - runtime: null, - }; - } - const secondSessionPath = path.resolve(file); - const second = readJson(secondSessionPath); - assert.strictEqual(second.coordinator, serviceEndpoint); - const secondSessionSecret = - second.cli_session_secret || second.session_secret; - assert(secondSessionSecret, "second tenant session omitted its session secret"); - assert.notStrictEqual( - second.tenant, - firstTenant, - "isolation proof requires a genuinely distinct hosted tenant" - ); - assert.notStrictEqual( - second.project, - firstProject, - "isolation proof requires a genuinely distinct hosted project" - ); - const call = (request) => - sendHostedControl(authenticatedRequest(secondSessionSecret, request)); - - const project = assertDenied( - await call({ type: "select_project", project: firstProject }), - /outside|not visible|tenant|project/i, - "cross-tenant project selection" - ); - const processes = await call({ type: "list_processes" }); - assert.strictEqual( - processes.type, - "process_statuses", - JSON.stringify(processes) - ); - assert(!JSON.stringify(processes).includes(firstProcess)); - const nodes = await call({ type: "list_node_descriptors" }); - assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes)); - assert(!JSON.stringify(nodes).includes(firstNode)); - const tasksAndLogs = assertDenied( - await call({ type: "list_task_events", process: firstProcess }), - /outside|scope|tenant|project|not active|requires an active|unknown/i, - "cross-tenant task and log listing" - ); - const debugResponse = await call({ - type: "debug_attach", - process: firstProcess, - }); - assert.strictEqual(debugResponse.type, "debug_attach", JSON.stringify(debugResponse)); - assert.strictEqual(debugResponse.authorization?.allowed, false); - assert.strictEqual(debugResponse.audit_event?.allowed, false); - assert.match( - debugResponse.authorization?.reason || "", - /outside|scope|permission|tenant|project|not active|unknown/i - ); - assert.strictEqual(debugResponse.charged_debug_read_bytes, 0); - const debug = { - denied: true, - reason: debugResponse.authorization.reason, - audited: true, - charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, - }; - const artifact = assertDenied( - await call({ - type: "create_artifact_download_link", - artifact: firstArtifact, - max_bytes: 1024, - ttl_seconds: 60, - }), - /outside|scope|tenant|project|not found|does not exist|unavailable/i, - "cross-tenant artifact download" - ); - const control = assertDenied( - await call({ type: "abort_process", process: firstProcess }), - /outside|scope|tenant|project|not active|requires an active|unknown/i, - "cross-tenant process control" - ); - - const secondCheckout = path.join(workRoot, "second-tenant-public-repo"); - stagePublicCheckout(manifest, secondCheckout); - const secondProjectDir = path.join( - secondCheckout, - "examples", - "hello-build" - ); - const secondControlDir = path.join(secondProjectDir, ".clusterflux"); - ensureDir(secondControlDir); - const secondProjectPath = path.join( - path.dirname(secondSessionPath), - "project.json" - ); - assert( - fs.existsSync(secondProjectPath), - `second tenant session is missing adjacent project.json: ${secondProjectPath}` - ); - fs.copyFileSync( - secondSessionPath, - path.join(secondControlDir, "session.json") - ); - fs.copyFileSync( - secondProjectPath, - path.join(secondControlDir, "project.json") - ); - fs.chmodSync(path.join(secondControlDir, "session.json"), 0o600); - fs.chmodSync(path.join(secondControlDir, "project.json"), 0o644); - const secondScope = [ - "--coordinator", - serviceEndpoint, - "--tenant", - second.tenant, - "--project-id", - second.project, - "--user", - second.user, - "--json", - ]; - const secondProjectList = runJson( - clusterflux, - ["project", "list", ...secondScope], - { cwd: secondProjectDir } - ); - assert(secondProjectList.project_count >= 1); - const secondProjectSelect = runJson( - clusterflux, - ["project", "select", ...secondScope, second.project], - { cwd: secondProjectDir } - ); - assert.strictEqual(secondProjectSelect.command, "project select"); - - const collisionStartedAt = Date.now(); - const secondGrant = runJson( - clusterflux, - ["node", "enroll", ...secondScope], - { cwd: secondProjectDir } - ); - const secondAttach = runJson( - clusterflux, - [ - "node", - "attach", - "--coordinator", - serviceEndpoint, - "--tenant", - second.tenant, - "--project-id", - second.project, - "--node", - firstNode, - "--enrollment-grant", - secondGrant.enrollment_grant.grant, - "--json", - ], - { cwd: secondProjectDir } - ); - assert.strictEqual(secondAttach.boundary.used_enrollment_exchange, true); - const secondStored = readNodeCredential(secondProjectDir, firstNode); - const secondCredentialDigest = sha256( - fs.readFileSync(secondStored.absolute) - ); - const secondIdentity = nodeIdentityFromPrivateKey( - secondStored.credential.private_key - ); - const secondWorkerArgs = [ - "--coordinator", - serviceEndpoint, - "--tenant", - second.tenant, - "--project-id", - second.project, - "--node", - firstNode, - "--worker", - "--project-root", - secondProjectDir, - "--assignment-poll-ms", - "500", - "--emit-ready", - ]; - const firstHelloWorker = spawnJsonLines( - clusterfluxNode, - [ - "--coordinator", - serviceEndpoint, - "--tenant", - firstTenant, - "--project-id", - firstProject, - "--node", - firstNode, - "--worker", - "--project-root", - firstHelloProjectDir, - "--assignment-poll-ms", - "500", - "--emit-ready", - ], - { - cwd: firstHelloProjectDir, - env: { ...process.env }, - } - ); - const firstHelloReady = await firstHelloWorker.waitFor( - (value) => value.node_status === "ready", - "first-tenant hello-build artifact worker ready for collision proof" - ); - assert.strictEqual(firstHelloReady.node, firstNode); - const firstCollisionHelloBuild = await runHostedHelloBuild({ - clusterflux, - projectDir: firstHelloProjectDir, - scope: firstScope, - outputFile: path.join(workRoot, "hello-clusterflux-first-tenant-collision"), - }); - assert.strictEqual( - firstCollisionHelloBuild.artifact, - firstHelloBuild.artifact, - "repeated deterministic first-tenant hello-build changed its ArtifactId" - ); - assert.strictEqual(firstCollisionHelloBuild.digest, firstHelloBuild.digest); - assert.strictEqual( - firstCollisionHelloBuild.executable_output, - firstHelloBuild.executable_output - ); - const secondWorker = spawnJsonLines(clusterfluxNode, secondWorkerArgs, { - cwd: secondProjectDir, - env: { ...process.env }, - }); - let secondHelloBuild; - try { - const secondReady = await secondWorker.waitFor( - (value) => value.node_status === "ready", - "same-name second-tenant worker ready" - ); - assert.strictEqual(secondReady.node, firstNode); - secondHelloBuild = await runHostedHelloBuild({ - clusterflux, - projectDir: secondProjectDir, - scope: secondScope, - outputFile: path.join(workRoot, "hello-clusterflux-second-tenant"), - }); - } finally { - await stopChild(secondWorker.child); - } - assert.strictEqual( - secondHelloBuild.artifact, - firstCollisionHelloBuild.artifact, - "deterministic two-tenant hello-build did not collide on ArtifactId" - ); - assert.strictEqual( - secondHelloBuild.digest, - firstCollisionHelloBuild.digest, - "deterministic two-tenant hello-build did not produce identical bytes" - ); - assert.strictEqual( - secondHelloBuild.executable_output, - firstCollisionHelloBuild.executable_output - ); - - const firstArtifactsAfterCollision = runJson( - clusterflux, - [ - "artifact", - "list", - ...firstScope, - "--process", - firstCollisionHelloBuild.process, - ], - { cwd: firstHelloProjectDir } - ); - assert( - firstArtifactsAfterCollision.artifacts.some( - (candidate) => - candidate.artifact === firstCollisionHelloBuild.artifact && - candidate.digest === firstCollisionHelloBuild.digest - ), - "the second tenant's same-ID artifact masked the first tenant's metadata" - ); - const secondCollisionLink = await call({ - type: "create_artifact_download_link", - artifact: secondHelloBuild.artifact, - max_bytes: secondHelloBuild.downloaded_bytes, - ttl_seconds: 60, - }); - assert.strictEqual( - secondCollisionLink.type, - "artifact_download_link", - JSON.stringify(secondCollisionLink) - ); - const secondCollisionLinkRevoked = await call({ - type: "revoke_artifact_download_link", - artifact: secondHelloBuild.artifact, - token_digest: secondCollisionLink.link.scoped_token_digest, - }); - assert.strictEqual( - secondCollisionLinkRevoked.type, - "artifact_download_link_revoked", - JSON.stringify(secondCollisionLinkRevoked) - ); - const firstDownloadAfterSecondLinkRevocation = runJson( - clusterflux, - [ - "artifact", - "download", - ...firstScope, - firstCollisionHelloBuild.artifact, - "--to", - path.join(workRoot, "hello-clusterflux-first-after-second-link-revoke"), - ], - { cwd: firstHelloProjectDir } - ); - assert.strictEqual( - firstDownloadAfterSecondLinkRevocation.local_download.verified_digest, - firstCollisionHelloBuild.digest - ); - await stopChild(firstHelloWorker.child); - - return { - evidence: { - executed: true, - second_tenant: second.tenant, - second_project: second.project, - project, - process_hidden: true, - node_hidden: true, - tasks_and_logs: tasksAndLogs, - debug, - artifact_and_download: artifact, - process_control: control, - scoped_node_and_artifact_collision: { - request: { - node: firstNode, - first_scope: { - tenant: firstTenant, - project: firstProject, - }, - second_scope: { - tenant: second.tenant, - project: second.project, - }, - example: "hello-build", - }, - expected: - "same-name Nodes coexist and deterministic hello-build artifacts share an ArtifactId without metadata, link, or download interference", - observed: { - first_process: firstCollisionHelloBuild.process, - second_process: secondHelloBuild.process, - same_artifact_id: secondHelloBuild.artifact, - first_digest: firstCollisionHelloBuild.digest, - second_digest: secondHelloBuild.digest, - first_downloaded_bytes: firstCollisionHelloBuild.downloaded_bytes, - second_downloaded_bytes: secondHelloBuild.downloaded_bytes, - exact_executable_output: secondHelloBuild.executable_output, - both_metadata_records_visible_to_owner: true, - cross_tenant_unique_artifact_denied: artifact.denied, - second_link_revoked: true, - first_download_survived_second_link_revocation: true, - }, - duration_ms: Math.max(1, Date.now() - collisionStartedAt), - }, - duration_ms: Math.max(1, Date.now() - startedAt), - }, - runtime: { - node: firstNode, - tenant: second.tenant, - project: second.project, - projectDir: secondProjectDir, - scope: secondScope, - workerArgs: secondWorkerArgs, - credentialPath: secondStored.absolute, - credentialDigest: secondCredentialDigest, - identity: secondIdentity, - }, - }; -} - -async function runLiveSignedHostileArtifactPath({ - clusterflux, - projectDir, - scope, - tenant, - project, - suffix, - securityNode, -}) { - const startedAt = Date.now(); - const runReport = runJson( - clusterflux, - ["run", "park-wake", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.status, "main_launched"); - const process = runReport.process; - await waitForCli( - "hostile-path probe process to become active", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", process], - { cwd: projectDir } - ), - (status) => status.state !== "not_active", - 120_000 - ); - - const invalidStartedAt = Date.now(); - const invalid = assertDenied( - await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "report_vfs_metadata", - { - type: "report_vfs_metadata", - tenant, - project, - process, - node: securityNode.node, - task: `hostile-path-${suffix}`, - artifact_path: "/vfs/artifacts/bad artifact!", - artifact_digest: sha256(Buffer.from("hostile-path-invalid")), - artifact_size_bytes: 20, - large_bytes_uploaded: false, - }, - { nonce: `strict-hostile-path-invalid-${suffix}-${Date.now()}` } - ) - ), - /invalid VFS artifact path|invalid artifact path|ArtifactId is invalid/i, - "correctly signed hostile artifact path" - ); - const invalidDurationMs = Math.max(1, Date.now() - invalidStartedAt); - - const validStartedAt = Date.now(); - const validArtifact = `strict-hostile-followup-${suffix}`; - const valid = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "report_vfs_metadata", - { - type: "report_vfs_metadata", - tenant, - project, - process, - node: securityNode.node, - task: `hostile-path-${suffix}`, - artifact_path: `/vfs/artifacts/${validArtifact}`, - artifact_digest: sha256(Buffer.from("hostile-path-valid")), - artifact_size_bytes: 18, - large_bytes_uploaded: false, - }, - { nonce: `strict-hostile-path-valid-${suffix}-${Date.now()}` } - ) - ); - assert.strictEqual(valid.type, "vfs_metadata_recorded", JSON.stringify(valid)); - const healthyHeartbeat = await sendHostedControl({ - type: "node_heartbeat", - tenant, - project, - node: securityNode.node, - node_signature: signedNodeHeartbeat( - tenant, - project, - securityNode.node, - securityNode.identity, - { nonce: `strict-hostile-path-health-${suffix}-${Date.now()}` } - ), - }); - assert.strictEqual( - healthyHeartbeat.type, - "node_heartbeat", - JSON.stringify(healthyHeartbeat) - ); - const validDurationMs = Math.max(1, Date.now() - validStartedAt); - const aborted = runJson( - clusterflux, - ["process", "abort", ...scope, "--process", process, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(aborted.abort_request.accepted, true); - await waitForCli( - "hostile-path probe process cleanup", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - return { - request: { - principal: "enrolled signed Node", - variant: "report_vfs_metadata", - process, - malformed_path: "/vfs/artifacts/bad artifact!", - }, - expected: - "structured InvalidArtifactPath rejection followed immediately by valid signed metadata and heartbeat on the same service instance", - observed: { - rejection: invalid, - valid_metadata_response: valid.type, - valid_artifact: validArtifact, - health_response: healthyHeartbeat.type, - process_cleanup: "not_active", - }, - invalid_duration_ms: invalidDurationMs, - valid_followup_duration_ms: validDurationMs, - duration_ms: Math.max(1, Date.now() - startedAt), - }; -} - -async function runLiveSoak({ - clusterflux, - projectDir, - scope, - securityNode, -}) { - if (!strictVpsRestart) { - return { executed: false, reason: "strict VPS measurement access not supplied" }; - } - const runReport = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.status, "main_launched"); - const processId = runReport.process; - const parked = await waitForCli( - "soak coordinator main to park without a capable node", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => - status.live_process?.main_state === "running" && - status.live_process?.main_wait_state === "waiting_for_task" && - status.current_task_count === 0 && - status.live_process?.connected_nodes?.length === 0, - 120000 - ); - - const pollSecurityNode = (nonce) => - sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "poll_task_assignment", - { - type: "poll_task_assignment", - tenant: securityNode.capability_body.tenant, - project: securityNode.capability_body.project, - node: securityNode.node, - }, - { nonce } - ) - ); - const drainedPriorAssignments = []; - for (let drainIndex = 0; drainIndex < 16; drainIndex += 1) { - const poll = await pollSecurityNode( - `strict-soak-drain-${drainIndex}-${Date.now()}` - ); - assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); - if (poll.assignment === null) break; - assert.notStrictEqual( - poll.assignment.process, - processId, - "soak process unexpectedly received a task assignment" - ); - drainedPriorAssignments.push({ - process: poll.assignment.process, - task: poll.assignment.task, - }); - assert(drainIndex < 15, "pre-soak assignment drain did not quiesce"); - } - - const startedAt = Date.now(); - const deadline = startedAt + strictSoakSeconds * 1000; - const samples = []; - let sampleIndex = 0; - while (true) { - const capabilities = await sendHostedControl( - signedNodeRequest( - securityNode.node, - securityNode.identity, - "report_node_capabilities", - securityNode.capability_body, - { nonce: `strict-soak-capabilities-${sampleIndex}-${Date.now()}` } - ) - ); - assert.strictEqual( - capabilities.type, - "node_capabilities_recorded", - JSON.stringify(capabilities) - ); - const poll = await pollSecurityNode( - `strict-soak-poll-${sampleIndex}-${Date.now()}` - ); - assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); - assert.strictEqual(poll.assignment, null); - const status = runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert.strictEqual(status.live_process.main_wait_state, "waiting_for_task"); - samples.push(strictInfrastructureSample(projectDir)); - sampleIndex += 1; - if (Date.now() >= deadline) break; - await delay(Math.min(30_000, deadline - Date.now())); - } - - const currentMemory = samples.map( - (sample) => sample.service_memory_current_bytes - ); - const serviceDisk = samples.map((sample) => sample.service_state_disk_bytes); - const databaseDisk = samples.map((sample) => sample.postgres_database_bytes); - const localDisk = samples.map((sample) => sample.local_node_state.bytes); - const spread = (values) => Math.max(...values) - Math.min(...values); - assert(spread(currentMemory) <= 128 * 1024 * 1024); - assert(spread(serviceDisk) <= 16 * 1024 * 1024); - assert(spread(databaseDisk) <= 32 * 1024 * 1024); - assert(spread(localDisk) <= 16 * 1024 * 1024); - assert.deepStrictEqual( - samples.at(-1).durable_rows, - samples[0].durable_rows, - "durable object counts grew during the parked-process soak" - ); - const aborted = runJson( - clusterflux, - ["process", "abort", ...scope, "--process", processId, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(aborted.abort_request.process_slot_released, true); - return { - executed: true, - process: processId, - duration_seconds: Math.floor((Date.now() - startedAt) / 1000), - sample_interval_seconds: 30, - sample_count: samples.length, - parked_main: { - main_state: parked.live_process.main_state, - main_wait_state: parked.live_process.main_wait_state, - }, - drained_prior_assignments: drainedPriorAssignments, - signed_node_poll_cycles: samples.length, - memory_spread_bytes: spread(currentMemory), - service_disk_spread_bytes: spread(serviceDisk), - postgres_disk_spread_bytes: spread(databaseDisk), - local_node_state_spread_bytes: spread(localDisk), - samples, - }; -} - -async function revokeSecurityNode({ - clusterflux, - projectDir, - scope, - securityNode, -}) { - const startedAt = Date.now(); - const revoked = runJson( - clusterflux, - ["node", "revoke", ...scope, "--node", securityNode.node, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(revoked.command, "node revoke"); - const denied = assertDenied( - await sendHostedControl({ - type: "node_heartbeat", - tenant: securityNode.capability_body.tenant, - project: securityNode.capability_body.project, - node: securityNode.node, - node_signature: signedNodeHeartbeat( - securityNode.capability_body.tenant, - securityNode.capability_body.project, - securityNode.node, - securityNode.identity, - { nonce: `strict-node-revoked-${Date.now()}` } - ), - }), - /revoked|unknown node|not recognized|not enrolled/i, - "revoked node credential" - ); - return { - node: securityNode.node, - denied, - duration_ms: Date.now() - startedAt, - }; -} - -async function runLiveQuotaPreallocation({ sessionSecret, suffix }) { - const startedAt = Date.now(); - const readSpawnQuota = async () => { - const status = await sendHostedControl( - authenticatedRequest(sessionSecret, { type: "quota_status" }) - ); - assert.strictEqual(status.type, "quota_status", JSON.stringify(status)); - const limit = status.limits?.limits?.Spawn; - const usage = status.usage?.Spawn; - const windowSeconds = status.window_seconds?.Spawn; - const windowStarted = status.window_started_epoch_seconds?.Spawn; - assert(Number.isInteger(limit) && limit > 0, JSON.stringify(status)); - assert(Number.isInteger(usage) && usage >= 0, JSON.stringify(status)); - assert( - Number.isInteger(windowSeconds) && windowSeconds > 0, - JSON.stringify(status) - ); - assert(Number.isInteger(windowStarted), JSON.stringify(status)); - return { limit, usage, windowSeconds, windowStarted }; - }; - - let spawnQuota = await readSpawnQuota(); - const windowElapsed = Math.max( - 0, - Math.floor(Date.now() / 1000) - spawnQuota.windowStarted - ); - if (spawnQuota.usage > 0 || windowElapsed > 1) { - await delay( - Math.max(1000, (spawnQuota.windowSeconds - windowElapsed + 1) * 1000) - ); - spawnQuota = await readSpawnQuota(); - } - let denial; - let deniedProcess; - let acceptedStarts = 0; - const maxAttempts = spawnQuota.limit - spawnQuota.usage + 2; - for (let index = 0; index < maxAttempts; index += 1) { - const processId = `vp-quota-${suffix}-${index}`; - const started = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "start_process", - process: processId, - restart: false, - }) - ); - if (started.type === "error") { - assert.match(started.message, /quota|spawn|limit|community tier/i); - denial = started; - deniedProcess = processId; - break; - } - assert.strictEqual(started.type, "process_started", JSON.stringify(started)); - acceptedStarts += 1; - const aborted = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "abort_process", - process: processId, - }) - ); - assert.strictEqual(aborted.type, "process_aborted", JSON.stringify(aborted)); - } - assert( - denial, - `live community-tier spawn quota was not reached in ${maxAttempts} attempts` - ); - const processes = await sendHostedControl( - authenticatedRequest(sessionSecret, { type: "list_processes" }) - ); - assert.strictEqual( - processes.type, - "process_statuses", - JSON.stringify(processes) - ); - assert( - !JSON.stringify(processes).includes(deniedProcess), - "quota-denied process was allocated before denial" - ); - const independentScope = await sendHostedControl( - authenticatedRequest(sessionSecret, { type: "list_node_descriptors" }) - ); - assert.strictEqual( - independentScope.type, - "node_descriptors", - `spawn quota exhaustion locked an independent read scope: ${JSON.stringify(independentScope)}` - ); - return { - duration_ms: Date.now() - startedAt, - limit: spawnQuota.limit, - initial_usage: spawnQuota.usage, - window_seconds: spawnQuota.windowSeconds, - accepted_starts_before_denial: acceptedStarts, - denied_process: deniedProcess, - denial: { - type: denial.type, - category: denial.error?.category || null, - message: denial.message, - }, - denied_process_allocated: false, - independent_scope: { - operation: "list_node_descriptors", - response: independentScope.type, - unaffected_by_spawn_quota: true, - }, - }; -} - -async function runLongJoinProof({ clusterflux, projectDir, scope }) { - const startedAt = Date.now(); - const runReport = runJson( - clusterflux, - ["run", "long-join", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.status, "main_launched"); - const terminal = await waitForCli( - "controlled task longer than two minutes", - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (tasks) => - rawTaskEvents(tasks).some( - (event) => - event.task_definition === "long_join_probe" && - event.terminal_state === "completed" - ), - 5 * 60 * 1000 - ); - const completed = rawTaskEvents(terminal).find( - (event) => - event.task_definition === "long_join_probe" && - event.terminal_state === "completed" - ); - const durationSeconds = Math.floor((Date.now() - startedAt) / 1000); - assert(durationSeconds > 120); - const released = await waitForCli( - "long join process automatic slot release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120000 - ); - return { - process: runReport.process, - task: completed.task, - terminal_state: completed.terminal_state, - result: completed.result, - duration_seconds: durationSeconds, - duration_ms: Math.max(1, Date.now() - startedAt), - process_state: released.state, - }; -} - -async function runRepeatedParkWakeProof({ clusterflux, projectDir, scope }) { - const runReport = runJson( - clusterflux, - ["run", "park-wake", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.status, "main_launched"); - const terminal = await waitForCli( - "repeated coordinator-main park/wake completion", - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (tasks) => { - const completed = new Set( - rawTaskEvents(tasks) - .filter( - (event) => - event.task_definition === "task_add_one" && - event.terminal_state === "completed" - ) - .map((event) => event.task) - ); - return completed.size >= 16; - }, - 120000 - ); - const completedTasks = [ - ...new Set( - rawTaskEvents(terminal) - .filter( - (event) => - event.task_definition === "task_add_one" && - event.terminal_state === "completed" - ) - .map((event) => event.task) - ), - ]; - const released = await waitForCli( - "park/wake process automatic slot release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120000 - ); - return { - process: runReport.process, - completed_cycles: completedTasks.length, - task_instances: completedTasks, - terminal_state: released.state, - }; -} - -async function runDroppedConnectionRollback({ - clusterflux, - projectDir, - scope, -}) { - const scenarioStartedAt = Date.now(); - const processId = "vp-current"; - const attempted = cp.spawnSync( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { - cwd: projectDir, - env: { - ...process.env, - CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION: "start_process", - }, - encoding: "utf8", - timeout: 5 * 60 * 1000, - } - ); - assert.notStrictEqual( - attempted.status, - 0, - `response-loss run unexpectedly succeeded: ${attempted.stdout}` - ); - assert.match( - `${attempted.stderr}\n${attempted.stdout}`, - /closed.*without a response|transport|response/i - ); - const released = await waitForCli( - "CLI launch guard rollback after lost start response", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 30000 - ); - return { - duration_ms: Date.now() - scenarioStartedAt, - process: processId, - failure_injection: "control transport dropped attempt-owned start_process response", - cli_exit_status: attempted.status, - rollback: "automatic CLI launch guard abort_process with matching launch_attempt", - terminal_state: released.state, - }; -} - -async function runLaunchAttemptOwnershipGuards({ - clusterflux, - projectDir, - scope, - sessionSecret, - activeProcess, -}) { - const startedAt = Date.now(); - const rejectedAttempt = `launch-guard-rejected-${Date.now()}`; - const rejection = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "start_process", - process: `${activeProcess}-contender`, - launch_attempt: rejectedAttempt, - restart: false, - }) - ); - assert.strictEqual(rejection.type, "error"); - assert.match(rejection.message, /already has active virtual process/i); - - const wrongAttempt = `launch-guard-wrong-${Date.now()}`; - const deniedAbort = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "abort_process", - process: activeProcess, - launch_attempt: wrongAttempt, - }) - ); - assert.strictEqual(deniedAbort.type, "error"); - assert.match(deniedAbort.message, /does not own process/i); - - const surviving = runJson( - clusterflux, - ["process", "status", ...scope, "--process", activeProcess], - { cwd: projectDir } - ); - assert.notStrictEqual(surviving.state, "not_active"); - assert.strictEqual(surviving.process, activeProcess); - return { - duration_ms: Date.now() - startedAt, - rejected_attempt: rejectedAttempt, - rejection: rejection.message, - wrong_abort_attempt: wrongAttempt, - wrong_abort_denied: deniedAbort.message, - existing_process_survived: true, - }; -} - -async function runLiveBandwidthPreallocation({ - sessionSecret, - artifact, - maxBytes, -}) { - const scenarioStartedAt = Date.now(); - const before = relayDurableState(); - const partialLink = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "create_artifact_download_link", - artifact, - max_bytes: maxBytes, - ttl_seconds: 300, - }) - ); - assert.strictEqual( - partialLink.type, - "artifact_download_link", - JSON.stringify(partialLink) - ); - const partialToken = partialLink.link.scoped_token_digest; - let partialChunk; - const partialDeadline = Date.now() + 120_000; - while (Date.now() < partialDeadline) { - const response = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "open_artifact_download_stream", - artifact, - max_bytes: maxBytes, - token_digest: partialToken, - chunk_bytes: 16, - }) - ); - assert.strictEqual( - response.type, - "artifact_download_stream", - JSON.stringify(response) - ); - if (response.content_bytes_available === true) { - partialChunk = response; - break; - } - await delay(50); - } - assert(partialChunk, "artifact relay did not return the first partial chunk"); - assert.strictEqual(partialChunk.content_eof, false); - assert.strictEqual(partialChunk.streamed_bytes, 16); - const partialRevoked = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "revoke_artifact_download_link", - artifact, - token_digest: partialToken, - }) - ); - assert.strictEqual( - partialRevoked.type, - "artifact_download_link_revoked", - JSON.stringify(partialRevoked) - ); - const afterPartialAbandon = relayDurableState(); - assert(afterPartialAbandon.ingress_used > before.ingress_used); - assert(afterPartialAbandon.egress_used > before.egress_used); - assert( - afterPartialAbandon.abandoned_or_failed_used > - before.abandoned_or_failed_used - ); - - let denial; - const acceptedReservations = []; - for (let index = 0; index < 64; index += 1) { - const response = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "create_artifact_download_link", - artifact, - max_bytes: maxBytes, - ttl_seconds: 300, - }) - ); - if (response.type === "error") { - assert.match( - response.message, - /artifact relay.*(?:concurrency|period byte budget)|bandwidth|reservation/i - ); - denial = response; - break; - } - assert.strictEqual(response.type, "artifact_download_link", JSON.stringify(response)); - acceptedReservations.push(response.link.scoped_token_digest); - } - assert(denial, "artifact relay preallocation was not denied in 64 reservations"); - for (const tokenDigest of acceptedReservations) { - const revoked = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "revoke_artifact_download_link", - artifact, - token_digest: tokenDigest, - }) - ); - assert.strictEqual( - revoked.type, - "artifact_download_link_revoked", - JSON.stringify(revoked) - ); - } - const afterRelease = relayDurableState(); - assert.strictEqual( - Object.keys(afterRelease.reservations || {}).length, - Object.keys(before.reservations || {}).length - ); - return { - scenario_started_at_ms: scenarioStartedAt, - artifact, - accepted_reservations_before_denial: acceptedReservations.length, - bytes_served_before_denial: partialChunk.streamed_bytes, - partial_abandon: { - ingress_delta: afterPartialAbandon.ingress_used - before.ingress_used, - egress_delta: afterPartialAbandon.egress_used - before.egress_used, - abandoned_delta: - afterPartialAbandon.abandoned_or_failed_used - - before.abandoned_or_failed_used, - reservation_released: !(partialToken in afterRelease.reservations), - }, - durable_before: before, - durable_after_release: afterRelease, - denial: { - type: denial.type, - category: denial.error?.category || null, - message: denial.message, - }, - }; -} - -function relayDurableState() { - if (!strictVpsRestart) { - throw new Error("strict relay accounting requires VPS Postgres access"); - } - const output = run( - "ssh", - sshArgs( - "sudo", - "-u", - "postgres", - "psql", - "-d", - "clusterflux", - "-At", - "-c", - "SELECT record::text FROM clusterflux_artifact_relay_state WHERE singleton = TRUE" - ) - ).trim(); - assert.notStrictEqual(output, "", "artifact relay durable state was not persisted"); - return JSON.parse(output.split(/\r?\n/).filter(Boolean).at(-1)); -} - -async function waitForHostedControlReady(timeoutMs = 60_000) { - const deadline = Date.now() + timeoutMs; - let lastError; - while (Date.now() < deadline) { - try { - const ping = await sendHostedControl({ type: "ping" }); - if (ping.type === "pong") return ping; - lastError = new Error(JSON.stringify(ping)); - } catch (error) { - lastError = error; - } - await delay(500); - } - throw new Error(`hosted service did not recover: ${lastError}`); -} - -async function publishRelayProbeArtifact({ clusterflux, projectDir, scope, label }) { - const runReport = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - await waitForCli( - `${label} flagship completion`, - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (report) => completedFlagship(rawTaskEvents(report)), - 10 * 60 * 1000 - ); - const artifacts = runJson( - clusterflux, - ["artifact", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ); - const artifact = artifacts.artifacts.find((candidate) => { - const digest = candidate.digest; - return ( - typeof digest === "string" && - /^sha256:[0-9a-f]{64}$/.test(digest) && - candidate.artifact === `release.tar-${digest.slice("sha256:".length)}` - ); - }); - assert(artifact, `${label} did not publish a relay probe artifact`); - return artifact; -} - -async function runRelayEmergencyDisable({ - clusterflux, - clusterfluxNode, - projectDir, - scope, - workerArgs, - sessionSecret, - maxBytes, -}) { - const startedAt = Date.now(); - if (!strictVpsRestart) { - throw new Error("strict relay disable proof requires VPS systemd access"); - } - const dropInDir = `/run/systemd/system/${strictServiceUnit}.d`; - const dropInPath = `${dropInDir}/90-strict-relay-disable.conf`; - run("ssh", sshArgs("sudo", "mkdir", "-p", dropInDir)); - cp.execFileSync("ssh", sshArgs("sudo", "tee", dropInPath), { - cwd: repo, - input: "[Service]\nEnvironment=CLUSTERFLUX_HOSTED_RELAY_DISABLED=true\n", - stdio: ["pipe", "ignore", "inherit"], - }); - let disabled; - let disabledArtifact; - let disabledWorker; - try { - run( - "ssh", - sshArgs( - "sudo", - "systemctl", - "daemon-reload" - ) - ); - run( - "ssh", - sshArgs("sudo", "systemctl", "restart", strictServiceUnit) - ); - await waitForHostedControlReady(); - disabledWorker = spawnJsonLines(clusterfluxNode, workerArgs, { - cwd: projectDir, - env: { ...process.env }, - }); - await disabledWorker.waitFor( - (value) => value.node_status === "ready", - "relay-disabled worker ready" - ); - disabledArtifact = await publishRelayProbeArtifact({ - clusterflux, - projectDir, - scope, - label: "relay-disabled", - }); - disabled = assertDenied( - await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "create_artifact_download_link", - artifact: disabledArtifact.artifact, - max_bytes: maxBytes, - ttl_seconds: 60, - }) - ), - /artifact relay is disabled/i, - "emergency-disabled artifact relay" - ); - } finally { - if (disabledWorker) await stopChild(disabledWorker.child); - run("ssh", sshArgs("sudo", "rm", "-f", dropInPath)); - run("ssh", sshArgs("sudo", "systemctl", "daemon-reload")); - run( - "ssh", - sshArgs("sudo", "systemctl", "restart", strictServiceUnit) - ); - await waitForHostedControlReady(); - } - const restoredWorker = spawnJsonLines(clusterfluxNode, workerArgs, { - cwd: projectDir, - env: { ...process.env }, - }); - try { - await restoredWorker.waitFor( - (value) => value.node_status === "ready", - "relay-restored worker ready" - ); - const restoredArtifact = await publishRelayProbeArtifact({ - clusterflux, - projectDir, - scope, - label: "relay-restored", - }); - const restored = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "create_artifact_download_link", - artifact: restoredArtifact.artifact, - max_bytes: maxBytes, - ttl_seconds: 60, - }) - ); - assert.strictEqual(restored.type, "artifact_download_link", JSON.stringify(restored)); - const revoked = await sendHostedControl( - authenticatedRequest(sessionSecret, { - type: "revoke_artifact_download_link", - artifact: restoredArtifact.artifact, - token_digest: restored.link.scoped_token_digest, - }) - ); - assert.strictEqual(revoked.type, "artifact_download_link_revoked"); - return { - evidence: { - duration_ms: Date.now() - startedAt, - disabled, - disabled_probe_artifact: disabledArtifact.artifact, - restored: restored.type, - restored_probe_artifact: restoredArtifact.artifact, - restored_reservation_released: revoked.type, - runtime_drop_in_removed: true, - }, - worker: restoredWorker, - }; - } catch (error) { - await stopChild(restoredWorker.child); - throw error; - } -} - -function deploymentProvenance() { - if (!strictVpsRestart) return null; - const systemGeneration = run( - "ssh", - sshArgs("readlink", "-f", "/run/current-system") - ).trim(); - const mainPid = run( - "ssh", - sshArgs( - "systemctl", - "show", - strictServiceUnit, - "--property=MainPID", - "--value" - ) - ).trim(); - assert.match(mainPid, /^[1-9][0-9]*$/, "hosted service has no live MainPID"); - const executable = run( - "ssh", - sshArgs("readlink", "-f", `/proc/${mainPid}/exe`) - ).trim(); - const binarySha256 = run( - "ssh", - sshArgs("sha256sum", `/proc/${mainPid}/exe`) - ) - .trim() - .split(/\s+/)[0]; - const fragment = run( - "ssh", - sshArgs("systemctl", "show", strictServiceUnit, "--property=FragmentPath", "--value") - ).trim(); - const serviceUnitSha256 = run( - "ssh", - sshArgs("sha256sum", fragment) - ) - .trim() - .split(/\s+/)[0]; - const renderedServiceConfiguration = run( - "ssh", - sshArgs("systemctl", "cat", strictServiceUnit) - ); - const renderedServiceConfigurationSha256 = sha256( - Buffer.from(renderedServiceConfiguration) - ); - return { - system_generation: systemGeneration, - hosted_service_executable: executable, - hosted_service_sha256: `sha256:${binarySha256}`, - service_unit: fragment, - service_unit_sha256: `sha256:${serviceUnitSha256}`, - service_configuration_sha256: renderedServiceConfigurationSha256, - }; -} - -function configurationProvenance() { - const configRepo = path.resolve( - process.env.CLUSTERFLUX_DEPLOYMENT_CONFIG_REPO || - path.join(repo, "..", "michelpaulissen.com") - ); - const status = run("git", ["status", "--short"], { cwd: configRepo }).trim(); - const revision = run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(); - return { - repository: configRepo, - revision, - evidence_identity: sha256(Buffer.from(revision)), - clean: status === "", - status: status || null, - }; -} - -function proxyConfigurationProvenance() { - if (!strictVpsRestart) return null; - const proxyUnit = process.env.CLUSTERFLUX_STRICT_PROXY_UNIT || "nginx.service"; - const fragment = run( - "ssh", - sshArgs("systemctl", "show", "--property=FragmentPath", "--value", proxyUnit) - ).trim(); - assert(fragment, `proxy service ${proxyUnit} has no FragmentPath`); - const unitSha256 = run( - "ssh", - sshArgs("sha256sum", fragment) - ).trim().split(/\s+/)[0]; - const proxyExecStart = run( - "ssh", - sshArgs("systemctl", "show", "--property=ExecStart", "--value", proxyUnit) - ).trim(); - const nginxExecutable = proxyExecStart.match(/path=([^ ;]+)/)?.[1]; - const nginxConfiguration = proxyExecStart.match(/ argv\[\]=.* -c ([^ ;]+)/)?.[1]; - assert(nginxExecutable, `proxy service ${proxyUnit} has no executable identity`); - assert(nginxConfiguration, `proxy service ${proxyUnit} has no configuration identity`); - const renderedConfiguration = run( - "ssh", - sshArgs(nginxExecutable, "-T", "-c", nginxConfiguration) - ); - const renderedConfigurationIdentity = sha256(Buffer.from(renderedConfiguration)); - const activeState = run( - "ssh", - sshArgs("systemctl", "is-active", proxyUnit) - ).trim(); - assert.strictEqual(activeState, "active", `proxy service ${proxyUnit} is not active`); - return { - proxy_unit: proxyUnit, - proxy_unit_fragment: fragment, - proxy_unit_sha256: `sha256:${unitSha256}`, - proxy_executable: nginxExecutable, - proxy_configuration: nginxConfiguration, - rendered_configuration_sha256: renderedConfigurationIdentity, - evidence_identity: renderedConfigurationIdentity, - active_state: activeState, - }; -} - -async function restartHostedServiceAndResume({ - clusterflux, - clusterfluxNode, - projectDir, - scope, - workerArgs, - workerNode, - credentialPath, - credentialDigest, - scopedCollisionRuntime, -}) { - if (!strictVpsRestart) { - return { - executed: false, - reason: "strict VPS restart access not supplied", - worker: null, - }; - } - const before = deploymentProvenance(); - run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); - const deadline = Date.now() + 120000; - let ping; - let lastError; - while (Date.now() < deadline) { - try { - ping = await sendHostedControl({ type: "ping" }); - if (ping.type === "pong") break; - } catch (error) { - lastError = error; - } - await delay(500); - } - assert.strictEqual( - ping?.type, - "pong", - `hosted service did not recover after restart: ${lastError?.message || "no pong"}` - ); - const afterFirstRestart = deploymentProvenance(); - assert.strictEqual(afterFirstRestart.system_generation, before.system_generation); - assert.strictEqual( - afterFirstRestart.hosted_service_sha256, - before.hosted_service_sha256 - ); - - const projects = runJson(clusterflux, ["project", "list", ...scope], { - cwd: projectDir, - }); - assert(projects.project_count >= 1, "CLI session/project did not survive restart"); - const ephemeralRun = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(ephemeralRun.status, "main_launched"); - const ephemeralProcess = ephemeralRun.process; - await waitForCli( - "pre-restart ephemeral main to park", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", ephemeralProcess], - { cwd: projectDir } - ), - (status) => status.live_process?.main_wait_state === "waiting_for_node", - 120000 - ); - - run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); - const secondDeadline = Date.now() + 120000; - ping = undefined; - lastError = undefined; - while (Date.now() < secondDeadline) { - try { - ping = await sendHostedControl({ type: "ping" }); - if (ping.type === "pong") break; - } catch (error) { - lastError = error; - } - await delay(500); - } - assert.strictEqual( - ping?.type, - "pong", - `hosted service did not recover after ephemeral-state restart: ${ - lastError?.message || "no pong" - }` - ); - const after = deploymentProvenance(); - assert.strictEqual(after.system_generation, before.system_generation); - assert.strictEqual(after.hosted_service_sha256, before.hosted_service_sha256); - const oldStatus = runJson( - clusterflux, - ["process", "status", ...scope, "--process", ephemeralProcess], - { cwd: projectDir } - ); - assert.strictEqual(oldStatus.state, "not_active"); - assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); - - const worker = spawnJsonLines(clusterfluxNode, workerArgs, { - cwd: projectDir, - env: { ...process.env }, - }); - const ready = await worker.waitFor( - (value) => value.node_status === "ready", - "persisted worker ready after hosted service restart" - ); - assert.strictEqual(ready.node, workerNode); - assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); - - let scopedNodeCollisionAfterRestart = null; - if (scopedCollisionRuntime) { - assert.strictEqual(scopedCollisionRuntime.node, workerNode); - assert.notStrictEqual( - scopedCollisionRuntime.tenant, - readJson(path.join(projectDir, ".clusterflux", "session.json")).tenant - ); - assert.strictEqual( - sha256(fs.readFileSync(scopedCollisionRuntime.credentialPath)), - scopedCollisionRuntime.credentialDigest - ); - const secondWorker = spawnJsonLines( - clusterfluxNode, - scopedCollisionRuntime.workerArgs, - { - cwd: scopedCollisionRuntime.projectDir, - env: { ...process.env }, - } - ); - try { - const secondReady = await secondWorker.waitFor( - (value) => value.node_status === "ready", - "same-name second-tenant worker ready after hosted service restart" - ); - assert.strictEqual(secondReady.node, workerNode); - const firstStatus = await waitForCli( - "first scoped same-name Node online after restart", - () => - runJson( - clusterflux, - ["node", "status", ...scope, "--node", workerNode], - { cwd: projectDir } - ), - (status) => nodeDescriptor(status, workerNode)?.online === true, - 30_000 - ); - const secondStatus = await waitForCli( - "second scoped same-name Node online after restart", - () => - runJson( - clusterflux, - [ - "node", - "status", - ...scopedCollisionRuntime.scope, - "--node", - workerNode, - ], - { cwd: scopedCollisionRuntime.projectDir } - ), - (status) => nodeDescriptor(status, workerNode)?.online === true, - 30_000 - ); - scopedNodeCollisionAfterRestart = { - node: workerNode, - first_scope_online: - nodeDescriptor(firstStatus, workerNode)?.online === true, - second_scope_online: - nodeDescriptor(secondStatus, workerNode)?.online === true, - both_credentials_reused_without_grants: true, - distinct_credential_digests: - credentialDigest !== scopedCollisionRuntime.credentialDigest, - }; - assert.strictEqual( - scopedNodeCollisionAfterRestart.distinct_credential_digests, - true, - "same-name scoped Nodes unexpectedly reused one credential" - ); - } finally { - await stopChild(secondWorker.child); - } - } - - const restartedRun = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(restartedRun.status, "main_launched"); - const restartedProcess = restartedRun.process; - const completed = await waitForCli( - "new process completion after hosted service restart", - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", restartedProcess], - { cwd: projectDir } - ), - (tasks) => completedFlagship(rawTaskEvents(tasks)), - 5 * 60 * 1000 - ); - const completedEvent = rawTaskEvents(completed).find( - (event) => - event.executor === "coordinator_main" && - event.terminal_state === "completed" - ); - assert(completedEvent, "post-restart flagship omitted its completed main event"); - return { - executed: true, - account_project_session_preserved: true, - node_identity_preserved: true, - old_process_terminated_honestly: true, - terminated_ephemeral_process: ephemeralProcess, - new_process: restartedProcess, - new_process_result: completedEvent.result, - scoped_node_collision_after_restart: scopedNodeCollisionAfterRestart, - before, - after_first_restart: afterFirstRestart, - after, - worker, - }; -} - -async function finishScopedNodeCollisionAfterRestart({ - clusterflux, - projectDir, - scope, - workerNode, - scopedCollisionRuntime, -}) { - const startedAt = Date.now(); - assert(scopedCollisionRuntime, "scoped collision runtime evidence is required"); - const revoked = runJson( - clusterflux, - [ - "node", - "revoke", - ...scopedCollisionRuntime.scope, - "--node", - scopedCollisionRuntime.node, - "--yes", - ], - { cwd: scopedCollisionRuntime.projectDir } - ); - assert.strictEqual(revoked.command, "node revoke"); - const revokedHeartbeat = assertDenied( - await sendHostedControl({ - type: "node_heartbeat", - tenant: scopedCollisionRuntime.tenant, - project: scopedCollisionRuntime.project, - node: scopedCollisionRuntime.node, - node_signature: signedNodeHeartbeat( - scopedCollisionRuntime.tenant, - scopedCollisionRuntime.project, - scopedCollisionRuntime.node, - scopedCollisionRuntime.identity, - { - nonce: `strict-scoped-node-revoked-${Date.now()}`, - } - ), - }), - /not enrolled|unknown node|revoked|credential/i, - "revoked second scoped Node" - ); - const firstStatus = await waitForCli( - "first same-name Node remains live after scoped revocation", - () => - runJson( - clusterflux, - ["node", "status", ...scope, "--node", workerNode], - { cwd: projectDir } - ), - (status) => nodeDescriptor(status, workerNode)?.online === true, - 30_000 - ); - return { - request: { - revoked_scope: { - tenant: scopedCollisionRuntime.tenant, - project: scopedCollisionRuntime.project, - node: scopedCollisionRuntime.node, - }, - retained_node: workerNode, - }, - expected: - "revocation affects only the selected scoped Node after both same-name credentials survive restart", - observed: { - revoked_command: revoked.command, - revoked_credential_denied: revokedHeartbeat.denied, - first_scope_node_online: - nodeDescriptor(firstStatus, workerNode)?.online === true, - }, - duration_ms: Math.max(1, Date.now() - startedAt), - }; -} - -async function finishUserCredentialSecurity({ - clusterflux, - projectDir, - scope, - sessionSecret, -}) { - const startedAt = Date.now(); - const forged = assertDenied( - await sendHostedControl( - authenticatedRequest( - `forged-session-${crypto.randomBytes(24).toString("hex")}`, - { type: "list_projects" } - ) - ), - /not recognized|invalid|authentication|session/i, - "forged user session" - ); - - let expired = null; - const expiredFile = process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE; - if (expiredFile) { - const oldSession = readJson(path.resolve(expiredFile)); - assert(oldSession.session_secret, "expired session evidence omitted its secret"); - expired = assertDenied( - await sendHostedControl( - authenticatedRequest(oldSession.session_secret, { type: "list_projects" }) - ), - /expired|not recognized/i, - "expired user session" - ); - } else if (strictFullRelease) { - throw new Error( - "strict full release requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE for a genuinely expired hosted session" - ); - } - - const logout = runJson(clusterflux, ["logout", ...scope, "--yes"], { - cwd: projectDir, - }); - assert.strictEqual(logout.server_session_revocation.revoked, true); - const revoked = assertDenied( - await sendHostedControl( - authenticatedRequest(sessionSecret, { type: "list_projects" }) - ), - /revoked|not recognized/i, - "revoked user session" - ); - return { - forged, - expired, - revoked, - duration_ms: Date.now() - startedAt, - }; -} - -async function dapVariables(client, variablesReference) { - const request = client.send("variables", { variablesReference }); - return (await client.response(request, "variables")).body.variables; -} - -async function runSameDefinitionDapIdentity({ - clusterfluxDap, - clusterflux, - projectDir, - scope, -}) { - const scenarioStartedAt = Date.now(); - const baselineContainers = new Set( - run("podman", ["ps", "-q"]) - .trim() - .split(/\s+/) - .filter(Boolean) - ); - const client = new DapClient({ - cwd: projectDir, - command: clusterfluxDap, - args: [], - env: { - ...process.env, - CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1", - CLUSTERFLUX_TEST_DAP_POST_COMMIT_OBSERVATION_FAILURE: "1", - }, - }); - let disconnected = false; - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await client.response(initialize, "initialize"); - const launch = client.send("launch", { - entry: "identity", - project: projectDir, - runtimeBackend: "live-services", - coordinatorEndpoint: serviceEndpoint, - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - const configurationDone = client.send("configurationDone"); - await client.response(configurationDone, "configurationDone"); - - const mainThreadDeadline = Date.now() + 5 * 60 * 1000; - let mainThread; - while (Date.now() < mainThreadDeadline) { - const threadsRequest = client.send("threads"); - const threads = (await client.response(threadsRequest, "threads")).body - .threads; - mainThread = threads.find((thread) => /coordinator main/i.test(thread.name)); - if (mainThread) break; - await delay(100); - } - assert(mainThread, "identity entry did not report its coordinator main"); - const launchWithoutBreakpointMs = Date.now() - scenarioStartedAt; - const postCommitDiagnostics = [ - "initial task observation failed after main_launched", - "initial process observation failed after main_launched", - ].map((expected) => { - const message = client.messages.find( - (candidate) => - candidate.type === "event" && - candidate.event === "output" && - String(candidate.body?.output || "").includes(expected) - ); - assert(message, `post-commit launch evidence omitted: ${expected}`); - return String(message.body.output).trim(); - }); - const prematurePostCommitEvents = client.messages.filter( - (message) => - message.type === "event" && - ["stopped", "terminated"].includes(message.event) - ); - assert.deepStrictEqual(prematurePostCommitEvents, []); - const processRequest = client.send("evaluate", { - expression: "virtual_process_id", - context: "watch", - }); - const processId = (await client.response(processRequest, "evaluate")).body - .result; - const identityContainerDeadline = Date.now() + 2 * 60 * 1000; - let identityContainerIds = new Set(); - let maximumNewContainerCount = 0; - let lastPodmanStates = []; - while (Date.now() < identityContainerDeadline) { - const containers = runJson("podman", ["ps", "--format", "json"]); - lastPodmanStates = containers; - identityContainerIds = new Set( - containers - .map((container) => container.Id || container.ID || container.IdHex || "") - .filter((id) => id && !baselineContainers.has(id)) - ); - maximumNewContainerCount = Math.max( - maximumNewContainerCount, - identityContainerIds.size - ); - if (identityContainerIds.size >= 2) break; - await delay(100); - } - const identityTaskSnapshot = - identityContainerIds.size >= 2 - ? null - : runJson( - clusterflux, - ["task", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert( - identityContainerIds.size >= 2, - `identity entry did not start two same-definition Podman containers; max=${maximumNewContainerCount} tasks=${JSON.stringify( - identityTaskSnapshot - )} podman=${JSON.stringify(lastPodmanStates)}` - ); - - const pause = client.send("pause", { threadId: mainThread.id }); - await client.response(pause, "pause"); - const paused = await client.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "pause", - 30_000 - ); - assert.strictEqual(typeof paused.body.allThreadsStopped, "boolean"); - - const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]); - const clusterfluxPausedContainers = podmanStates.filter((container) => { - const id = container.Id || container.ID || container.IdHex || ""; - const state = String(container.State || container.Status || "").toLowerCase(); - return identityContainerIds.has(id) && state.includes("paused"); - }); - assert( - clusterfluxPausedContainers.length >= 2, - `expected two newly paused Clusterflux containers: ${JSON.stringify( - podmanStates - )}` - ); - - const threadsRequest = client.send("threads"); - const threads = (await client.response(threadsRequest, "threads")).body - .threads; - const identityThreads = threads.filter((thread) => - /identity probe/i.test(thread.name) - ); - assert.strictEqual( - identityThreads.length, - 2, - `expected two same-definition DAP threads: ${JSON.stringify(threads)}` - ); - assert.notStrictEqual(identityThreads[0].id, identityThreads[1].id); - - const threadEvidence = []; - for (const threadRecord of identityThreads) { - const stackRequest = client.send("stackTrace", { - threadId: threadRecord.id, - startFrame: 0, - levels: 1, - }); - const frames = (await client.response(stackRequest, "stackTrace")).body - .stackFrames; - assert.strictEqual(frames.length, 1); - const scopesRequest = client.send("scopes", { frameId: frames[0].id }); - const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; - const argsScope = scopes.find( - (scopeRecord) => scopeRecord.name === "Task Args and Handles" - ); - const runtimeScope = scopes.find( - (scopeRecord) => scopeRecord.name === "Clusterflux Runtime" - ); - assert(argsScope && runtimeScope); - const args = await dapVariables(client, argsScope.variablesReference); - const runtime = await dapVariables(client, runtimeScope.variablesReference); - const runtimeValue = (name) => - runtime.find((variable) => variable.name === name)?.value; - assert.strictEqual(runtimeValue("state"), "Frozen"); - threadEvidence.push({ - thread_id: threadRecord.id, - task_instance: runtimeValue("virtual_thread"), - attempt_id: runtimeValue("task_attempt_id"), - arguments: args.map((variable) => ({ - name: variable.name, - value: variable.value, - })), - }); - } - assert.notStrictEqual( - threadEvidence[0].task_instance, - threadEvidence[1].task_instance - ); - assert.notStrictEqual(threadEvidence[0].attempt_id, threadEvidence[1].attempt_id); - const argumentText = threadEvidence - .flatMap((record) => record.arguments.map((argument) => argument.value)) - .join(" "); - assert.match(argumentText, /slow-first/); - assert.match(argumentText, /fast-second/); - - const continued = client.send("continue", { - threadId: paused.body.threadId, - }); - await client.response(continued, "continue"); - await client.waitFor( - (message) => message.type === "event" && message.event === "terminated", - 5 * 60 * 1000 - ); - - const taskList = runJson( - clusterflux, - ["task", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - const identityEvents = rawTaskEvents(taskList).filter( - (event) => - event.task_definition === "identity_probe" && - event.terminal_state === "completed" - ); - assert.strictEqual(identityEvents.length, 2, JSON.stringify(identityEvents)); - assert.notStrictEqual(identityEvents[0].task, identityEvents[1].task); - assert.notStrictEqual(identityEvents[0].attempt_id, identityEvents[1].attempt_id); - assert.match(JSON.stringify(identityEvents[0].result), /fast-second/); - assert.match(JSON.stringify(identityEvents[1].result), /slow-first/); - - const released = await waitForCli( - "same-definition identity process release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - await client.close(); - disconnected = true; - return { - duration_ms: Date.now() - scenarioStartedAt, - process: processId, - launch_without_breakpoint: { - request: { - entry: "identity", - runtime_backend: "live-services", - configured_breakpoints: [], - }, - expected: "launch commits and reports a real coordinator-main thread", - observed: { - thread_id: mainThread.id, - thread_name: mainThread.name, - }, - duration_ms: launchWithoutBreakpointMs, - }, - post_main_launched_observation_recovery: { - request: { - entry: "identity", - fault_injections: [ - "initial task snapshot observation failure", - "initial process-status observation failure", - ], - }, - expected: - "main_launched is committed, no rollback or false stop occurs, and the observer reconnects until threads appear", - observed: { - diagnostics: postCommitDiagnostics, - main_thread: mainThread, - premature_stop_or_termination_events: - prematurePostCommitEvents.length, - }, - duration_ms: launchWithoutBreakpointMs, - }, - same_definition_identity: { - expected: - "two concurrent instances retain distinct thread, argument, attempt, output, freeze, and terminal identities", - observed: { - thread_evidence: threadEvidence, - completion_order: identityEvents.map((event) => ({ - task_instance: event.task, - attempt_id: event.attempt_id, - result: event.result, - })), - paused_container_count: clusterfluxPausedContainers.length, - terminal_state: released.state, - }, - duration_ms: Date.now() - scenarioStartedAt, - }, - thread_evidence: threadEvidence, - completion_order: identityEvents.map((event) => ({ - task_instance: event.task, - attempt_id: event.attempt_id, - result: event.result, - })), - podman_paused_container_ids: clusterfluxPausedContainers.map( - (container) => container.Id || container.ID || container.IdHex - ), - dap_all_threads_stopped: paused.body.allThreadsStopped, - terminal_state: released.state, - }; - } finally { - if (!disconnected) { - await client.close().catch(() => { - if (client.child.exitCode === null) client.child.kill("SIGKILL"); - }); - } - } -} - -async function runLiveDapEditRestart({ - clusterfluxDap, - clusterflux, - projectDir, - scope, - worker, -}) { - const scenarioStartedAt = Date.now(); - const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs")); - const originalSource = fs.readFileSync(sourcePath, "utf8"); - const sourceLines = originalSource.split(/\r?\n/); - const lineFor = (needle) => { - const line = sourceLines.findIndex((candidate) => candidate.includes(needle)) + 1; - assert(line > 0, `flagship source omitted ${needle}`); - return line; - }; - const mainLine = lineFor("pub async fn restart_main()"); - const taskLine = lineFor("fn task_trap("); - const client = new DapClient({ - cwd: projectDir, - command: clusterfluxDap, - args: [], - env: { - ...process.env, - CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1", - CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE: "1", - CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE: "1", - CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE: "1", - CLUSTERFLUX_TEST_DAP_OBSERVER_DEBUG_EPOCH_WAIT_FAILURE: "1", - CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION: "1", - CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS: "750", - CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION: "3", - }, - }); - let sourceRestored = false; - let disconnected = false; - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await client.response(initialize, "initialize"); - - const launch = client.send("launch", { - entry: "restart", - project: projectDir, - runtimeBackend: "live-services", - coordinatorEndpoint: serviceEndpoint, - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const configurationStartedAt = Date.now(); - const configurationDone = client.send("configurationDone"); - const configurationResponse = await client.response( - configurationDone, - "configurationDone" - ); - const configurationResponseMs = Date.now() - configurationStartedAt; - assert( - configurationResponseMs < 2_000, - `configurationDone blocked for ${configurationResponseMs} ms` - ); - - const threadDeadline = Date.now() + 5 * 60 * 1000; - let runningThreads = []; - let threadObservationResponse; - while (Date.now() < threadDeadline) { - const threadsRequest = client.send("threads"); - threadObservationResponse = await client.response(threadsRequest, "threads"); - runningThreads = threadObservationResponse.body.threads; - if (runningThreads.length > 0) break; - await delay(100); - } - assert(runningThreads.length > 0, "asynchronous live DAP launch reported no threads"); - const initialObserverFaults = []; - for (const expected of [ - { - fault: "task snapshot read transport failure", - diagnostic: "runtime snapshot observation failed", - }, - { - fault: "process-status read transport failure", - diagnostic: "runtime process observation failed", - }, - { - fault: "breakpoint inspection and fallback event transport failures", - diagnostic: "runtime breakpoint and fallback event observation failed", - }, - ]) { - const message = await client.waitFor( - (candidate) => - candidate.seq > configurationResponse.seq && - candidate.type === "event" && - candidate.event === "output" && - String(candidate.body?.output || "").includes(expected.diagnostic), - 120_000 - ); - initialObserverFaults.push({ - ...expected, - expected: "reconnect and eventually report live threads without a stop", - observed: String(message.body.output).trim(), - }); - } - const recoveredThreadsRequest = client.send("threads"); - const recoveredThreadsResponse = await client.response( - recoveredThreadsRequest, - "threads" - ); - assert( - recoveredThreadsResponse.body.threads.length > 0, - "observer faults removed threads after recovery" - ); - const prematureLaunchEvents = client.messages.filter( - (message) => - message.seq > configurationResponse.seq && - message.seq < recoveredThreadsResponse.seq && - message.type === "event" && - ["stopped", "terminated"].includes(message.event) - ); - assert.deepStrictEqual( - prematureLaunchEvents, - [], - "post-launch observer recovery fabricated a stop or terminated the launch" - ); - const postLaunchObservationRecoveryMs = - Date.now() - configurationStartedAt; - const pauseStartedAt = Date.now(); - const pause = client.send("pause", { threadId: runningThreads[0].id }); - await client.response(pause, "pause"); - const pauseResponseMs = Date.now() - pauseStartedAt; - assert(pauseResponseMs < 2_000, `pause blocked for ${pauseResponseMs} ms`); - const mainStop = await client.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "pause" - ); - assert.strictEqual(mainStop.body.allThreadsStopped, true); - - const inspectStartedAt = Date.now(); - const commandStatus = client.send("evaluate", { - expression: "command_status", - context: "watch", - }); - const inspected = await client.response(commandStatus, "evaluate"); - const inspectionResponseMs = Date.now() - inspectStartedAt; - assert( - inspectionResponseMs < 2_000, - `inspection blocked for ${inspectionResponseMs} ms` - ); - assert.match(inspected.body.result, /debug epoch|frozen/i); - - const breakpointStartedAt = Date.now(); - const staleRevisionRequest = client.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: mainLine }], - }); - const newestRevisionRequest = client.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: taskLine }], - }); - const staleRevisionResponse = await client.response( - staleRevisionRequest, - "setBreakpoints" - ); - const newestRevisionResponse = await client.response( - newestRevisionRequest, - "setBreakpoints" - ); - const breakpointResponseMs = Date.now() - breakpointStartedAt; - assert( - breakpointResponseMs < 2_000, - `setBreakpoints blocked for ${breakpointResponseMs} ms` - ); - assert.deepStrictEqual( - staleRevisionResponse.body.breakpoints.map( - (breakpoint) => breakpoint.verified - ), - [false] - ); - assert.deepStrictEqual( - newestRevisionResponse.body.breakpoints.map( - (breakpoint) => breakpoint.verified - ), - [false] - ); - assert.deepStrictEqual( - newestRevisionResponse.body.breakpoints.map( - (breakpoint) => breakpoint.message - ), - ["Pending coordinator breakpoint installation"] - ); - const newestRevisionInstalled = await client.waitFor( - (message) => - message.seq > newestRevisionResponse.seq && - message.type === "event" && - message.event === "breakpoint" && - message.body.reason === "changed" && - message.body.breakpoint?.verified === true && - message.body.breakpoint?.line === taskLine, - 30_000 - ); - await delay(1_000); - const staleRevisionEvents = client.messages.filter( - (message) => - message.seq > newestRevisionResponse.seq && - message.type === "event" && - message.event === "breakpoint" && - message.body.reason === "changed" && - message.body.breakpoint?.line === mainLine - ); - assert.deepStrictEqual( - staleRevisionEvents, - [], - "stale breakpoint revision completion overwrote the newest set" - ); - - const rejectedRevisionRequest = client.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: taskLine }], - }); - const rejectedRevisionResponse = await client.response( - rejectedRevisionRequest, - "setBreakpoints" - ); - const rejectedBreakpoint = await client.waitFor( - (message) => - message.seq > rejectedRevisionResponse.seq && - message.type === "event" && - message.event === "breakpoint" && - message.body.reason === "changed" && - message.body.breakpoint?.verified === false && - message.body.breakpoint?.line === taskLine && - /coordinator breakpoint installation failed/i.test( - message.body.breakpoint?.message || "" - ), - 30_000 - ); - assert.strictEqual( - rejectedBreakpoint.body.breakpoint.id, - rejectedRevisionResponse.body.breakpoints[0].id - ); - const retryBreakpoints = client.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: taskLine }], - }); - const retryBreakpointResponse = await client.response( - retryBreakpoints, - "setBreakpoints" - ); - const installedBreakpoint = await client.waitFor( - (message) => - message.seq > retryBreakpointResponse.seq && - message.type === "event" && - message.event === "breakpoint" && - message.body.reason === "changed" && - message.body.breakpoint?.verified === true && - message.body.breakpoint?.line === taskLine, - 30_000 - ); - assert.strictEqual( - installedBreakpoint.body.breakpoint.id, - retryBreakpointResponse.body.breakpoints[0].id - ); - const breakpointInstallationMs = Date.now() - breakpointStartedAt; - - const continueStartedAt = Date.now(); - const mainContinue = client.send("continue", { - threadId: mainStop.body.threadId, - }); - const mainContinueResponse = await client.response(mainContinue, "continue"); - const continueResponseMs = Date.now() - continueStartedAt; - assert( - continueResponseMs < 2_000, - `continue blocked for ${continueResponseMs} ms` - ); - const taskStop = await client.waitFor( - (message) => - message.seq > mainStop.seq && - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint" - ); - const realBreakpointMs = Date.now() - breakpointStartedAt; - assert.strictEqual(taskStop.body.allThreadsStopped, true); - const observerReconnect = await client.waitFor( - (message) => - message.seq > mainContinueResponse.seq && - message.type === "event" && - message.event === "output" && - String(message.body?.output || "").includes( - "injected one transient connection loss" - ), - 30_000 - ); - assert( - observerReconnect.seq < taskStop.seq, - "observer reconnection must complete before the real breakpoint stop" - ); - assert.strictEqual( - client.messages.filter( - (message) => - message.seq > mainStop.seq && - message.seq < taskStop.seq && - message.type === "event" && - message.event === "stopped" - ).length, - 0, - "observer reconnection fabricated a stopped target before the real breakpoint" - ); - assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId); - const reconnectDiagnostics = [ - "runtime observer injected one transient connection loss", - "runtime snapshot observation failed", - "runtime process observation failed", - "runtime breakpoint and fallback event observation failed", - "runtime Debug Epoch observation failed", - ].map((diagnostic) => { - const message = client.messages.find( - (candidate) => - candidate.seq > mainContinueResponse.seq && - candidate.seq < taskStop.seq && - candidate.type === "event" && - candidate.event === "output" && - String(candidate.body?.output || "").includes(diagnostic) - ); - assert( - message, - `observer reconnection evidence omitted diagnostic: ${diagnostic}` - ); - return String(message.body.output).trim(); - }); - - const stackTrace = client.send("stackTrace", { - threadId: taskStop.body.threadId, - startFrame: 0, - levels: 1, - }); - const taskFrames = (await client.response(stackTrace, "stackTrace")).body - .stackFrames; - assert.strictEqual(taskFrames[0].line, taskLine); - - const scopesRequest = client.send("scopes", { frameId: taskFrames[0].id }); - const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; - const argsScope = scopes.find( - (candidate) => candidate.name === "Task Args and Handles" - ); - const runtimeScope = scopes.find( - (candidate) => candidate.name === "Clusterflux Runtime" - ); - assert(argsScope && runtimeScope); - const taskArgs = await dapVariables(client, argsScope.variablesReference); - assert( - taskArgs.some( - (variable) => - variable.name === "arg_0" && String(variable.value).includes("0") - ) - ); - const runtime = await dapVariables(client, runtimeScope.variablesReference); - assert( - runtime.some( - (variable) => - variable.name === "runtime_backend" && variable.value === "LiveServices" - ) - ); - assert( - runtime.some( - (variable) => variable.name === "state" && variable.value === "Frozen" - ) - ); - const dapProcess = runtime.find( - (variable) => variable.name === "virtual_process_id" - )?.value; - assert.match(dapProcess || "", /^vp-[0-9a-f]{12}$/); - - const taskContinue = client.send("continue", { threadId: taskStop.body.threadId }); - await client.response(taskContinue, "continue"); - const failedStop = await client.waitFor( - (message) => - message.seq > taskStop.seq && - message.type === "event" && - message.event === "stopped" && - message.body.reason === "exception", - 5 * 60 * 1000 - ); - assert.strictEqual(failedStop.body.allThreadsStopped, false); - - const beforeEdit = runJson( - clusterflux, - ["task", "list", ...scope, "--process", dapProcess], - { cwd: projectDir } - ); - const originalTaskEvent = rawTaskEvents(beforeEdit).find( - (event) => - event.task_definition === "task_trap" && - event.terminal_state === "failed" - ); - assert(originalTaskEvent, "DAP restart entry omitted its original child event"); - assert(originalTaskEvent.attempt_id, "failed attempt omitted its identity"); - - const originalTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { - #[cfg(target_arch = "wasm32")] - core::arch::wasm32::unreachable(); - #[cfg(not(target_arch = "wasm32"))] - panic!("intentional task trap") -}`; - const replacementTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { - _input + 42 // strict hosted compatible restart probe -}`; - const editedSource = originalSource.replace(originalTaskBody, replacementTaskBody); - assert.notStrictEqual(editedSource, originalSource); - fs.writeFileSync(sourcePath, editedSource); - - const restartFrame = client.send("restartFrame", { - frameId: taskFrames[0].id, - }); - const restartResponse = await client.response(restartFrame, "restartFrame"); - const restartedStop = await client.waitFor( - (message) => - message.seq > restartResponse.seq && - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint" - ); - assert.strictEqual(restartedStop.body.allThreadsStopped, true); - const restartedContinue = client.send("continue", { - threadId: restartedStop.body.threadId, - }); - await client.response(restartedContinue, "continue"); - const restartedNodeReport = await worker.waitFor( - (value) => - value.node_status === "completed" && - value.virtual_thread === originalTaskEvent.task && - value.task_assignment_response?.task_spec?.task_definition === - "task_trap", - "edited task to execute on the live restarted node", - 5 * 60 * 1000 - ); - assert.strictEqual( - restartedNodeReport.node_status, - "completed", - `edited task restart failed on the live node: ${JSON.stringify(restartedNodeReport)}` - ); - assert.match(restartedNodeReport.stdout_tail, /"SmallJson":42/); - const restartedTask = restartedNodeReport.virtual_thread; - - const afterEdit = await waitForCli( - "edited live task event", - () => - runJson(clusterflux, ["task", "list", ...scope, "--process", dapProcess], { - cwd: projectDir, - }), - (tasks) => - rawTaskEvents(tasks).some( - (event) => - event.task === restartedTask && - event.terminal_state === "completed" && - JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) - ), - 5 * 60 * 1000 - ); - const restartedEvent = rawTaskEvents(afterEdit).find( - (event) => - event.task === restartedTask && - event.terminal_state === "completed" && - JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) - ); - assert(restartedEvent, "replacement attempt omitted its completed event"); - assert(restartedEvent.attempt_id, "replacement attempt omitted its identity"); - assert.notStrictEqual(restartedEvent.attempt_id, originalTaskEvent.attempt_id); - await client.waitFor( - (message) => message.type === "event" && message.event === "terminated", - 5 * 60 * 1000 - ); - const disconnectStartedAt = Date.now(); - await client.close(); - disconnected = true; - const disconnectResponseMs = Date.now() - disconnectStartedAt; - assert( - disconnectResponseMs < 2_000, - `disconnect blocked for ${disconnectResponseMs} ms` - ); - fs.writeFileSync(sourcePath, originalSource); - sourceRestored = true; - - return { - duration_ms: Date.now() - scenarioStartedAt, - process: dapProcess, - main_source_line: mainLine, - task_breakpoint_line: taskLine, - launch_started_without_breakpoint: true, - configuration_response_ms: configurationResponseMs, - pause_response_ms: pauseResponseMs, - inspection_response_ms: inspectionResponseMs, - breakpoint_response_ms: breakpointResponseMs, - breakpoint_installation_ms: breakpointInstallationMs, - continue_response_ms: continueResponseMs, - disconnect_response_ms: disconnectResponseMs, - pause_all_threads_stopped: mainStop.body.allThreadsStopped, - task_all_threads_stopped: taskStop.body.allThreadsStopped, - original_task_instance: originalTaskEvent.task, - original_attempt: originalTaskEvent.attempt_id, - restarted_task_instance: restartedTask, - restarted_attempt: restartedEvent.attempt_id, - restarted_result: restartedEvent.result, - compatible_source_edit: "task_trap: trap -> input + 42", - original_arguments_preserved: true, - clean_vfs_boundary_used: true, - breakpoint_install_failure_reported: true, - real_breakpoint: { - request: { - source: sourcePath, - line: taskLine, - runtime_backend: "live-services", - }, - expected: - "coordinator installation is verified before a real all-stop breakpoint event", - observed: { - installed_line: installedBreakpoint.body.breakpoint.line, - stop_reason: taskStop.body.reason, - stopped_line: taskFrames[0].line, - all_threads_stopped: taskStop.body.allThreadsStopped, - }, - duration_ms: Math.max(1, realBreakpointMs), - }, - debug_interaction: { - request: "Pause, inspect command_status, Continue, and Disconnect", - expected: "each control remains responsive during observer recovery", - observed: { - pause_response_ms: pauseResponseMs, - inspect_response_ms: inspectionResponseMs, - continue_response_ms: continueResponseMs, - disconnect_response_ms: disconnectResponseMs, - pause_stop_reason: mainStop.body.reason, - }, - duration_ms: Math.max( - 1, - pauseResponseMs + - inspectionResponseMs + - continueResponseMs + - disconnectResponseMs - ), - }, - post_launch_observation_recovery: { - fault_injections: initialObserverFaults, - expected: - "main_launched remains committed; reconnect; threads appear; no rollback or fabricated stop", - observed: { - threads: runningThreads, - premature_stop_or_termination_events: prematureLaunchEvents.length, - terminal_state: "completed after replacement attempt", - }, - duration_ms: postLaunchObservationRecoveryMs, - }, - observer_reconnection: { - fault_injections: reconnectDiagnostics, - expected: - "all recoverable reads reconnect before the real breakpoint stop", - observed: { - real_stop_reason: taskStop.body.reason, - false_stops_before_real_stop: 0, - }, - duration_ms: Date.now() - continueStartedAt, - }, - breakpoint_revisions: { - expected: "revision 2 remains authoritative after delayed revision 1", - delayed_revision: 1, - authoritative_revision: 2, - authoritative_line: newestRevisionInstalled.body.breakpoint.line, - stale_completion_events: staleRevisionEvents.length, - failure_revision: 3, - recovery_revision: 4, - observed: "newest revision verified; stale completion ignored", - duration_ms: breakpointInstallationMs, - }, - observer_idle_request_rate_bound_per_second: 0.8, - observer_reconnected_without_false_stop: true, - observer_fallback_reconnected: true, - observer_debug_epoch_wait_reconnected: true, - }; - } finally { - if (!sourceRestored) fs.writeFileSync(sourcePath, originalSource); - if (!disconnected) { - await client.close().catch(() => { - if (client.child.exitCode === null) client.child.kill("SIGKILL"); - }); - } - } -} - -function copyProjectControlState(sourceRoot, targetRoot) { - const source = path.join(sourceRoot, ".clusterflux"); - const target = path.join(targetRoot, ".clusterflux"); - ensureDir(target); - for (const file of ["session.json", "project.json"]) { - const from = path.join(source, file); - assert(fs.existsSync(from), `missing shared project control state ${from}`); - fs.copyFileSync(from, path.join(target, file)); - } - fs.chmodSync(path.join(target, "session.json"), 0o600); - const sourceNodes = path.join(source, "nodes"); - assert(fs.existsSync(sourceNodes), "persisted node credential directory is missing"); - fs.cpSync(sourceNodes, path.join(target, "nodes"), { - recursive: true, - force: true, - }); -} - -async function runHostedHelloBuild({ - clusterflux, - projectDir, - scope, - outputFile, -}) { - const startedAt = Date.now(); - const runReport = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.status, "main_launched"); - const tasks = await waitForCli( - "hosted hello-build completion", - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (report) => { - const events = rawTaskEvents(report); - return events.some( - (event) => - event.task_definition === "compile" && - event.terminal_state === "completed" - ); - }, - 10 * 60 * 1000 - ); - const events = rawTaskEvents(tasks); - assert( - events.some( - (event) => - event.task_definition === "snapshot_current_project" && - event.terminal_state === "completed" - ) - ); - const released = await waitForCli( - "hosted hello-build process release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", runReport.process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - const artifacts = runJson( - clusterflux, - ["artifact", "list", ...scope, "--process", runReport.process], - { cwd: projectDir } - ); - const artifact = artifacts.artifacts.find( - (candidate) => - typeof candidate.digest === "string" && - /^sha256:[0-9a-f]{64}$/.test(candidate.digest) && - candidate.artifact.startsWith("hello-clusterflux-") - ); - assert(artifact, "hello-build did not retain its executable artifact"); - await waitForCli( - "hosted hello-build retaining node online", - () => runJson(clusterflux, ["node", "list", ...scope], { cwd: projectDir }), - (report) => - report.response?.descriptors?.some( - (descriptor) => - descriptor.online === true && - descriptor.artifact_locations?.includes(artifact.artifact) - ), - 120_000 - ); - const download = runJson( - clusterflux, - [ - "artifact", - "download", - ...scope, - artifact.artifact, - "--to", - outputFile, - ], - { cwd: projectDir } - ); - assert.strictEqual(download.local_download.verified_digest, artifact.digest); - assert.strictEqual(sha256(fs.readFileSync(outputFile)), artifact.digest); - fs.chmodSync(outputFile, 0o755); - const output = run(outputFile, []).trim(); - assert.strictEqual(output, "hello from a real Clusterflux build"); - return { - duration_ms: Date.now() - startedAt, - process: runReport.process, - terminal_state: released.state, - artifact: artifact.artifact, - digest: artifact.digest, - downloaded_bytes: download.local_download.bytes_written, - downloaded_sha256: sha256(fs.readFileSync(outputFile)), - executable_output: output, - }; -} - -async function runHostedRecoveryBuild({ - clusterfluxDap, - clusterflux, - projectDir, - scope, -}) { - const startedAt = Date.now(); - const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs")); - const original = fs.readFileSync(sourcePath, "utf8"); - const failing = '"exit 23".to_owned()'; - const replacement = - '"printf \'recovered\\n\' > /clusterflux/output/recovering.txt".to_owned()'; - assert(original.includes(failing)); - const client = new DapClient({ - cwd: projectDir, - command: clusterfluxDap, - env: { ...process.env, CLUSTERFLUX_DAP_TIMEOUT_MS: "120000" }, - }); - let restored = false; - let closed = false; - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await client.response(initialize, "initialize"); - const launch = client.send("launch", { - entry: "build", - project: projectDir, - runtimeBackend: "live-services", - coordinatorEndpoint: serviceEndpoint, - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - const configured = client.send("configurationDone"); - await client.response(configured, "configurationDone"); - const failedStop = await client.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "exception", - 10 * 60 * 1000 - ); - assert.strictEqual(failedStop.body.allThreadsStopped, false); - const threadRequest = client.send("threads"); - const threads = (await client.response(threadRequest, "threads")).body.threads; - const recoveringThread = threads.find((thread) => /build lane/.test(thread.name)); - assert(recoveringThread, "recovery-build failed lane is not visible in DAP"); - const processId = threads - .map((thread) => /vp-[0-9a-f]{12}/.exec(thread.name)?.[0]) - .find(Boolean); - assert(processId, "recovery-build DAP threads omitted the process identity"); - const before = runJson( - clusterflux, - ["task", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - const beforeEvents = rawTaskEvents(before); - const originalFailure = beforeEvents.find( - (event) => - event.task_definition === "build_lane" && - event.terminal_state === "failed" - ); - assert(originalFailure?.attempt_id); - assert( - beforeEvents.some( - (event) => - event.task_definition === "build_lane" && - event.task !== originalFailure.task && - event.terminal_state === "completed" - ) - ); - const waiting = runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert.strictEqual(waiting.live_process.main_state, "running"); - fs.writeFileSync(sourcePath, original.replace(failing, replacement)); - const restart = client.send("restartFrame", { - threadId: recoveringThread.id, - }); - await client.response(restart, "restartFrame"); - await client.waitFor( - (message) => message.type === "event" && message.event === "terminated", - 10 * 60 * 1000 - ); - assert( - client.messages.some( - (message) => - message.type === "event" && - message.event === "thread" && - message.body.reason === "exited" && - message.body.threadId === recoveringThread.id - ) - ); - const after = await waitForCli( - "hosted recovery replacement event", - () => - runJson( - clusterflux, - ["task", "list", ...scope, "--process", processId], - { cwd: projectDir } - ), - (report) => - rawTaskEvents(report).some( - (event) => - event.task === originalFailure.task && - event.terminal_state === "completed" - ), - 120_000 - ); - const replacementEvent = rawTaskEvents(after).find( - (event) => - event.task === originalFailure.task && - event.terminal_state === "completed" - ); - assert(replacementEvent?.attempt_id); - assert.notStrictEqual(replacementEvent.attempt_id, originalFailure.attempt_id); - assert( - rawTaskEvents(after).some( - (event) => - event.executor === "coordinator_main" && - event.terminal_state === "completed" - ), - "replacement result did not satisfy the original coordinator-main join" - ); - const artifacts = runJson( - clusterflux, - ["artifact", "list", ...scope, "--process", processId], - { cwd: projectDir } - ).artifacts; - assert(artifacts.some((artifact) => artifact.artifact.startsWith("stable.txt-"))); - assert( - artifacts.some((artifact) => artifact.artifact.startsWith("recovering.txt-")) - ); - const released = await waitForCli( - "hosted recovery process release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120_000 - ); - fs.writeFileSync(sourcePath, original); - restored = true; - await client.close(); - closed = true; - return { - duration_ms: Date.now() - startedAt, - process: processId, - logical_task: originalFailure.task, - original_attempt: originalFailure.attempt_id, - replacement_attempt: replacementEvent.attempt_id, - original_join_completed: true, - terminal_state: released.state, - }; - } finally { - if (!restored) fs.writeFileSync(sourcePath, original); - if (!closed) { - await client.close().catch(() => { - if (client.child.exitCode === null) client.child.kill("SIGKILL"); - }); - } - } -} - -async function main() { - requireEnabled(); - const manifest = readJson(manifestPath); - for (const asset of manifest.assets ?? []) { - asset.file = path.isAbsolute(asset.file) - ? asset.file - : path.resolve(path.dirname(manifestPath), asset.file); - } - assert.strictEqual(manifest.kind, "clusterflux-public-release"); - assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint); - assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443"); - const qualityGates = strictQualityGateEvidence(manifest); - - const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-happy-")); - const installDir = path.join(workRoot, "install"); - const checkout = path.join(workRoot, "public-repo"); - const downloadPath = path.join(workRoot, "release.tar"); - const extractedArtifact = path.join(workRoot, "artifact"); - ensureDir(installDir); - run("tar", ["-xzf", binaryArchive(manifest), "-C", installDir]); - const publicSource = stagePublicCheckout(manifest, checkout); - - const clusterflux = executable(installDir, "clusterflux"); - const clusterfluxNode = executable(installDir, "clusterflux-node"); - const clusterfluxDap = executable(installDir, "clusterflux-debug-dap"); - const projectDir = path.join(checkout, "tests/fixtures/runtime-conformance"); - const helloProjectDir = path.join(checkout, "examples/hello-build"); - const recoveryProjectDir = path.join(checkout, "examples/recovery-build"); - const suffix = String(Date.now()); - const workerNode = `worker-${suffix}`; - - const loginStartedAt = Date.now(); - let loginSession; - let receivedDefaultProject; - if (reuseSessionFile) { - const sourceSessionPath = path.resolve(reuseSessionFile); - const sourceProjectPath = path.join(path.dirname(sourceSessionPath), "project.json"); - const sourceSession = readJson(sourceSessionPath); - const sourceProject = readJson(sourceProjectPath); - assert.strictEqual(sourceSession.kind, "human"); - assert.strictEqual(sourceSession.coordinator, serviceEndpoint); - assert.strictEqual(sourceProject.coordinator, serviceEndpoint); - assert.strictEqual(sourceProject.tenant, sourceSession.tenant); - assert.strictEqual(sourceProject.project, sourceSession.project); - assert.strictEqual(sourceProject.user, sourceSession.user); - assert( - Number(sourceSession.expires_at) > Math.floor(Date.now() / 1000) + 300, - "reused CLI session expires too soon for the live journey" - ); - const destinationConfig = path.join(projectDir, ".clusterflux"); - ensureDir(destinationConfig); - fs.copyFileSync(sourceSessionPath, path.join(destinationConfig, "session.json")); - fs.copyFileSync(sourceProjectPath, path.join(destinationConfig, "project.json")); - fs.chmodSync(path.join(destinationConfig, "session.json"), 0o600); - fs.chmodSync(path.join(destinationConfig, "project.json"), 0o644); - loginSession = sourceSession; - receivedDefaultProject = true; - } else { - const login = runJson( - clusterflux, - ["login", "--browser", "--json"], - { - cwd: projectDir, - env: { ...process.env, CLUSTERFLUX_BROWSER_OPEN_COMMAND: browserOpenCommand }, - } - ); - assert.strictEqual(login.plan.coordinator, serviceEndpoint); - assert.strictEqual(login.boundary.scoped_cli_session_received, true); - assert.strictEqual(login.boundary.provider_tokens_exposed_to_cli, false); - assert.strictEqual(login.boundary.provider_tokens_sent_to_nodes, false); - loginSession = login.coordinator_response.session; - assert(loginSession, "hosted browser login omitted its scoped session"); - receivedDefaultProject = - typeof loginSession.project === "string" && loginSession.project.length > 0; - assert(receivedDefaultProject, "login did not return its server-owned project"); - } - const sessionSecret = - loginSession.cli_session_secret || loginSession.session_secret; - const loginDurationMs = Math.max(1, Date.now() - loginStartedAt); - assert(sessionSecret, "hosted browser login omitted the CLI session credential"); - const tenant = loginSession.tenant; - const project = loginSession.project; - const user = loginSession.user; - for (const [name, value] of Object.entries({ tenant, project, user })) { - assert.strictEqual(typeof value, "string", `hosted session omitted ${name}`); - assert.notStrictEqual(value, "", `hosted session returned an empty ${name}`); - } - const scope = [ - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--user", - user, - "--json", - ]; - - if (!reuseSessionFile) { - const projectInit = runJson( - clusterflux, - [ - "project", - "init", - ...scope, - "--new-project", - project, - "--name", - "CLI Happy Path", - "--yes", - ], - { cwd: projectDir } - ); - assert.strictEqual(projectInit.command, "project init"); - assert.strictEqual(projectInit.project_config_written, true); - assert.strictEqual(projectInit.coordinator_response.type, "project_created"); - } - - const projectList = runJson(clusterflux, ["project", "list", ...scope], { - cwd: projectDir, - }); - assert(projectList.project_count >= 1); - const projectSelect = runJson( - clusterflux, - ["project", "select", ...scope, project], - { cwd: projectDir } - ); - assert.strictEqual(projectSelect.command, "project select"); - - const inspection = runJson(clusterflux, ["inspect", "--project", ".", "--json"], { - cwd: projectDir, - }); - assert( - inspection.metadata.environments.some((environment) => environment.name === "linux") - ); - assert(Array.isArray(inspection.source_provider_statuses)); - assert(inspection.source_provider_manifest); - - const runReport = runJson( - clusterflux, - ["run", "build", "--project", ".", "--json"], - { cwd: projectDir } - ); - assert.strictEqual(runReport.command, "run"); - assert.strictEqual(runReport.status, "main_launched"); - assert.strictEqual(runReport.task_launch.type, "main_launched"); - const processId = runReport.process; - - const parkedStatus = await waitForCli( - "hosted coordinator main to park before node enrollment", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => - status.live_process?.main_state === "running" && - status.live_process?.main_wait_state === "waiting_for_node" && - status.live_process?.connected_nodes?.length === 0, - 120000 - ); - const launchAttemptOwnership = await runLaunchAttemptOwnershipGuards({ - clusterflux, - projectDir, - scope, - sessionSecret, - activeProcess: processId, - }); - - const nodeLifecycleStartedAt = Date.now(); - const grant = runJson(clusterflux, ["node", "enroll", ...scope], { - cwd: projectDir, - }); - assert.strictEqual(grant.command, "node enroll"); - const attach = runJson( - clusterflux, - [ - "node", - "attach", - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--node", - workerNode, - "--enrollment-grant", - grant.enrollment_grant.grant, - "--json", - ], - { cwd: projectDir } - ); - assert.strictEqual(attach.command, "node attach"); - assert.strictEqual(attach.boundary.used_enrollment_exchange, true); - - const credentialDir = path.join(projectDir, ".clusterflux", "nodes"); - const credentialFiles = fs - .readdirSync(credentialDir) - .filter((file) => file.endsWith(".json")); - assert.strictEqual(credentialFiles.length, 1, "expected one persisted node credential"); - const credentialPath = path.join(credentialDir, credentialFiles[0]); - const credentialBefore = fs.readFileSync(credentialPath); - const credentialDigest = sha256(credentialBefore); - if (process.platform !== "win32") { - assert.strictEqual(fs.statSync(credentialPath).mode & 0o777, 0o600); - } - - const workerArgsFor = (projectRoot) => [ - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--node", - workerNode, - "--worker", - "--project-root", - projectRoot, - "--assignment-poll-ms", - "500", - "--emit-ready", - ]; - const workerArgs = workerArgsFor(projectDir); - const spawnWorker = (projectRoot = projectDir) => - spawnJsonLines(clusterfluxNode, workerArgsFor(projectRoot), { - cwd: projectRoot, - env: { ...process.env }, - }); - - let worker = spawnWorker(); - try { - const firstReady = await worker.waitFor( - (value) => value.node_status === "ready", - "first persisted worker ready" - ); - assert.strictEqual(firstReady.node, workerNode); - const initiallyOnline = runJson( - clusterflux, - ["node", "status", ...scope, "--node", workerNode], - { cwd: projectDir } - ); - assert.strictEqual(nodeDescriptor(initiallyOnline, workerNode)?.online, true); - - const completedTasks = await waitForCli( - "real hosted flagship completion", - () => - runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { - cwd: projectDir, - }), - (tasks) => completedFlagship(rawTaskEvents(tasks)) - ); - const firstEvents = rawTaskEvents(completedTasks); - assert( - firstEvents - .filter((event) => event.executor === "node") - .every((event) => event.node === workerNode) - ); - assert( - firstEvents.some( - (event) => - event.task_definition === "package_release" && - event.terminal_state === "completed" && - contentAddressedArtifactId(event, "release.tar") - ) - ); - - const releasedFlagshipStatus = await waitForCli( - "completed flagship process to release its active slot", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120000 - ); - - const offlineStartedAt = Date.now(); - await stopChild(worker.child); - const observedOffline = await waitForCli( - "server-derived worker stale/offline transition", - () => - runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { - cwd: projectDir, - }), - (status) => nodeDescriptor(status, workerNode)?.online === false, - 120000 - ); - worker = spawnWorker(); - const secondReady = await worker.waitFor( - (value) => value.node_status === "ready", - "restarted persisted worker ready" - ); - assert.strictEqual(secondReady.node, workerNode); - assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); - - const nodeStatus = await waitForCli( - "server-derived worker online recovery", - () => - runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { - cwd: projectDir, - }), - (status) => nodeDescriptor(status, workerNode)?.online === true, - 30000 - ); - assert(JSON.stringify(nodeStatus.response).includes(workerNode)); - assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); - const nodeLivenessTransition = { - initial_online: nodeDescriptor(initiallyOnline, workerNode).online, - stale_offline: nodeDescriptor(observedOffline, workerNode).online, - recovered_online: nodeDescriptor(nodeStatus, workerNode).online, - offline_detection_ms: Date.now() - offlineStartedAt, - persisted_identity_reused: true, - duration_ms: Math.max(1, Date.now() - nodeLifecycleStartedAt), - }; - - const processStatus = runJson( - clusterflux, - ["process", "status", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert.strictEqual(processStatus.command, "process status"); - assert(processStatus.current_task_count >= firstEvents.length); - - const logs = runJson( - clusterflux, - ["logs", ...scope, "--process", processId], - { cwd: projectDir } - ); - assert(logs.log_entries.length >= 4); - - const artifacts = runJson( - clusterflux, - ["artifact", "list", ...scope, "--process", processId], - { cwd: projectDir } - ); - const releaseArtifact = artifacts.artifacts.find((artifact) => { - const digest = artifact.digest; - return ( - typeof digest === "string" && - /^sha256:[0-9a-f]{64}$/.test(digest) && - artifact.artifact === `release.tar-${digest.slice("sha256:".length)}` - ); - }); - assert(releaseArtifact, "hosted flagship did not publish release.tar"); - assert.strictEqual(releaseArtifact.state, "metadata_flushed"); - - const download = runJson( - clusterflux, - [ - "artifact", - "download", - ...scope, - releaseArtifact.artifact, - "--to", - downloadPath, - ], - { cwd: projectDir } - ); - assert.strictEqual(download.download_session.link_issued, true); - assert.strictEqual(download.local_download.local_bytes_written_by_cli, true); - assert.strictEqual(download.local_download.verified_digest, releaseArtifact.digest); - assert.strictEqual(sha256(fs.readFileSync(downloadPath)), releaseArtifact.digest); - - ensureDir(extractedArtifact); - const archiveEntries = run("tar", ["-tf", downloadPath]); - assert.match(archiveEntries, /(?:^|\n)hello-clusterflux(?:\n|$)/); - run("tar", ["-xf", downloadPath, "-C", extractedArtifact]); - const builtExecutable = path.join(extractedArtifact, "hello-clusterflux"); - assert(fs.existsSync(builtExecutable)); - const builtOutput = run(builtExecutable, []).trim(); - assert.strictEqual(builtOutput, "hello from a real Clusterflux build"); - - copyProjectControlState(projectDir, helloProjectDir); - copyProjectControlState(projectDir, recoveryProjectDir); - await stopChild(worker.child); - worker = spawnWorker(helloProjectDir); - await worker.waitFor( - (value) => value.node_status === "ready", - "hello-build worker ready" - ); - const helloBuild = await runHostedHelloBuild({ - clusterflux, - projectDir: helloProjectDir, - scope, - outputFile: path.join(workRoot, "hello-clusterflux"), - }); - await stopChild(worker.child); - worker = spawnWorker(recoveryProjectDir); - await worker.waitFor( - (value) => value.node_status === "ready", - "recovery-build worker ready" - ); - const recoveryBuild = await runHostedRecoveryBuild({ - clusterfluxDap, - clusterflux, - projectDir: recoveryProjectDir, - scope, - }); - await stopChild(worker.child); - worker = spawnWorker(); - await worker.waitFor( - (value) => value.node_status === "ready", - "runtime-conformance worker restored" - ); - - const dapEditRestart = await runLiveDapEditRestart({ - clusterfluxDap, - clusterflux, - projectDir, - scope, - worker, - }); - - const processCancellationStartedAt = Date.now(); - const restart = runJson( - clusterflux, - ["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(restart.restart_request.accepted, true); - const cancel = runJson( - clusterflux, - ["process", "cancel", ...scope, "--process", dapEditRestart.process, "--yes"], - { cwd: projectDir } - ); - assert.strictEqual(cancel.cancel_request.accepted, true); - const releasedDap = await waitForCli( - "cancelled DAP process slot release", - () => - runJson( - clusterflux, - ["process", "status", ...scope, "--process", dapEditRestart.process], - { cwd: projectDir } - ), - (status) => status.state === "not_active", - 120000 - ); - assert.strictEqual(releasedDap.state, "not_active"); - const processCancellationLifecycle = { - request: { - restart_process: dapEditRestart.process, - cancel_process: dapEditRestart.process, - }, - expected: - "explicit process cancellation terminates active participants and releases the slot", - observed: { - restart_accepted: restart.restart_request.accepted, - cancel_accepted: cancel.cancel_request.accepted, - terminal_state: releasedDap.state, - }, - duration_ms: Math.max(1, Date.now() - processCancellationStartedAt), - }; - - const identityWorkerNode = `identity-worker-${suffix}`; - const identityWorkerGrant = runJson( - clusterflux, - ["node", "enroll", ...scope], - { cwd: projectDir } - ); - runJson( - clusterflux, - [ - "node", - "attach", - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--node", - identityWorkerNode, - "--enrollment-grant", - identityWorkerGrant.enrollment_grant.grant, - "--json", - ], - { cwd: projectDir } - ); - const identityWorker = spawnJsonLines( - clusterfluxNode, - [ - "--coordinator", - serviceEndpoint, - "--tenant", - tenant, - "--project-id", - project, - "--node", - identityWorkerNode, - "--worker", - "--project-root", - projectDir, - "--assignment-poll-ms", - "500", - "--emit-ready", - ], - { cwd: projectDir, env: { ...process.env } } - ); - let sameDefinitionIdentity; - try { - const identityWorkerReady = await identityWorker.waitFor( - (value) => value.node_status === "ready", - "same-definition identity worker ready" - ); - assert.strictEqual(identityWorkerReady.node, identityWorkerNode); - await waitForCli( - "same-definition identity worker online", - () => - runJson( - clusterflux, - ["node", "status", ...scope, "--node", identityWorkerNode], - { cwd: projectDir } - ), - (status) => nodeDescriptor(status, identityWorkerNode)?.online === true, - 30_000 - ); - await stopChild(worker.child); - worker = spawnWorker(); - const refreshedPrimaryReady = await worker.waitFor( - (value) => value.node_status === "ready", - "same-definition primary worker capability refresh" - ); - assert.strictEqual(refreshedPrimaryReady.node, workerNode); - const primaryStatus = await waitForCli( - "same-definition primary worker online after refresh", - () => - runJson( - clusterflux, - ["node", "status", ...scope, "--node", workerNode], - { cwd: projectDir } - ), - (status) => nodeDescriptor(status, workerNode)?.online === true, - 30_000 - ); - const identityStatus = runJson( - clusterflux, - ["node", "status", ...scope, "--node", identityWorkerNode], - { cwd: projectDir } - ); - const primarySnapshots = - nodeDescriptor(primaryStatus, workerNode)?.source_snapshots || []; - const identitySnapshots = - nodeDescriptor(identityStatus, identityWorkerNode)?.source_snapshots || []; - assert( - primarySnapshots.some((snapshot) => identitySnapshots.includes(snapshot)), - `same-definition workers do not share a current source snapshot: ${JSON.stringify({ - primarySnapshots, - identitySnapshots, - })}` - ); - sameDefinitionIdentity = await runSameDefinitionDapIdentity({ - clusterfluxDap, - clusterflux, - projectDir, - scope, - }); - } finally { - await stopChild(identityWorker.child); - runJson( - clusterflux, - ["node", "revoke", ...scope, "--node", identityWorkerNode, "--yes"], - { cwd: projectDir } - ); - } - const repeatedParkWake = await runRepeatedParkWakeProof({ - clusterflux, - projectDir, - scope, - }); - const longJoin = await runLongJoinProof({ - clusterflux, - projectDir, - scope, - }); - - await stopChild(worker.child); - const tenantIsolationResult = await runSecondTenantIsolation({ - clusterflux, - clusterfluxNode, - firstScope: scope, - firstHelloBuild: helloBuild, - firstHelloProjectDir: helloProjectDir, - workRoot, - firstSessionSecret: sessionSecret, - firstTenant: tenant, - firstProject: project, - firstProcess: dapEditRestart.process, - firstNode: workerNode, - firstArtifact: releaseArtifact.artifact, - manifest, - }); - const tenantIsolation = tenantIsolationResult.evidence; - const scopedCollisionRuntime = tenantIsolationResult.runtime; - worker = spawnWorker(); - const collisionRecoveryReady = await worker.waitFor( - (value) => value.node_status === "ready", - "primary worker restored after two-tenant collision proof" - ); - assert.strictEqual(collisionRecoveryReady.node, workerNode); - - const securityNode = await prepareLiveNodeCredentialSecurity({ - clusterflux, - projectDir, - scope, - tenant, - project, - suffix, - bundle: { - digest: runReport.bundle_digest, - moduleBase64: fs - .readFileSync( - path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) - ) - .toString("base64"), - entryExport: runReport.entry_export, - entryStableId: runReport.entry_stable_id, - }, - }); - const droppedConnectionRollback = await runDroppedConnectionRollback({ - clusterflux, - projectDir, - scope, - }); - const bandwidthPreallocation = await runLiveBandwidthPreallocation({ - sessionSecret, - artifact: releaseArtifact.artifact, - maxBytes: download.local_download.bytes_written, - }); - await stopChild(worker.child); - const relayEmergencyDisableResult = await runRelayEmergencyDisable({ - clusterflux, - clusterfluxNode, - projectDir, - scope, - workerArgs, - sessionSecret, - maxBytes: download.local_download.bytes_written, - }); - worker = relayEmergencyDisableResult.worker; - const relayEmergencyDisable = relayEmergencyDisableResult.evidence; - const agentCredentialSecurity = await runLiveAgentCredentialSecurity({ - clusterfluxDap, - clusterflux, - projectDir, - scope, - sessionSecret, - tenant, - project, - suffix, - bundle: { - digest: runReport.bundle_digest, - moduleBase64: fs - .readFileSync( - path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) - ) - .toString("base64"), - entryExport: runReport.entry_export, - entryStableId: runReport.entry_stable_id, - }, - securityNode, - workerRuntime: { - node: workerNode, - child: worker.child, - worker, - identity: nodeIdentityFromPrivateKey( - readNodeCredential(projectDir, workerNode).credential.private_key - ), - }, - }); - const mainBeforeChildLifecycle = - agentCredentialSecurity.main_before_child_lifecycle; - - await stopChild(worker.child); - const liveSoak = await runLiveSoak({ - clusterflux, - projectDir, - scope, - securityNode, - }); - const malformedIdentifiers = await runMalformedIdentifierSuite({ - clusterflux, - projectDir, - scope, - sessionSecret, - tenant, - project, - user, - suffix, - securityNode, - bundle: { - digest: runReport.bundle_digest, - moduleBase64: fs - .readFileSync( - path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) - ) - .toString("base64"), - entryExport: runReport.entry_export, - entryStableId: runReport.entry_stable_id, - }, - releaseArtifact, - }); - const signedHostileArtifactPath = await runLiveSignedHostileArtifactPath({ - clusterflux, - projectDir, - scope, - tenant, - project, - suffix, - securityNode, - }); - const revokedNodeCredential = await revokeSecurityNode({ - clusterflux, - projectDir, - scope, - securityNode, - }); - const quotaPreallocation = await runLiveQuotaPreallocation({ - sessionSecret, - suffix, - }); - const hostedLoginIsolation = await runHostedLoginIsolationBeforeRestart(); - const serviceRestart = await restartHostedServiceAndResume({ - clusterflux, - clusterfluxNode, - projectDir, - scope, - workerArgs, - workerNode, - credentialPath, - credentialDigest, - scopedCollisionRuntime, - }); - if (serviceRestart.worker) worker = serviceRestart.worker; - const scopedNodeRevocationAfterRestart = scopedCollisionRuntime - ? await finishScopedNodeCollisionAfterRestart({ - clusterflux, - projectDir, - scope, - workerNode, - scopedCollisionRuntime, - }) - : { - executed: false, - reason: "second tenant runtime session was not supplied", - duration_ms: 1, - }; - const relayAfterRestart = relayDurableState(); - assert( - relayAfterRestart.ingress_used >= - bandwidthPreallocation.durable_after_release.ingress_used - ); - assert( - relayAfterRestart.egress_used >= - bandwidthPreallocation.durable_after_release.egress_used - ); - assert( - relayAfterRestart.abandoned_or_failed_used >= - bandwidthPreallocation.durable_after_release.abandoned_or_failed_used - ); - bandwidthPreallocation.durable_after_restart = relayAfterRestart; - bandwidthPreallocation.restart_persistence = true; - bandwidthPreallocation.duration_ms = - Date.now() - bandwidthPreallocation.scenario_started_at_ms; - delete bandwidthPreallocation.scenario_started_at_ms; - await finishHostedLoginIsolationAfterRestart(hostedLoginIsolation); - const userCredentialSecurity = await finishUserCredentialSecurity({ - clusterflux, - projectDir, - scope, - sessionSecret, - }); - const configProvenance = configurationProvenance(); - const proxyConfigProvenance = proxyConfigurationProvenance(); - const concurrentCompileTasks = [ - ...new Set( - firstEvents - .filter( - (event) => - event.task_definition === "compile_linux" && - event.terminal_state === "completed" - ) - .map((event) => event.task) - ), - ]; - assert(concurrentCompileTasks.length >= 2); - const extensionAsset = manifest.assets.find((asset) => - asset.name.endsWith(".vsix") - ); - const requirement = ({ id, passed, evidence, durationMs }) => { - const result = { - id, - passed: passed === true, - evidence, - }; - if (Number.isFinite(durationMs) && durationMs > 0) { - result.duration_ms = durationMs; - } else if (strictFullRelease) { - throw new Error( - `strict release scenario ${id} omitted a measured positive duration` - ); - } - return result; - }; - const qualityCheck = (id) => qualityGates?.checks?.[id]; - const nodeEnrollmentEvidence = { - request: "one-time enrollment followed by a grant-free worker restart", - expected: "the persisted node credential is reused unchanged", - observed: { - used_enrollment_exchange: attach.boundary.used_enrollment_exchange, - restarted_node: secondReady.node, - credential_digest: credentialDigest, - liveness: nodeLivenessTransition, - service_restart_executed: serviceRestart.executed, - }, - }; - const loginEvidence = { - request: reuseSessionFile - ? "reuse a still-valid scoped browser-login session" - : "perform browser OIDC login", - expected: "server-owned default project persists and can be selected", - observed: { - fresh_login: !reuseSessionFile, - tenant, - project, - user, - received_default_project: receivedDefaultProject, - project_select_command: projectSelect.command, - }, - }; - const credentialSecurityEvidence = { - expected: - "forged, replayed, expired, body-modified, and revoked user/Node/Agent credentials are denied", - observed: { - user: userCredentialSecurity, - node: { - initial: securityNode.evidence, - revoked: revokedNodeCredential.denied, - }, - agent: { - replay: agentCredentialSecurity.replay, - expired: agentCredentialSecurity.expired, - forged: agentCredentialSecurity.forged, - body_modified: agentCredentialSecurity.body_modified, - revoked: agentCredentialSecurity.revoked, - }, - }, - }; - const credentialSecurityDurationMs = - userCredentialSecurity.duration_ms + - securityNode.duration_ms + - revokedNodeCredential.duration_ms + - agentCredentialSecurity.duration_ms; - const relaySecurityEvidence = { - expected: - "limits are scoped independently and abandoned transfer bytes remain charged without a shared soft lockout", - observed: { - spawn_quota: quotaPreallocation, - relay_accounting: bandwidthPreallocation, - emergency_disable: relayEmergencyDisable, - }, - }; - const relaySecurityDurationMs = - quotaPreallocation.duration_ms + - bandwidthPreallocation.duration_ms + - relayEmergencyDisable.duration_ms; - const strictRequirementLedger = [ - requirement({ - id: "01_formatting", - passed: qualityCheck("formatting")?.status === "passed", - evidence: qualityCheck("formatting"), - durationMs: qualityCheck("formatting")?.duration_ms, - }), - requirement({ - id: "02_clippy_warnings_denied", - passed: qualityCheck("clippy_warnings_denied")?.status === "passed", - evidence: qualityCheck("clippy_warnings_denied"), - durationMs: qualityCheck("clippy_warnings_denied")?.duration_ms, - }), - requirement({ - id: "03_public_workspace_tests", - passed: qualityCheck("public_workspace_tests")?.status === "passed", - evidence: qualityCheck("public_workspace_tests"), - durationMs: qualityCheck("public_workspace_tests")?.duration_ms, - }), - requirement({ - id: "04_private_hosted_policy_locked_tests", - passed: - qualityCheck("private_hosted_policy_locked_tests")?.status === - "passed", - evidence: qualityCheck("private_hosted_policy_locked_tests"), - durationMs: qualityCheck("private_hosted_policy_locked_tests") - ?.duration_ms, - }), - requirement({ - id: "05_wasm_example_builds", - passed: qualityCheck("wasm_example_builds")?.status === "passed", - evidence: qualityCheck("wasm_example_builds"), - durationMs: qualityCheck("wasm_example_builds")?.duration_ms, - }), - requirement({ - id: "06_filtered_public_tree_build_and_tests", - passed: - qualityCheck("filtered_public_tree_build_and_tests")?.status === - "passed", - evidence: qualityCheck("filtered_public_tree_build_and_tests"), - durationMs: qualityCheck("filtered_public_tree_build_and_tests") - ?.duration_ms, - }), - requirement({ - id: "07_final_vsix_checks", - passed: Boolean( - extensionAsset && - qualityCheck("vscode_extension_candidate")?.status === "passed" && - extensionAsset.sha256 === - manifest.release_candidate.extension_sha256 && - qualityCheck("vscode_extension_candidate")?.candidate_vsix - ?.sha256 === extensionAsset.sha256 - ), - evidence: qualityCheck("vscode_extension_candidate"), - durationMs: qualityCheck("vscode_extension_candidate")?.duration_ms, - }), - requirement({ - id: "08_browser_login_and_persistent_default_project", - passed: Boolean( - sessionSecret && - tenant && - project && - receivedDefaultProject && - projectSelect.command === "project select" - ), - evidence: loginEvidence, - durationMs: loginDurationMs, - }), - requirement({ - id: "09_one_time_node_enrollment_and_restart", - passed: - attach.boundary.used_enrollment_exchange === true && - secondReady.node === workerNode && - nodeLivenessTransition.persisted_identity_reused === true && - serviceRestart.executed === true, - evidence: nodeEnrollmentEvidence, - durationMs: nodeLivenessTransition.duration_ms, - }), - requirement({ - id: "10_real_hello_build_and_artifact_download", - passed: - helloBuild.executable_output === - "hello from a real Clusterflux build" && - helloBuild.downloaded_bytes > 0 && - helloBuild.downloaded_sha256 === helloBuild.digest, - evidence: helloBuild, - durationMs: helloBuild.duration_ms, - }), - requirement({ - id: "11_recovery_retry_and_original_join", - passed: - recoveryBuild.original_join_completed === true && - recoveryBuild.original_attempt !== - recoveryBuild.replacement_attempt && - recoveryBuild.terminal_state === "not_active", - evidence: recoveryBuild, - durationMs: recoveryBuild.duration_ms, - }), - requirement({ - id: "12_long_lived_coordinator_main", - passed: - longJoin.duration_seconds > 120 && - longJoin.terminal_state === "completed" && - longJoin.process_state === "not_active", - evidence: longJoin, - durationMs: longJoin.duration_ms, - }), - requirement({ - id: "13_main_completes_before_child_lifecycle", - passed: - mainBeforeChildLifecycle.observed.active_after_main === true && - mainBeforeChildLifecycle.observed.debug_state_after_main === - "debug_breakpoints" && - mainBeforeChildLifecycle.observed.child_terminal_state === - "completed" && - mainBeforeChildLifecycle.observed.final_process_state === - "not_active" && - mainBeforeChildLifecycle.observed.subsequent_run_status === - "main_launched" && - qualityCheck("process_lifecycle_regressions")?.status === "passed", - evidence: { - live: mainBeforeChildLifecycle, - compiled_failure_and_cancellation: - qualityCheck("process_lifecycle_regressions"), - }, - durationMs: - mainBeforeChildLifecycle.duration_ms + - (qualityCheck("process_lifecycle_regressions")?.duration_ms || 0), - }), - requirement({ - id: "14_dap_launch_without_breakpoint", - passed: - sameDefinitionIdentity.launch_without_breakpoint.observed.thread_id > - 0 && - sameDefinitionIdentity.terminal_state === "not_active", - evidence: sameDefinitionIdentity.launch_without_breakpoint, - durationMs: - sameDefinitionIdentity.launch_without_breakpoint.duration_ms, - }), - requirement({ - id: "15_real_breakpoint_installation_and_hit", - passed: - dapEditRestart.real_breakpoint.observed.stop_reason === - "breakpoint" && - dapEditRestart.real_breakpoint.observed.stopped_line === - dapEditRestart.real_breakpoint.request.line && - dapEditRestart.real_breakpoint.observed.all_threads_stopped === true, - evidence: dapEditRestart.real_breakpoint, - durationMs: dapEditRestart.real_breakpoint.duration_ms, - }), - requirement({ - id: "16_attach_with_preconfigured_breakpoint", - passed: - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .observed.initially_verified === false && - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .observed.installation_event_verified === true && - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .observed.stop_reason === "breakpoint" && - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .observed.stopped_line === - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .request.line, - evidence: - agentCredentialSecurity.attach_with_preconfigured_breakpoint, - durationMs: - agentCredentialSecurity.attach_with_preconfigured_breakpoint - .duration_ms, - }), - requirement({ - id: "17_breakpoint_revision_ordering", - passed: - dapEditRestart.breakpoint_revisions.authoritative_revision > - dapEditRestart.breakpoint_revisions.delayed_revision && - dapEditRestart.breakpoint_revisions.stale_completion_events === 0, - evidence: dapEditRestart.breakpoint_revisions, - durationMs: dapEditRestart.breakpoint_revisions.duration_ms, - }), - requirement({ - id: "18_continue_pause_inspect_disconnect", - passed: - dapEditRestart.debug_interaction.observed.pause_response_ms < - 2_000 && - dapEditRestart.debug_interaction.observed.inspect_response_ms < - 2_000 && - dapEditRestart.debug_interaction.observed.continue_response_ms < - 2_000 && - dapEditRestart.debug_interaction.observed.disconnect_response_ms < - 2_000, - evidence: dapEditRestart.debug_interaction, - durationMs: dapEditRestart.debug_interaction.duration_ms, - }), - requirement({ - id: "19_distinct_same_definition_instances", - passed: - sameDefinitionIdentity.thread_evidence.length === 2 && - sameDefinitionIdentity.completion_order.length === 2 && - sameDefinitionIdentity.thread_evidence[0].task_instance !== - sameDefinitionIdentity.thread_evidence[1].task_instance && - sameDefinitionIdentity.thread_evidence[0].attempt_id !== - sameDefinitionIdentity.thread_evidence[1].attempt_id && - concurrentCompileTasks.length >= 2, - evidence: sameDefinitionIdentity.same_definition_identity, - durationMs: - sameDefinitionIdentity.same_definition_identity.duration_ms, - }), - requirement({ - id: "20_real_podman_partial_debug_epoch", - passed: - agentCredentialSecurity.partial_freeze.partially_frozen === true && - agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === - false && - agentCredentialSecurity.partial_freeze.podman_paused_container_ids - .length > 0 && - agentCredentialSecurity.partial_freeze.resumed_participants.length > - 0, - evidence: agentCredentialSecurity.partial_freeze, - durationMs: agentCredentialSecurity.partial_freeze.elapsed_ms, - }), - requirement({ - id: "21_post_main_launched_observation_recovery", - passed: - sameDefinitionIdentity.post_main_launched_observation_recovery - .observed.diagnostics.length === 2 && - sameDefinitionIdentity.post_main_launched_observation_recovery - .observed - .premature_stop_or_termination_events === 0 && - sameDefinitionIdentity.post_main_launched_observation_recovery - .observed.main_thread.id > 0, - evidence: - sameDefinitionIdentity.post_main_launched_observation_recovery, - durationMs: - sameDefinitionIdentity.post_main_launched_observation_recovery - .duration_ms, - }), - requirement({ - id: "22_observer_reconnection", - passed: - dapEditRestart.observer_reconnection.fault_injections.length === 5 && - dapEditRestart.observer_reconnection.observed - .false_stops_before_real_stop === 0 && - dapEditRestart.observer_reconnection.observed.real_stop_reason === - "breakpoint", - evidence: dapEditRestart.observer_reconnection, - durationMs: dapEditRestart.observer_reconnection.duration_ms, - }), - requirement({ - id: "23_malformed_identifiers_preserve_service", - passed: - malformedIdentifiers.valid_after_every_rejection === true && - malformedIdentifiers.all_rejected_for_intended_reason === true && - malformedIdentifiers.valid_authenticated_action_after_suite === - true && - malformedIdentifiers.valid_signed_rendezvous_after_suite === true && - malformedIdentifiers.rejected_requests.every( - (entry) => - Number.isFinite(entry.duration_ms) && entry.duration_ms > 0 - ), - evidence: malformedIdentifiers, - durationMs: malformedIdentifiers.duration_ms, - }), - requirement({ - id: "24_cross_tenant_access_denied", - passed: - tenantIsolation.executed === true && - tenantIsolation.process_hidden === true && - tenantIsolation.node_hidden === true && - tenantIsolation.tasks_and_logs.denied === true && - tenantIsolation.debug.denied === true && - tenantIsolation.artifact_and_download.denied === true, - evidence: tenantIsolation, - durationMs: tenantIsolation.duration_ms, - }), - requirement({ - id: "25_forged_replayed_expired_revoked_credentials_denied", - passed: - Boolean( - userCredentialSecurity.expired && - userCredentialSecurity.forged && - userCredentialSecurity.revoked - ) && - securityNode.evidence.replay.denied === true && - securityNode.evidence.expired.denied === true && - securityNode.evidence.forged.denied === true && - revokedNodeCredential.denied.denied === true && - agentCredentialSecurity.revoked.denied === true, - evidence: credentialSecurityEvidence, - durationMs: credentialSecurityDurationMs, - }), - requirement({ - id: "26_relay_limits_and_abandoned_transfer_charging", - passed: - quotaPreallocation.denied_process_allocated === false && - quotaPreallocation.independent_scope - .unaffected_by_spawn_quota === true && - bandwidthPreallocation.bytes_served_before_denial > 0 && - bandwidthPreallocation.partial_abandon.ingress_delta > 0 && - bandwidthPreallocation.partial_abandon.egress_delta > 0 && - bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && - bandwidthPreallocation.partial_abandon.reservation_released === - true && - bandwidthPreallocation.restart_persistence === true && - relayEmergencyDisable.disabled.denied === true && - relayEmergencyDisable.runtime_drop_in_removed === true, - evidence: relaySecurityEvidence, - durationMs: relaySecurityDurationMs, - }), - requirement({ - id: "27_existing_process_rejection_preserves_process", - passed: launchAttemptOwnership.existing_process_survived === true, - evidence: launchAttemptOwnership, - durationMs: launchAttemptOwnership.duration_ms, - }), - requirement({ - id: "28_precommit_launch_failure_scoped_cleanup", - passed: - droppedConnectionRollback.cli_exit_status !== 0 && - droppedConnectionRollback.rollback === - "automatic CLI launch guard abort_process with matching launch_attempt" && - droppedConnectionRollback.terminal_state === "not_active", - evidence: droppedConnectionRollback, - durationMs: droppedConnectionRollback.duration_ms, - }), - requirement({ - id: "29_explicit_process_cancellation_cleanup", - passed: - processCancellationLifecycle.observed.restart_accepted === true && - processCancellationLifecycle.observed.cancel_accepted === true && - processCancellationLifecycle.observed.terminal_state === - "not_active", - evidence: processCancellationLifecycle, - durationMs: processCancellationLifecycle.duration_ms, - }), - requirement({ - id: "30_scoped_node_and_artifact_collision", - passed: - tenantIsolation.scoped_node_and_artifact_collision?.observed - ?.both_metadata_records_visible_to_owner === true && - tenantIsolation.scoped_node_and_artifact_collision?.observed - ?.first_download_survived_second_link_revocation === true && - tenantIsolation.scoped_node_and_artifact_collision?.observed - ?.same_artifact_id === helloBuild.artifact && - serviceRestart.scoped_node_collision_after_restart - ?.first_scope_online === true && - serviceRestart.scoped_node_collision_after_restart - ?.second_scope_online === true && - serviceRestart.scoped_node_collision_after_restart - ?.both_credentials_reused_without_grants === true && - scopedNodeRevocationAfterRestart.observed - ?.revoked_credential_denied === true && - scopedNodeRevocationAfterRestart.observed - ?.first_scope_node_online === true, - evidence: { - live_collision: - tenantIsolation.scoped_node_and_artifact_collision, - restart: - serviceRestart.scoped_node_collision_after_restart, - scoped_revocation: scopedNodeRevocationAfterRestart, - }, - durationMs: - tenantIsolation.scoped_node_and_artifact_collision?.duration_ms + - scopedNodeRevocationAfterRestart.duration_ms, - }), - requirement({ - id: "31_signed_hostile_artifact_path_preserves_service", - passed: - signedHostileArtifactPath.observed.rejection.denied === true && - signedHostileArtifactPath.observed.valid_metadata_response === - "vfs_metadata_recorded" && - signedHostileArtifactPath.observed.health_response === - "node_heartbeat" && - signedHostileArtifactPath.observed.process_cleanup === - "not_active", - evidence: signedHostileArtifactPath, - durationMs: signedHostileArtifactPath.duration_ms, - }), - ]; - const fullReleasePassed = - strictFullRelease && - proxyConfigProvenance?.active_state === "active" && - strictRequirementLedger.every((requirement) => requirement.passed === true); - const namedScenarios = strictRequirementLedger.map((requirement) => ({ - id: requirement.id, - name: requirement.id - .replace(/^\d+_/, "") - .replaceAll("_", " "), - status: requirement.passed ? "passed" : "failed", - ...(requirement.duration_ms - ? { duration_ms: requirement.duration_ms } - : {}), - })); - - const report = { - kind: "clusterflux-cli-happy-path-live", - release_name: manifest.release_name, - source_commit: manifest.source_commit, - source_tree_digest: manifest.source_tree_digest, - public_tree_identity: manifest.public_tree_identity, - binary_digests: manifest.binary_digests, - service_addr: serviceAddr, - default_hosted_coordinator_endpoint: serviceEndpoint, - public_repository_url: publicSource.published ? publicSource.url : null, - public_source: publicSource, - tenant, - project, - worker_node: workerNode, - process: processId, - signed_in: true, - fresh_hosted_login: !reuseSessionFile, - reused_scoped_cli_session: Boolean(reuseSessionFile), - received_default_project: receivedDefaultProject, - process_started_before_node: true, - parked_main_observed: { - main_state: parkedStatus.live_process.main_state, - main_wait_state: parkedStatus.live_process.main_wait_state, - connected_nodes: parkedStatus.live_process.connected_nodes, - }, - node_enrollment_grants_used: 2, - persisted_node_credential_mode: - process.platform === "win32" ? null : fs.statSync(credentialPath).mode & 0o777, - persisted_node_credential_digest: credentialDigest, - node_restarted_without_grant: true, - restarted_node_ready: secondReady.node, - node_liveness_transition: nodeLivenessTransition, - flagship_terminal_state: releasedFlagshipStatus.state, - flagship_task_definitions: [ - "prepare_source", - "compile_linux", - "package_release", - ], - rootless_podman_flagship_completed: true, - downstream_release_artifact: releaseArtifact, - artifact_download: { - bytes_written: download.local_download.bytes_written, - verified_digest: download.local_download.verified_digest, - executable_output: builtOutput, - }, - flagship_process_released_automatically: true, - concurrent_same_definition_tasks: sameDefinitionIdentity, - flagship_same_definition_task_instances: concurrentCompileTasks, - repeated_park_wake: repeatedParkWake, - long_join: longJoin, - main_before_child_lifecycle: mainBeforeChildLifecycle, - live_dap_edit_restart: dapEditRestart, - process_restart_requested: true, - process_cancel_requested: true, - dap_process_aborted: true, - process_cancellation_lifecycle: processCancellationLifecycle, - tenant_isolation: tenantIsolation, - scoped_node_revocation_after_restart: - scopedNodeRevocationAfterRestart, - credential_security: { - user: userCredentialSecurity, - node: { - node: securityNode.node, - credential_digest: securityNode.credential_digest, - ...securityNode.evidence, - revoked: revokedNodeCredential.denied, - }, - agent: agentCredentialSecurity, - }, - quota_preallocation: quotaPreallocation, - bandwidth_preallocation: bandwidthPreallocation, - relay_emergency_disable: relayEmergencyDisable, - hosted_login_isolation: hostedLoginIsolation, - dropped_connection_rollback: droppedConnectionRollback, - launch_attempt_ownership: launchAttemptOwnership, - live_soak: liveSoak, - hosted_service_restart: { - ...serviceRestart, - worker: undefined, - }, - release_binding: { - manifest: manifestPath, - source_commit: manifest.source_commit, - source_tree_digest: manifest.source_tree_digest, - source_tree_clean: manifest.source_tree_clean, - public_tree_identity: manifest.public_tree_identity, - binary_digests: manifest.binary_digests, - release_candidate: manifest.release_candidate, - deployment: serviceRestart.after || deploymentProvenance(), - configuration: configProvenance, - proxy_configuration: proxyConfigProvenance, - }, - commands, - quality_gates: qualityGates, - hello_build: helloBuild, - recovery_build: recoveryBuild, - malformed_identifiers: malformedIdentifiers, - signed_hostile_artifact_path: signedHostileArtifactPath, - named_scenarios: namedScenarios, - scenario_skips: [], - strict_requirement_ledger: strictRequirementLedger, - acceptance_result: fullReleasePassed ? "passed" : "partial", - }; - ensureDir(path.dirname(reportPath)); - fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); - if (strictFullRelease && !fullReleasePassed) { - const failed = strictRequirementLedger - .filter((requirement) => requirement.passed !== true) - .map((requirement) => requirement.id); - throw new Error(`strict full release failed: ${failed.join(", ")}`); - } - console.log(`CLI happy-path live smoke passed: ${reportPath}`); - } finally { - await stopChild(worker?.child); - } -} - -main().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/cli-install-smoke.js b/scripts/cli-install-smoke.js deleted file mode 100755 index 878543e..0000000 --- a/scripts/cli-install-smoke.js +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-install-")); -const installRoot = path.join(temp, "install"); -const targetDir = - process.env.CLUSTERFLUX_CLI_INSTALL_CARGO_TARGET_DIR || - process.env.CARGO_TARGET_DIR || - path.join(repo, "target"); -const project = path.join(repo, "tests/fixtures/runtime-conformance"); -const binName = process.platform === "win32" ? "clusterflux.exe" : "clusterflux"; -const installedBin = path.join(installRoot, "bin", binName); - -try { - cp.execFileSync( - "cargo", - [ - "install", - "--path", - "crates/clusterflux-cli", - "--bin", - "clusterflux", - "--root", - installRoot, - "--debug" - ], - { - cwd: repo, - env: { - ...process.env, - CARGO_TARGET_DIR: targetDir - }, - stdio: "inherit" - } - ); - - assert(fs.existsSync(installedBin), "installed clusterflux binary must exist"); - - const inspection = JSON.parse( - cp.execFileSync( - installedBin, - ["bundle", "inspect", "--project", project, "--json"], - { cwd: repo, encoding: "utf8" } - ) - ); - - assert.strictEqual(inspection.project, project); - assert.strictEqual(inspection.metadata.embeds_full_container_images, false); - assert(inspection.metadata.environments.some((env) => env.name === "linux")); - assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/lib.rs")); -} finally { - fs.rmSync(temp, { recursive: true, force: true }); -} - -console.log("CLI install smoke passed"); diff --git a/scripts/cli-local-run-smoke.js b/scripts/cli-local-run-smoke.js deleted file mode 100755 index e2fc299..0000000 --- a/scripts/cli-local-run-smoke.js +++ /dev/null @@ -1,269 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { configurePodmanTestEnvironment } = require("./podman-test-env"); - -const repo = path.resolve(__dirname, ".."); -configurePodmanTestEnvironment(repo); -if ( - !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && - cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && - cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 -) { - cp.execFileSync( - "nix", - ["shell", "nixpkgs#podman", "--command", "node", __filename], - { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_PODMAN_NIX_SHELL: "1", - }, - stdio: "inherit", - } - ); - process.exit(0); -} -const project = path.join(repo, "tests/fixtures/runtime-conformance"); - -cp.execFileSync( - "cargo", - ["build", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node"], - { cwd: repo, stdio: "inherit" } -); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function runCli(args, env = {}) { - return new Promise((resolve, reject) => { - const child = cp.spawn( - "cargo", - ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], - { - cwd: repo, - env: { - ...process.env, - ...env - } - } - ); - const cliPid = child.pid; - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - child.on("exit", (code) => { - if (code !== 0) { - reject(new Error(`CLI run failed with code ${code}\n${stderr}`)); - return; - } - try { - resolve({ pid: cliPid, report: JSON.parse(stdout) }); - } catch (error) { - reject(new Error(`CLI output was not JSON: ${stdout}\n${error.stack || error.message}`)); - } - }); - }); -} - -(async () => { - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - assert(Number.isInteger(coordinator.pid)); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const { pid: cliPid, report } = await runCli([ - "run", - "--coordinator", - `${addr.host}:${addr.port}`, - "--project", - project, - "--json", - ]); - assert(Number.isInteger(cliPid)); - assert.notStrictEqual(cliPid, coordinator.pid); - assert.strictEqual(report.plan.entry, "build"); - assert.deepStrictEqual(report.plan.session, "Anonymous"); - assert.strictEqual(report.boundary.cli_process_started_node_process, true); - assert.strictEqual(report.boundary.cli_process_started_coordinator_process, false); - assert(Number.isInteger(report.boundary.spawned_node_process_id)); - assert.notStrictEqual(report.boundary.spawned_node_process_id, cliPid); - assert.notStrictEqual(report.boundary.spawned_node_process_id, coordinator.pid); - assert.strictEqual(report.boundary.node_session_requests, 0); - assert.strictEqual(report.node_report.node_status, "completed"); - assert.strictEqual(report.node_report.execution_substrate, "wasm"); - assert.strictEqual(report.node_report.task_spawn_host_import, true); - assert.strictEqual( - report.node_report.pre_node_process_status.processes.length, - 1 - ); - assert.deepStrictEqual( - report.node_report.pre_node_process_status.processes[0].connected_nodes, - [] - ); - assert.strictEqual( - report.node_report.pre_node_process_status.processes[0].main_state, - "running" - ); - assert.strictEqual( - report.node_report.pre_node_process_status.processes[0].main_wait_state, - "waiting_for_node", - "the coordinator must expose that the capless main is parked on placement before a node exists" - ); - assert.strictEqual( - report.node_report.pre_node_process_status.processes[0].main_task_instance, - report.node_report.run.task_instance - ); - assert.strictEqual(report.node_report.run.status, "main_launched"); - assert.strictEqual(report.node_report.join.type, "task_joined"); - const process = report.node_report.run.process; - assert.strictEqual(process, "vp-current"); - - const events = await send(addr, { - type: "list_task_events", - tenant: "tenant", - project: "project", - actor_user: "user", - process - }); - assert.strictEqual(events.type, "task_events"); - assert(events.events.length >= 4); - assert( - events.events - .filter((event) => event.executor === "node") - .every((event) => event.node === "node-cli-local") - ); - assert( - events.events.some( - (event) => - event.executor === "coordinator_main" && - event.node === "coordinator-main" - ) - ); - assert(events.events.every((event) => event.process === process)); - assert.deepStrictEqual( - new Set(events.events.map((event) => event.task_definition)), - new Set([ - report.node_report.run.task_definition, - "prepare_source", - "compile_linux", - "package_release", - ]) - ); - assert.strictEqual( - new Set(events.events.map((event) => event.task)).size, - events.events.length, - "every live task event must retain its unique instance identity" - ); - assert( - events.events.some( - (event) => event.task === report.node_report.run.task_instance - ) - ); - assert(events.events.some((event) => event.task.endsWith(":child:1"))); - assert(events.events.some((event) => event.task.endsWith(":child:2"))); - assert(events.events.some((event) => event.task.endsWith(":child:3"))); - assert(events.events.some((event) => event.artifact_path)); - } finally { - coordinator.kill("SIGTERM"); - } - - const { pid: autoCliPid, report: autoReport } = await runCli([ - "run", - "--local", - "--project", - project, - "--json", - ]); - assert(Number.isInteger(autoCliPid)); - assert.strictEqual(autoReport.plan.entry, "build"); - assert.deepStrictEqual(autoReport.plan.coordinator, "LocalOnly"); - assert.deepStrictEqual(autoReport.plan.session, "Anonymous"); - assert.strictEqual(autoReport.boundary.cli_process_started_node_process, true); - assert.strictEqual(autoReport.boundary.cli_process_started_coordinator_process, true); - assert.match(autoReport.boundary.coordinator_address, /^127\.0\.0\.1:\d+$/); - assert(Number.isInteger(autoReport.boundary.coordinator_process_id)); - assert(Number.isInteger(autoReport.boundary.spawned_node_process_id)); - assert.notStrictEqual(autoReport.boundary.coordinator_process_id, autoCliPid); - assert.notStrictEqual(autoReport.boundary.spawned_node_process_id, autoCliPid); - assert.notStrictEqual( - autoReport.boundary.spawned_node_process_id, - autoReport.boundary.coordinator_process_id - ); - assert.strictEqual(autoReport.boundary.node_session_requests, 0); - assert.strictEqual(autoReport.node_report.node_status, "completed"); - assert.strictEqual(autoReport.node_report.execution_substrate, "wasm"); - assert.strictEqual(autoReport.node_report.task_spawn_host_import, true); - assert.strictEqual(autoReport.node_report.run.status, "main_launched"); - assert.strictEqual(autoReport.node_report.join.type, "task_joined"); - - console.log("CLI local run smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/cli-login-smoke.js b/scripts/cli-login-smoke.js deleted file mode 100644 index 61b053b..0000000 --- a/scripts/cli-login-smoke.js +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const coordinator = "https://coord.example.test"; -const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; - -function clusterflux(args) { - return JSON.parse( - cp.execFileSync( - "cargo", - ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], - { cwd: repo, encoding: "utf8" } - ) - ); -} - -function clusterfluxRaw(args, env = {}) { - return cp.spawnSync( - "cargo", - ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], - { - cwd: repo, - encoding: "utf8", - env: { - ...process.env, - ...env, - }, - } - ); -} - -const browser = clusterflux(["login", "--plan", "--coordinator", coordinator, "--json"]); -assert.strictEqual(browser.coordinator, coordinator); -assert(browser.human_flow.Browser, "browser login should be available for human users"); -assert.strictEqual(browser.human_flow.Browser.authorization_url, null); -assert.strictEqual(browser.human_flow.Browser.server_owns_state, true); -assert.strictEqual(browser.human_flow.Browser.server_owns_nonce, true); -assert.strictEqual(browser.human_flow.Browser.pkce_required, true); -assert.strictEqual(browser.human_flow.Browser.hosted_callback, true); -assert.strictEqual(browser.human_flow.Browser.cli_receives_provider_authorization_code, false); -assert.strictEqual(browser.human_flow.Browser.cli_submits_identity_claims, false); - -const defaultBrowser = clusterflux(["login", "--plan", "--json"]); -assert.strictEqual(defaultBrowser.coordinator, defaultHostedCoordinatorEndpoint); -assert(defaultBrowser.human_flow.Browser); - -const nonInteractiveBrowser = clusterfluxRaw( - ["login", "--browser", "--non-interactive", "--coordinator", coordinator, "--json"], - { - CLUSTERFLUX_BROWSER_OPEN_COMMAND: - "node -e 'require(\"fs\").writeFileSync(\"/tmp/clusterflux-browser-should-not-open\", \"opened\")'", - } -); -assert.strictEqual(nonInteractiveBrowser.status, 20, nonInteractiveBrowser.stderr); -assert.doesNotMatch(nonInteractiveBrowser.stderr, /Opening Clusterflux browser login/); -const nonInteractiveReport = JSON.parse(nonInteractiveBrowser.stdout); -assert.strictEqual(nonInteractiveReport.status, "authentication_required"); -assert.strictEqual(nonInteractiveReport.non_interactive, true); -assert.strictEqual(nonInteractiveReport.browser_opened, false); -assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); -assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); -assert( - nonInteractiveReport.machine_error.next_actions.includes( - "rerun without --non-interactive to open the browser" - ) -); - -console.log("CLI login smoke passed"); diff --git a/scripts/cli-output-mode-smoke.js b/scripts/cli-output-mode-smoke.js deleted file mode 100644 index 6cced83..0000000 --- a/scripts/cli-output-mode-smoke.js +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const project = path.join(repo, "tests/fixtures/runtime-conformance"); -const isolatedCwd = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-output-")); -const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-home-")); - -function clusterflux(args, env = {}, cwd = isolatedCwd) { - return cp.execFileSync( - "cargo", - [ - "run", - "-q", - "--manifest-path", - path.join(repo, "Cargo.toml"), - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - ...args, - ], - { - cwd, - encoding: "utf8", - env: { - ...process.env, - HOME: isolatedHome, - USERPROFILE: isolatedHome, - XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), - XDG_DATA_HOME: path.join(isolatedHome, ".local", "share"), - XDG_STATE_HOME: path.join(isolatedHome, ".local", "state"), - ...env, - }, - } - ); -} - -function json(args, env, cwd) { - return JSON.parse(clusterflux(args, env, cwd)); -} - -function assertHuman(name, output, requiredPatterns) { - assert( - !output.trimStart().startsWith("{"), - `${name} default output should be human-readable text, not JSON` - ); - for (const pattern of requiredPatterns) { - assert.match(output, pattern, `${name} human output missing ${pattern}`); - } -} - -const helpHuman = clusterflux(["help"]); -assertHuman("help", helpHuman, [ - /Primary workflow:/, - /clusterflux login --browser/, - /clusterflux project init/, - /clusterflux node attach; clusterflux-node --worker/, - /Clusterflux: Launch Virtual Process/, - /Hosted account creation happens in the browser login flow/, - /--json/, -]); - -const loginHuman = clusterflux([ - "login", - "--plan", - "--coordinator", - "https://coord.example.test", -]); -assertHuman("login", loginHuman, [ - /Clusterflux login/, - /flow: browser/, -]); - -const loginJson = json([ - "login", - "--plan", - "--coordinator", - "https://coord.example.test", - "--json", -]); -assert.strictEqual(loginJson.coordinator, "https://coord.example.test"); -assert(loginJson.human_flow.Browser); -assert.strictEqual(loginJson.human_flow.Browser.hosted_callback, true); -assert.strictEqual(loginJson.human_flow.Browser.cli_submits_identity_claims, false); - -const doctorHuman = clusterflux(["doctor"]); -assertHuman("doctor", doctorHuman, [ - /Clusterflux doctor/, - /coordinator reachability: not_configured/, - /dependencies:/, - /auth:/, - /node capabilities:/, - /node readiness: (ready_to_attach|local_dependencies_missing|limited_capabilities)/, - /node next:/, -]); - -const doctorJson = json(["doctor", "--json"]); -assert.strictEqual(doctorJson.coordinator_reachability.checked, false); -assert.strictEqual(doctorJson.coordinator_reachability.status, "not_configured"); -assert( - ["ready_to_attach", "local_dependencies_missing", "limited_capabilities"].includes( - doctorJson.node_readiness_summary.status - ) -); -assert.strictEqual(doctorJson.node_readiness_summary.explicit_attach_required, true); -assert.strictEqual( - doctorJson.node_readiness_summary.command_execution_capability, - true -); -assert(Array.isArray(doctorJson.node_readiness_summary.missing_local_dependencies)); -assert(doctorJson.node_readiness_summary.next_actions.length >= 2); - -const authJson = json(["auth", "status", "--json"], { - CLUSTERFLUX_TOKEN: "token", - CLUSTERFLUX_TOKEN_EXPIRES_AT: "2026-07-04T00:00:00Z", -}, isolatedCwd); -assert.strictEqual(authJson.session.kind, "human"); -assert.strictEqual(authJson.session.token_expiry_posture, "expires_at"); -assert.strictEqual(authJson.session.expires_at, "2026-07-04T00:00:00Z"); -assert.strictEqual(authJson.coordinator_account_status.checked, false); -assert.strictEqual(authJson.coordinator_account_status.account_status, "unknown"); -assert.strictEqual( - authJson.coordinator_account_status.private_moderation_details_exposed, - false -); - -const inspectHuman = clusterflux(["bundle", "inspect", "--project", project]); -assertHuman("bundle inspect", inspectHuman, [ - /Clusterflux bundle inspect/, - /bundle: sha256:/, - /environments:/, -]); - -const inspectJson = json(["bundle", "inspect", "--project", project, "--json"]); -assert.strictEqual(inspectJson.project, project); -assert.match(inspectJson.metadata.identity, /^sha256:/); -assert.match(inspectJson.metadata.wasm_code, /^sha256:/); -assert.strictEqual(inspectJson.metadata.task_metadata.default_entrypoint, "build"); -assert.deepStrictEqual(inspectJson.metadata.task_metadata.entrypoints, [ - "build", - "fail", - "identity", - "long-join", - "park-wake", - "restart", -]); -assert.strictEqual( - inspectJson.metadata.source_metadata.transfer_policy.coordinator_receives_source_bytes_by_default, - false -); -assert.strictEqual( - inspectJson.metadata.source_metadata.transfer_policy.default_full_repo_tarball, - false -); -assert.strictEqual(inspectJson.metadata.debug_metadata.dap_virtual_process, true); -assert.strictEqual( - inspectJson.metadata.large_input_policy.selected_inputs_are_content_digests, - true -); -assert.strictEqual(inspectJson.metadata.large_input_policy.selected_input_bytes_included, false); -assert.strictEqual(inspectJson.metadata.large_input_policy.full_repository_bytes_included, false); -assert.strictEqual( - inspectJson.metadata.large_input_policy.silent_task_argument_serialization, - false -); -assert(inspectJson.metadata.large_input_policy.supported_handle_types.includes("SourceSnapshot")); -assert.strictEqual( - inspectJson.metadata.restart_compatibility.source_edits_can_restart_from_clean_task_boundary, - true -); -assert.strictEqual( - inspectJson.metadata.restart_compatibility.requires_clean_checkpoint_boundary, - true -); -assert.strictEqual( - inspectJson.metadata.restart_compatibility.compares_task_abi, - inspectJson.metadata.task_metadata.task_abi -); -assert.strictEqual( - inspectJson.metadata.restart_compatibility.incompatible_changes_require_whole_process_restart, - true -); - -console.log("CLI output mode smoke passed"); diff --git a/scripts/coordinator-wire.js b/scripts/coordinator-wire.js deleted file mode 100644 index 7761736..0000000 --- a/scripts/coordinator-wire.js +++ /dev/null @@ -1,43 +0,0 @@ -let requestId = 0; - -function coordinatorWireRequest(payload, prefix = "acceptance") { - if (payload && payload.type === "coordinator_request") return payload; - if (!payload || typeof payload.type !== "string" || !payload.type.trim()) { - throw new Error("coordinator payload must have a non-empty type"); - } - requestId += 1; - return { - type: "coordinator_request", - protocol_version: 1, - request_id: `${prefix}-${process.pid}-${requestId}`, - operation: payload.type, - authentication: authenticationMetadata(payload), - payload, - }; -} - -function authenticationMetadata(payload) { - if (payload.type === "authenticated") { - return { - kind: "cli_session", - session: true, - request_operation: payload.request?.type || "unknown", - }; - } - if (payload.type === "signed_node" || payload.node_signature) { - return { kind: "node_signature", node: payload.node || null }; - } - if (payload.agent_signature) { - return { - kind: "agent_signature", - agent: payload.actor_agent || null, - fingerprint: payload.agent_public_key_fingerprint || null, - }; - } - if (payload.admin_token) { - return { kind: "admin_credential" }; - } - return { kind: "none" }; -} - -module.exports = { coordinatorWireRequest }; diff --git a/scripts/dap-client.js b/scripts/dap-client.js deleted file mode 100644 index 32d720c..0000000 --- a/scripts/dap-client.js +++ /dev/null @@ -1,135 +0,0 @@ -const cp = require("child_process"); - -class DapClient { - constructor({ - cwd = process.cwd(), - env = process.env, - command = "cargo", - args = [ - "run", - "-q", - "-p", - "clusterflux-dap", - "--bin", - "clusterflux-debug-dap", - ], - } = {}) { - this.child = cp.spawn( - command, - args, - { cwd, env } - ); - this.seq = 1; - this.buffer = Buffer.alloc(0); - this.messages = []; - this.waiters = []; - this.stderr = ""; - - this.child.stdout.on("data", (chunk) => { - this.buffer = Buffer.concat([this.buffer, chunk]); - this.parse(); - }); - this.child.stderr.on("data", (chunk) => { - this.stderr += chunk.toString(); - }); - this.child.on("exit", () => this.flushWaiters()); - } - - send(command, args = {}) { - const seq = this.seq++; - const message = { seq, type: "request", command, arguments: args }; - const payload = Buffer.from(JSON.stringify(message)); - this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); - this.child.stdin.write(payload); - return seq; - } - - async response(seq, command) { - const message = await this.waitFor( - (item) => - item.type === "response" && - item.request_seq === seq && - item.command === command - ); - if (!message.success) { - throw new Error( - `DAP ${command} failed: ${message.message || JSON.stringify(message)}\nAdapter stderr:\n${this.stderr}` - ); - } - return message; - } - - async failure(seq, command) { - const message = await this.waitFor( - (item) => - item.type === "response" && - item.request_seq === seq && - item.command === command - ); - if (message.success) { - throw new Error(`DAP ${command} unexpectedly succeeded`); - } - return message; - } - - waitFor( - predicate, - timeoutMs = Number(process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || 120000) - ) { - const existing = this.messages.find(predicate); - if (existing) return Promise.resolve(existing); - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - this.child.kill("SIGKILL"); - const recent = this.messages - .slice(-10) - .map((message) => JSON.stringify(message)) - .join("\n"); - reject( - new Error( - `timed out waiting for DAP message\nRecent DAP messages:\n${recent}\nAdapter stderr:\n${this.stderr}` - ) - ); - }, timeoutMs); - this.waiters.push({ predicate, resolve, timer }); - }); - } - - parse() { - while (true) { - const headerEnd = this.buffer.indexOf("\r\n\r\n"); - if (headerEnd < 0) return; - const header = this.buffer.slice(0, headerEnd).toString(); - const match = header.match(/Content-Length: (\d+)/i); - if (!match) throw new Error(`bad DAP header: ${header}`); - const length = Number(match[1]); - const start = headerEnd + 4; - const end = start + length; - if (this.buffer.length < end) return; - const payload = this.buffer.slice(start, end).toString(); - this.buffer = this.buffer.slice(end); - this.messages.push(JSON.parse(payload)); - this.flushWaiters(); - } - } - - flushWaiters() { - for (const waiter of [...this.waiters]) { - const message = this.messages.find(waiter.predicate); - if (!message) continue; - clearTimeout(waiter.timer); - this.waiters.splice(this.waiters.indexOf(waiter), 1); - waiter.resolve(message); - } - } - - async close() { - if (this.child.exitCode !== null) return; - const seq = this.send("disconnect"); - await this.response(seq, "disconnect"); - this.child.stdin.end(); - } -} - -module.exports = { DapClient }; diff --git a/scripts/dap-smoke.js b/scripts/dap-smoke.js deleted file mode 100644 index 3ab3318..0000000 --- a/scripts/dap-smoke.js +++ /dev/null @@ -1,510 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); -const { DapClient } = require("./dap-client"); - -(async () => { - const repo = path.resolve(__dirname, ".."); - const project = path.join(repo, "tests/fixtures/runtime-conformance"); - const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs")); - const sourceLines = fs.readFileSync(sourcePath, "utf8").split(/\r?\n/); - const buildMainLine = - sourceLines.findIndex((line) => line.includes("pub async fn build_main()")) + 1; - assert(buildMainLine > 0, "flagship source must contain build_main"); - - const hostileDapClient = new DapClient(); - try { - const initialize = hostileDapClient.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await hostileDapClient.response(initialize, "initialize"); - for (const processId of [ - "", - " ", - "bad\u0000process", - "bad process!", - "x".repeat(256), - ]) { - const malformedLaunch = hostileDapClient.send("launch", { - entry: "build", - project, - runtimeBackend: "local-services", - processId, - }); - const rejection = await hostileDapClient.failure( - malformedLaunch, - "launch" - ); - assert.match( - rejection.message, - /invalid DAP processId: ProcessId is invalid/, - `unexpected malformed DAP identifier response: ${JSON.stringify(rejection)}` - ); - } - const validLaunch = hostileDapClient.send("launch", { - entry: "build", - project, - runtimeBackend: "local-services", - processId: "vp-valid-after-malformed", - }); - await hostileDapClient.response(validLaunch, "launch"); - await hostileDapClient.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - await hostileDapClient.close(); - } catch (error) { - if (hostileDapClient.child.exitCode === null) { - hostileDapClient.child.kill("SIGKILL"); - } - throw error; - } - - const asynchronousClient = new DapClient(); - try { - const initialize = asynchronousClient.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await asynchronousClient.response(initialize, "initialize"); - const launch = asynchronousClient.send("launch", { - entry: "long-join", - project, - runtimeBackend: "local-services", - }); - await asynchronousClient.response(launch, "launch"); - await asynchronousClient.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const configuredAt = Date.now(); - const configurationDone = asynchronousClient.send("configurationDone"); - await asynchronousClient.response(configurationDone, "configurationDone"); - assert( - Date.now() - configuredAt < 2_000, - "configurationDone must not wait for bundle build or runtime completion" - ); - - const threadDeadline = Date.now() + 120_000; - let runningThreads = []; - while (Date.now() < threadDeadline) { - const threadsRequest = asynchronousClient.send("threads"); - runningThreads = ( - await asynchronousClient.response(threadsRequest, "threads") - ).body.threads; - if (runningThreads.length > 0) break; - await new Promise((resolve) => setTimeout(resolve, 100)); - } - assert(runningThreads.length > 0, "asynchronous launch never reported a live thread"); - - const pauseAt = Date.now(); - const pause = asynchronousClient.send("pause", { - threadId: runningThreads[0].id, - }); - await asynchronousClient.response(pause, "pause"); - assert(Date.now() - pauseAt < 2_000, "pause response was blocked by runtime work"); - const paused = await asynchronousClient.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "pause" - ); - - const continueAt = Date.now(); - const continued = asynchronousClient.send("continue", { - threadId: paused.body.threadId, - }); - await asynchronousClient.response(continued, "continue"); - assert( - Date.now() - continueAt < 2_000, - "continue response was blocked by runtime observation" - ); - - const disconnectAt = Date.now(); - await asynchronousClient.close(); - assert( - Date.now() - disconnectAt < 2_000, - "disconnect response was blocked by runtime observation" - ); - } catch (error) { - if (asynchronousClient.child.exitCode === null) { - asynchronousClient.child.kill("SIGKILL"); - } - throw error; - } - - const client = new DapClient(); - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await client.response(initialize, "initialize"); - - const launch = client.send("launch", { - entry: "build", - project, - runtimeBackend: "local-services", - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const breakpoints = client.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: buildMainLine }], - }); - const breakpointResponse = await client.response( - breakpoints, - "setBreakpoints" - ); - assert.strictEqual(breakpointResponse.body.breakpoints.length, 1); - assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, false); - - const configurationDone = client.send("configurationDone"); - await client.response(configurationDone, "configurationDone"); - const installedBreakpoint = await client.waitFor( - (message) => - message.type === "event" && - message.event === "breakpoint" && - message.body?.breakpoint?.verified === true - ); - assert.strictEqual(installedBreakpoint.body.breakpoint.line, buildMainLine); - const stopped = await client.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint" - ); - assert.strictEqual(stopped.body.allThreadsStopped, true); - assert.match(stopped.body.description, /confirmed by every active participant/i); - - const threadsRequest = client.send("threads"); - const threads = (await client.response(threadsRequest, "threads")).body - .threads; - const mainThread = threads.find((thread) => - thread.name.includes("build coordinator main") - ); - assert(mainThread, "the running Wasm entrypoint must be the DAP coordinator-main thread"); - assert.strictEqual(stopped.body.threadId, mainThread.id); - - const stackRequest = client.send("stackTrace", { - threadId: mainThread.id, - startFrame: 0, - levels: 1, - }); - const stack = (await client.response(stackRequest, "stackTrace")).body - .stackFrames; - assert.strictEqual(stack.length, 1); - assert.strictEqual(stack[0].line, buildMainLine); - assert.strictEqual(stack[0].source.path, sourcePath); - assert.match(stack[0].name, /build_main::wasm/); - assert.doesNotMatch(stack[0].name, /podman|cmd\.exe|powershell|pid|native child/i); - - const sourceRequest = client.send("source", { source: stack[0].source }); - const source = (await client.response(sourceRequest, "source")).body; - assert.match(source.content, /build_main/); - assert.match(source.mimeType, /rust/); - - const scopesRequest = client.send("scopes", { frameId: stack[0].id }); - const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; - const localsScope = scopes.find((scope) => scope.name === "Source Locals"); - const wasmScope = scopes.find((scope) => scope.name === "Wasm Frame Locals"); - const argsScope = scopes.find( - (scope) => scope.name === "Task Args and Handles" - ); - const runtimeScope = scopes.find( - (scope) => scope.name === "Clusterflux Runtime" - ); - assert(localsScope && wasmScope && argsScope && runtimeScope); - - const localsRequest = client.send("variables", { - variablesReference: localsScope.variablesReference, - }); - const locals = (await client.response(localsRequest, "variables")).body - .variables; - assert( - locals.some( - (variable) => - variable.name === "unavailable-local-diagnostic" && - String(variable.value).includes("cannot be inspected") - ) - ); - - const wasmRequest = client.send("variables", { - variablesReference: wasmScope.variablesReference, - }); - const wasmLocals = (await client.response(wasmRequest, "variables")).body - .variables; - assert.deepStrictEqual( - wasmLocals.map((variable) => variable.name), - ["wasm-local-diagnostic"] - ); - assert.match(wasmLocals[0].value, /did not report inspectable Wasm frame locals/); - - const argsRequest = client.send("variables", { - variablesReference: argsScope.variablesReference, - }); - const args = (await client.response(argsRequest, "variables")).body.variables; - assert.deepStrictEqual( - args.map((variable) => variable.name), - ["runtime-boundary-diagnostic"] - ); - assert.match(args[0].value, /reported no task arguments or handles/); - - const runtimeRequest = client.send("variables", { - variablesReference: runtimeScope.variablesReference, - }); - const runtime = (await client.response(runtimeRequest, "variables")).body - .variables; - const value = (name) => runtime.find((variable) => variable.name === name)?.value; - assert.strictEqual(value("runtime_backend"), "LocalServices"); - assert.strictEqual(value("state"), "Frozen"); - assert.strictEqual(value("debug_epoch"), 1); - assert.strictEqual(value("coordinator_task_events"), 0); - assert.match( - String(value("command_status")), - /frozen through local services at executing Wasm probe/ - ); - - const step = client.send("next", { threadId: mainThread.id }); - const stepFailure = await client.failure(step, "next"); - assert.match(stepFailure.message, /source stepping is not yet available/i); - assert.match(stepFailure.message, /synthetic step/i); - - const restart = client.send("restartFrame", { frameId: stack[0].id }); - const restartFailure = await client.failure(restart, "restartFrame"); - assert.match(restartFailure.message, /checkpoint boundary|still active/i); - - const incompatibleRestart = client.send("restartFrame", { - frameId: stack[0].id, - sourceCompatibility: "incompatible", - }); - const incompatibleFailure = await client.failure( - incompatibleRestart, - "restartFrame" - ); - assert.match(incompatibleFailure.message, /incompatible source edit/i); - assert.match(incompatibleFailure.message, /whole virtual-process restart/i); - - await client.close(); - } catch (error) { - if (client.child.exitCode === null) client.child.kill("SIGKILL"); - throw error; - } - - const failMainLine = - sourceLines.findIndex((line) => line.includes("pub async fn fail_main()")) + 1; - assert(failMainLine > 0, "flagship source must contain fail_main"); - const taskTrapLine = - sourceLines.findIndex((line) => line.includes("fn task_trap(")) + 1; - assert(taskTrapLine > 0, "flagship source must contain task_trap"); - const restartClient = new DapClient(); - try { - const initialize = restartClient.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await restartClient.response(initialize, "initialize"); - const launch = restartClient.send("launch", { - entry: "fail", - project, - runtimeBackend: "local-services", - }); - await restartClient.response(launch, "launch"); - await restartClient.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - const breakpoints = restartClient.send("setBreakpoints", { - source: { path: sourcePath }, - breakpoints: [{ line: failMainLine }, { line: taskTrapLine }], - }); - const breakpointResponse = await restartClient.response( - breakpoints, - "setBreakpoints" - ); - assert.deepStrictEqual( - breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), - [false, false] - ); - const configurationDone = restartClient.send("configurationDone"); - await restartClient.response(configurationDone, "configurationDone"); - const installedLines = []; - while (installedLines.length < 2) { - const installed = await restartClient.waitFor( - (message) => - message.type === "event" && - message.event === "breakpoint" && - message.body?.breakpoint?.verified === true && - !installedLines.includes(message.body.breakpoint.line) - ); - installedLines.push(installed.body.breakpoint.line); - } - assert.deepStrictEqual(installedLines.sort((a, b) => a - b), [ - failMainLine, - taskTrapLine, - ].sort((a, b) => a - b)); - const initialStop = await restartClient.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint" - ); - assert.strictEqual(initialStop.body.allThreadsStopped, true); - const threadsRequest = restartClient.send("threads"); - const threads = ( - await restartClient.response(threadsRequest, "threads") - ).body.threads; - const failThread = threads.find( - (thread) => thread.id === initialStop.body.threadId - ); - assert(failThread, "failed entrypoint must remain a virtual task thread"); - const stackRequest = restartClient.send("stackTrace", { - threadId: failThread.id, - startFrame: 0, - levels: 1, - }); - const failedStack = ( - await restartClient.response(stackRequest, "stackTrace") - ).body.stackFrames; - assert.strictEqual(failedStack[0].line, failMainLine); - - const continueRequest = restartClient.send("continue", { - threadId: failThread.id, - }); - await restartClient.response(continueRequest, "continue"); - const childStop = await restartClient.waitFor( - (message) => - message.seq > initialStop.seq && - message.type === "event" && - message.event === "stopped" && - message.body.reason === "breakpoint", - 70000 - ); - assert.strictEqual(childStop.body.allThreadsStopped, true); - - const childThreadsRequest = restartClient.send("threads"); - const childThreads = ( - await restartClient.response(childThreadsRequest, "threads") - ).body.threads; - const childThread = childThreads.find( - (thread) => thread.id === childStop.body.threadId - ); - assert(childThread, "executing child task must become a DAP virtual thread"); - assert.notStrictEqual(childThread.id, failThread.id); - assert.match(childThread.name, /task trap/i); - - const childStackRequest = restartClient.send("stackTrace", { - threadId: childThread.id, - startFrame: 0, - levels: 1, - }); - const childStack = ( - await restartClient.response(childStackRequest, "stackTrace") - ).body.stackFrames; - assert.strictEqual(childStack[0].line, taskTrapLine); - assert.match(childStack[0].name, /task_trap::wasm/); - - const childScopesRequest = restartClient.send("scopes", { - frameId: childStack[0].id, - }); - const childScopes = ( - await restartClient.response(childScopesRequest, "scopes") - ).body.scopes; - const childArgsScope = childScopes.find( - (scope) => scope.name === "Task Args and Handles" - ); - assert(childArgsScope, "child task argument scope must be present"); - const childArgsRequest = restartClient.send("variables", { - variablesReference: childArgsScope.variablesReference, - }); - const childArgs = ( - await restartClient.response(childArgsRequest, "variables") - ).body.variables; - assert( - childArgs.some( - (variable) => - variable.name === "arg_0" && - /SmallJson\(Number\(0\)\)/.test(String(variable.value)) - ), - "child task argument must come from the frozen node participant" - ); - - const parentScopesRequest = restartClient.send("scopes", { - frameId: failedStack[0].id, - }); - const parentScopes = ( - await restartClient.response(parentScopesRequest, "scopes") - ).body.scopes; - const parentArgsScope = parentScopes.find( - (scope) => scope.name === "Task Args and Handles" - ); - const parentArgsRequest = restartClient.send("variables", { - variablesReference: parentArgsScope.variablesReference, - }); - const parentArgs = ( - await restartClient.response(parentArgsRequest, "variables") - ).body.variables; - assert( - parentArgs.some( - (variable) => - /^task_handle_\d+$/.test(variable.name) && - /definition=task_trap instance=ti:.*:child:\d+ state=active/.test( - variable.value - ) && - variable.type === "runtime-handle" - ), - "parent task handle must come from its live Wasm host registry" - ); - - const continueChildRequest = restartClient.send("continue", { - threadId: childThread.id, - }); - await restartClient.response(continueChildRequest, "continue"); - await restartClient.waitFor( - (message) => - message.seq > childStop.seq && - message.type === "event" && - message.event === "terminated" - ); - - const terminalThreadsRequest = restartClient.send("threads"); - const terminalThreads = ( - await restartClient.response(terminalThreadsRequest, "threads") - ).body.threads; - assert.deepStrictEqual( - terminalThreads, - [], - "terminated processes must not retain stale virtual threads" - ); - - const restartRequest = restartClient.send("restartFrame", { - frameId: failedStack[0].id, - }); - const restartFailure = await restartClient.failure( - restartRequest, - "restartFrame" - ); - assert.match( - restartFailure.message, - /does not map to a virtual task/i, - "a frame from a terminated process must not restart a stale task" - ); - await restartClient.close(); - } catch (error) { - if (restartClient.child.exitCode === null) restartClient.child.kill("SIGKILL"); - throw error; - } - - console.log("DAP smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/deploy-release-candidate.sh b/scripts/deploy-release-candidate.sh deleted file mode 100755 index 8560767..0000000 --- a/scripts/deploy-release-candidate.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -manifest=${1:?usage: deploy-release-candidate.sh MANIFEST} -: "${CLUSTERFLUX_DEPLOY_COMMAND:?set the documented exact-candidate deployment command}" -: "${CLUSTERFLUX_STRICT_SSH_TARGET:?set the production SSH target}" - -service_unit=${CLUSTERFLUX_STRICT_SERVICE_UNIT:-clusterflux-hosted.service} -proxy_unit=${CLUSTERFLUX_STRICT_PROXY_UNIT:-nginx.service} -evidence_path=${CLUSTERFLUX_DEPLOYMENT_EVIDENCE_PATH:-target/acceptance/deployment.json} -staging=$(mktemp -d) -trap 'rm -rf "$staging"' EXIT - -IFS=$'\t' read -r archive expected_archive_sha < <( - node - "$manifest" <<'NODE' -const fs = require("fs"); -const path = require("path"); -const manifestPath = path.resolve(process.argv[2]); -const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); -const asset = (manifest.assets || []).find((entry) => entry.name.startsWith("clusterflux-hosted-")); -if (!asset) throw new Error("candidate manifest has no hosted archive"); -const file = path.isAbsolute(asset.file) - ? asset.file - : path.resolve(path.dirname(manifestPath), asset.file); -process.stdout.write(`${file}\t${asset.sha256}\n`); -NODE -) - -actual_archive_sha=$(sha256sum "$archive" | awk '{print $1}') -test "$actual_archive_sha" = "${expected_archive_sha#sha256:}" -tar -xzf "$archive" -C "$staging" -candidate_coordinator=$(find "$staging" -type f -name clusterflux-hosted-service -perm -u+x -print -quit) -test -n "$candidate_coordinator" -candidate_sha=$(sha256sum "$candidate_coordinator" | awk '{print $1}') - -export CLUSTERFLUX_CANDIDATE_ARCHIVE="$archive" -export CLUSTERFLUX_CANDIDATE_ARCHIVE_SHA256="sha256:$actual_archive_sha" -export CLUSTERFLUX_CANDIDATE_COORDINATOR="$candidate_coordinator" -export CLUSTERFLUX_CANDIDATE_COORDINATOR_SHA256="sha256:$candidate_sha" -bash -euo pipefail -c "$CLUSTERFLUX_DEPLOY_COMMAND" - -main_pid=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=MainPID --value "$service_unit") -test "$main_pid" != 0 -remote_executable=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" readlink -f "/proc/$main_pid/exe") -remote_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$remote_executable" | awk '{print $1}') -test "$remote_sha" = "$candidate_sha" -service_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$service_unit") -service_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$service_fragment" | awk '{print $1}') -service_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl cat "$service_unit" | sha256sum | awk '{print $1}') -proxy_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$proxy_unit") -proxy_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$proxy_fragment" | awk '{print $1}') -proxy_exec_start=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=ExecStart --value "$proxy_unit") -proxy_executable=$(sed -n 's/.*path=\([^ ;]*\).*/\1/p' <<<"$proxy_exec_start") -proxy_configuration=$(sed -n 's/.* argv\[\]=.* -c \([^ ;]*\).*/\1/p' <<<"$proxy_exec_start") -test -n "$proxy_executable" -test -n "$proxy_configuration" -proxy_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" "$proxy_executable" -T -c "$proxy_configuration" 2>/dev/null | sha256sum | awk '{print $1}') -mkdir -p "$(dirname "$evidence_path")" - -EVIDENCE_PATH="$evidence_path" SERVICE_UNIT="$service_unit" \ -SERVICE_FRAGMENT="$service_fragment" SERVICE_FRAGMENT_SHA="$service_fragment_sha" \ -SERVICE_CONFIGURATION_SHA="$service_configuration_sha" \ -REMOTE_EXECUTABLE="$remote_executable" REMOTE_SHA="$remote_sha" \ -PROXY_UNIT="$proxy_unit" PROXY_FRAGMENT="$proxy_fragment" \ -PROXY_FRAGMENT_SHA="$proxy_fragment_sha" PROXY_EXECUTABLE="$proxy_executable" \ -PROXY_CONFIGURATION="$proxy_configuration" PROXY_CONFIGURATION_SHA="$proxy_configuration_sha" node <<'NODE' -const fs = require("fs"); -fs.writeFileSync(process.env.EVIDENCE_PATH, JSON.stringify({ - kind: "clusterflux-exact-candidate-deployment", - service_unit: process.env.SERVICE_UNIT, - service_unit_fragment: process.env.SERVICE_FRAGMENT, - service_unit_sha256: `sha256:${process.env.SERVICE_FRAGMENT_SHA}`, - service_configuration_sha256: `sha256:${process.env.SERVICE_CONFIGURATION_SHA}`, - hosted_service_executable: process.env.REMOTE_EXECUTABLE, - hosted_service_sha256: `sha256:${process.env.REMOTE_SHA}`, - proxy_unit: process.env.PROXY_UNIT, - proxy_unit_fragment: process.env.PROXY_FRAGMENT, - proxy_unit_sha256: `sha256:${process.env.PROXY_FRAGMENT_SHA}`, - proxy_executable: process.env.PROXY_EXECUTABLE, - proxy_configuration: process.env.PROXY_CONFIGURATION, - proxy_configuration_sha256: `sha256:${process.env.PROXY_CONFIGURATION_SHA}`, -}, null, 2) + "\n"); -NODE diff --git a/scripts/flagship-demo-smoke.js b/scripts/flagship-demo-smoke.js deleted file mode 100644 index 98a43f2..0000000 --- a/scripts/flagship-demo-smoke.js +++ /dev/null @@ -1,80 +0,0 @@ -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const hello = path.join(repo, "examples/hello-build"); -const recovery = path.join(repo, "examples/recovery-build"); -const conformance = path.join(repo, "tests/fixtures/runtime-conformance"); -const source = fs.readFileSync(path.join(hello, "src/lib.rs"), "utf8"); -const nonblank = source.split(/\r?\n/).filter((line) => line.trim()).length; - -assert(nonblank <= 80, "hello-build source must stay below 80 nonblank lines"); -for (const forbidden of [ - "#[cfg", - "unsafe", - "extern \"C\"", - "no_mangle", - "sha256:", - ".task_id(", - "#[test]", - "unwrap()", - "expect(", -]) { - assert(!source.includes(forbidden), "hello-build leaked implementation detail: " + forbidden); -} -assert.strictEqual((source.match(/#\[clusterflux::task/g) || []).length, 1); -assert.strictEqual((source.match(/#\[clusterflux::main/g) || []).length, 1); -assert(source.includes("source::current_project().snapshot().await?")); -assert(source.includes("clusterflux::spawn!(compile(source))")); -assert(source.includes(".on(clusterflux::env!(\"linux\"))")); -assert(source.includes(".run()")); -assert(source.includes("fs::publish")); -assert(fs.existsSync(path.join(hello, "fixture/hello-clusterflux.c"))); -assert(fs.existsSync(path.join(hello, "envs/linux/Containerfile"))); - -const recoverySource = fs.readFileSync(path.join(recovery, "src/lib.rs"), "utf8"); -assert.strictEqual((recoverySource.match(/spawn!\(build_lane/g) || []).length, 2); -assert(recoverySource.includes("TaskFailurePolicy::AwaitOperator")); -assert(recoverySource.includes("\"exit 23\"")); -for (const forbidden of ["#[cfg", "unsafe", "extern \"C\"", "no_mangle", ".task_id("]) { - assert(!recoverySource.includes(forbidden), "recovery-build leaked " + forbidden); -} - -const fixtureSource = fs.readFileSync(path.join(conformance, "src/lib.rs"), "utf8"); -assert(fixtureSource.includes("task_trap")); -assert(fixtureSource.includes("cooperative_cancellation_probe")); -assert(!fs.existsSync(path.join(repo, "examples", "launch-" + "build-demo"))); -assert(!fs.existsSync(path.join(hello, "src/bin/sdk-product-runtime.rs"))); - -const args = [ - "run", - "-q", - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - "build", - "--project", - hello, - "--json", -]; -const report = JSON.parse(cp.execFileSync("cargo", args, { - cwd: repo, - env: process.env, - encoding: "utf8", -})); -assert(report.bundle_artifact); -assert(report.bundle.metadata.selected_inputs.some((input) => input.path === "src/lib.rs")); -assert(report.bundle.metadata.task_metadata.entrypoints.includes("build")); -const tasks = JSON.parse( - fs.readFileSync( - path.resolve(repo, report.bundle_artifact.directory, "task-descriptors.json"), - "utf8" - ) -); -assert(tasks.some((task) => task.name === "compile")); -assert(tasks.some((task) => task.name === "snapshot_current_project")); -console.log("primary and recovery example smoke passed"); diff --git a/scripts/hostile-input-contract-smoke.js b/scripts/hostile-input-contract-smoke.js deleted file mode 100644 index 8d5173a..0000000 --- a/scripts/hostile-input-contract-smoke.js +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); -const { - agentIdentity, - signedAgentWorkflowRequest, -} = require("./agent-signing"); -const { - nodeIdentity, - signedNodeHeartbeat, -} = require("./node-signing"); - -const repo = path.resolve(__dirname, ".."); - -const signingInstrumentAgent = agentIdentity( - "hostile-input-contract-agent", - "agent-hostile-input-contract" -); -const explicitlyEmptyAgentNonce = signedAgentWorkflowRequest( - signingInstrumentAgent, - { - type: "start_process", - tenant: "tenant", - project: "project", - actor_agent: "agent-hostile-input-contract", - process: "process", - launch_attempt: "attempt", - restart: false, - }, - { nonce: "" } -); -assert.strictEqual( - explicitlyEmptyAgentNonce.agent_signature.nonce, - "", - "Agent signing instrument replaced an explicitly empty hostile nonce" -); -const signingInstrumentNode = nodeIdentity( - "hostile-input-contract-node", - "node-hostile-input-contract" -); -assert.strictEqual( - signedNodeHeartbeat( - "tenant", - "project", - "node-hostile-input-contract", - signingInstrumentNode, - { nonce: "" } - ).nonce, - "", - "Node signing instrument replaced an explicitly empty hostile nonce" -); - -function read(relativePath) { - return fs.readFileSync(path.join(repo, relativePath), "utf8"); -} - -function maybeRead(segments) { - const fullPath = path.join(repo, ...segments); - if (!fs.existsSync(fullPath)) return null; - return fs.readFileSync(fullPath, "utf8"); -} - -function expect(source, name, pattern) { - assert.match(source, pattern, `missing hostile-input evidence: ${name}`); -} - -const coreSource = read("crates/clusterflux-core/src/source.rs"); -const coreCapabilities = read("crates/clusterflux-core/src/capability.rs"); -const coordinatorService = [ - read("crates/clusterflux-coordinator/src/service.rs"), - read("crates/clusterflux-coordinator/src/service/routing.rs"), - read("crates/clusterflux-coordinator/src/service/signed_nodes.rs"), - read("crates/clusterflux-coordinator/src/service/logs.rs"), - read("crates/clusterflux-coordinator/src/service/tests.rs"), -].join("\n"); -const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); -const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); -const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); -const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); - -for (const [name, pattern] of [ - ["source manifests validate shape", /pub fn validate_public_mvp\(&self\)[\s\S]*self\.validate_shape\(\)\?/], - ["source manifests reject invalid digests", /SourceManifestError::InvalidDigest/], - ["source manifests reject invalid custom providers", /SourceManifestError::InvalidProviderId/], - ["source manifests reject control characters", /DescriptionControlCharacter/], - ["source manifests reject coordinator checkout access", /CoordinatorCheckoutAccess/], - ["source manifests reject default source-byte upload", /CoordinatorReceivesSourceBytes/], -]) { - expect(coreSource, name, pattern); -} - -for (const [name, pattern] of [ - ["capability reports validate public shape", /pub fn validate_public_report\(&self\)/], - ["capability reports validate architecture labels", /InvalidArchitecture/], - ["capability reports validate OS labels", /InvalidOsLabel/], - ["capability reports validate source providers", /InvalidSourceProvider/], - ["source provider ids reject path traversal", /valid_source_provider_id/], -]) { - expect(coreCapabilities, name, pattern); -} - -for (const [name, pattern] of [ - ["coordinator task log tails are bounded", /MAX_TASK_LOG_TAIL_BYTES: usize = 256 \* 1024/], - ["coordinator validates reported stdout tails", /ReportTaskLog[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], - ["coordinator validates completed task stdout tails", /TaskCompleted[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], - ["coordinator rejects oversized log tail in unit coverage", /"x"\.repeat\(MAX_TASK_LOG_TAIL_BYTES \+ 1\)/], -]) { - expect(coordinatorService, name, pattern); -} - -for (const [name, pattern] of [ - ["service rejects malformed node capability report", /fn service_rejects_malformed_node_capability_report\(\)/], - ["capability report rejection leaves descriptors empty", /assert!\(service\.node_descriptors\.is_empty\(\)\)/], - ["node capability report rejects cross-scope writes", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], - ["task completion rejects cross-scope writes", /task completion outside node scope|outside/], -]) { - expect(coordinatorService, name, pattern); -} - -for (const [name, source, patterns] of [ - [ - "artifact download smoke", - artifactDownloadSmoke, - [ - /const crossTenant = await send/, - /const crossProject = await send/, - /const guessed = await send/, - /const crossActorOpen = await send/, - /token is invalid/, - /artifact does not exist/, - ], - ], - [ - "operator panel smoke", - operatorPanelSmoke, - [ - /render_operator_panel/, - /submit_panel_event/, - /assert\(!JSON\.stringify\(panel\)\.includes\(" max_bytes[\s\S]*contains unsupported characters/], - ["tokens are bounded", /fn validate_token[\s\S]*value\.len\(\) > max_bytes[\s\S]*contains unsupported characters/], - ["control request bodies are bounded", /MAX_CONTROL_FRAME_BYTES[\s\S]*control request too large/], - ["identity protocol rejects unknown authority fields", /deny_unknown_fields/], - ]) { - expect(hostedService, name, pattern); - } - - for (const [name, pattern] of [ - ["old client identity protocol is rejected", /hosted_login_protocol_rejects_client_identity_and_provider_configuration/], - ["raw operator action is rejected", /rawOperatorDenied[\s\S]*hosted_operator_request envelope/], - ["unsigned client identity is rejected", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], - ["cross-tenant process inspection is rejected", /crossTenantTaskEventsDenied[\s\S]*scope\|denied\|unauthorized/], - ]) { - expect(hostedTests, name, pattern); - } -} - -console.log("Hostile input contract smoke passed"); diff --git a/scripts/migrate-clusterflux-state.sh b/scripts/migrate-clusterflux-state.sh deleted file mode 100755 index 356091d..0000000 --- a/scripts/migrate-clusterflux-state.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -migrate_path() { - local source="$1" - local destination="$2" - if [[ ! -e "$source" ]]; then - return - fi - if [[ -e "$destination" ]]; then - printf 'refusing to overwrite existing destination: %s\n' "$destination" >&2 - exit 1 - fi - mv -- "$source" "$destination" - printf 'migrated %s -> %s\n' "$source" "$destination" -} - -migrate_path "${HOME}/.disasmer" "${HOME}/.clusterflux" -migrate_path "${XDG_CONFIG_HOME:-${HOME}/.config}/disasmer" \ - "${XDG_CONFIG_HOME:-${HOME}/.config}/clusterflux" -migrate_path "${PWD}/.disasmer" "${PWD}/.clusterflux" -migrate_path "${PWD}/disasmer.toml" "${PWD}/clusterflux.toml" diff --git a/scripts/node-attach-smoke.js b/scripts/node-attach-smoke.js deleted file mode 100755 index c7c9dc2..0000000 --- a/scripts/node-attach-smoke.js +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); - -const repo = path.resolve(__dirname, ".."); -const identities = new Map(); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function nodeIdentity(node) { - const existing = identities.get(node); - if (existing) return existing; - const { privateKey: privateKeyObject, publicKey } = - crypto.generateKeyPairSync("ed25519"); - const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); - const publicDer = publicKey.export({ - format: "der", - type: "spki", - }); - const privateSeed = Buffer.from(privateDer).subarray(-32); - const identity = { - privateKey: `ed25519:${privateSeed.toString("base64")}`, - publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, - privateKeyObject, - }; - identities.set(node, identity); - return identity; -} - -function nodeSignatureMessage( - node, - requestKind, - payloadDigest, - nonce, - issuedAtEpochSeconds -) { - const parts = [ - "clusterflux-node-request-signature:v2", - node, - requestKind, - payloadDigest, - nonce, - String(issuedAtEpochSeconds), - ]; - return Buffer.concat( - parts.flatMap((part) => [ - Buffer.from(`${Buffer.byteLength(part)}:`), - Buffer.from(part), - Buffer.from("\n"), - ]) - ); -} - -function signedNodeHeartbeat(tenant, project, node, identity) { - const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`; - const issuedAt = Math.floor(Date.now() / 1000); - const payloadDigest = `sha256:${crypto - .createHash("sha256") - .update(JSON.stringify({ node, project, tenant, type: "node_heartbeat" })) - .digest("hex")}`; - const signature = crypto.sign( - null, - nodeSignatureMessage(node, "node_heartbeat", payloadDigest, nonce, issuedAt), - identity.privateKeyObject - ); - return { - nonce, - issued_at_epoch_seconds: issuedAt, - signature: `ed25519:${signature.toString("base64")}`, - }; -} - -function runAttach(addr, grant) { - const identity = nodeIdentity("node-attach"); - return new Promise((resolve, reject) => { - const child = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - "node", - "attach", - "--coordinator", - `${addr.host}:${addr.port}`, - "--tenant", - "tenant", - "--project-id", - "project", - "--node", - "node-attach", - "--public-key", - identity.publicKey, - "--enrollment-grant", - grant, - "--cap", - "quic-direct", - "--json", - ], - { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, - }, - } - ); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - child.on("exit", (code) => { - if (code !== 0) { - reject(new Error(`node attach failed with code ${code}\n${stderr}`)); - return; - } - try { - resolve(JSON.parse(stdout)); - } catch (error) { - reject( - new Error(`node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) - ); - } - }); - }); -} - -(async () => { - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback", - ], - { cwd: repo } - ); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const grant = await send(addr, { - type: "create_node_enrollment_grant", - tenant: "tenant", - project: "project", - actor_user: "operator", - ttl_seconds: 900 - }); - assert.strictEqual(grant.type, "node_enrollment_grant_created"); - assert.strictEqual(grant.tenant, "tenant"); - assert.strictEqual(grant.project, "project"); - assert.match(grant.grant, /^node_grant_[A-Za-z0-9_-]+$/); - assert.strictEqual(grant.scope, "node:attach"); - assert(grant.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); - assert(grant.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 900); - - const report = await runAttach(addr, grant.grant); - assert.strictEqual(report.plan.node, "node-attach"); - assert.strictEqual(report.plan.coordinator, `${addr.host}:${addr.port}`); - assert.strictEqual(report.plan.enrollment.grant, grant.grant); - assert.match(report.plan.enrollment.public_key_fingerprint, /^sha256:[0-9a-f]{64}$/); - assert.strictEqual( - report.plan.enrollment.exchanges_short_lived_grant_for_long_lived_node_identity, - true - ); - assert.ok(report.plan.capabilities.arch.length > 0); - assert.ok(report.plan.capabilities.source_providers.includes("filesystem")); - assert.ok(report.plan.capabilities.capabilities.includes("QuicDirect")); - assert.strictEqual(report.plan.detection.auto_detected, true); - assert.strictEqual(report.plan.detection.arch, report.plan.capabilities.arch); - assert.deepStrictEqual(report.plan.detection.manual_capability_overrides, ["quic-direct"]); - assert( - report.plan.detection.recognized_capability_overrides.includes("QuicDirect") - ); - assert.strictEqual( - report.plan.detection.os_arch_capabilities_require_manual_flags, - false - ); - assert.strictEqual(report.plan.detection.command_backend, "native-command"); - assert.strictEqual(report.plan.detection.command_backend_available, true); - assert( - report.plan.detection.source_provider_backends.some( - (provider) => provider.provider === "filesystem" && provider.detected - ) - ); - assert( - report.grant_disclosures.length > 0, - "node attach should disclose capability grants before reporting capabilities" - ); - assert( - report.grant_disclosures.every( - (disclosure) => disclosure.coordinator_policy_limited === true - ), - "node attach should mark all capability grants as coordinator-policy-limited" - ); - assert( - report.grant_disclosures.some( - (disclosure) => disclosure.grant === "native_command_execution" - ), - "node attach should disclose native command execution when detected" - ); - assert( - report.grant_disclosures.some( - (disclosure) => disclosure.grant === "source_access" - ), - "node attach should disclose source access when detected" - ); - assert.strictEqual(report.boundary.cli_contacted_coordinator, true); - assert.strictEqual(report.boundary.used_enrollment_exchange, true); - assert.strictEqual(report.boundary.coordinator_session_requests, 3); - assert.strictEqual(report.coordinator_response.type, "node_enrollment_exchanged"); - assert.strictEqual(report.coordinator_response.node, "node-attach"); - assert.strictEqual(report.coordinator_response.credential.node, "node-attach"); - assert.strictEqual(report.coordinator_response.credential.scope, "node:attach"); - assert.strictEqual(report.coordinator_response.credential.credential_kind, "NodeCredential"); - assert.match( - report.coordinator_response.credential.capability_policy_digest, - /^sha256:[0-9a-f]{64}$/ - ); - assert.strictEqual(report.heartbeat_response.type, "node_heartbeat"); - assert.strictEqual(report.capability_response.type, "node_capabilities_recorded"); - - const heartbeat = await send(addr, { - type: "node_heartbeat", - tenant: "tenant", - project: "project", - node: "node-attach", - node_signature: signedNodeHeartbeat( - "tenant", - "project", - "node-attach", - nodeIdentity("node-attach") - ), - }); - assert.strictEqual(heartbeat.type, "node_heartbeat"); - assert.strictEqual(heartbeat.node, "node-attach"); - - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("Node attach smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/node-lifecycle-contract-smoke.js b/scripts/node-lifecycle-contract-smoke.js deleted file mode 100755 index 78d4387..0000000 --- a/scripts/node-lifecycle-contract-smoke.js +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); - -function read(relativePath) { - return fs.readFileSync(path.join(repo, relativePath), "utf8"); -} - -function expect(source, name, pattern) { - assert.match(source, pattern, `missing node lifecycle evidence: ${name}`); -} - -const nodeMain = read("crates/clusterflux-node/src/daemon.rs"); -const nodeIdentity = read("crates/clusterflux-node/src/node_identity.rs"); -const cliNode = read("crates/clusterflux-cli/src/node.rs"); -const nodeTaskReports = read("crates/clusterflux-node/src/task_reports.rs"); -const nodeDebugAgent = read("crates/clusterflux-node/src/debug_agent.rs"); -const nodeLib = read("crates/clusterflux-node/src/lib.rs"); -const sharedWasmtimeRuntime = `${read("crates/clusterflux-wasm-runtime/src/lib.rs")}\n${read("crates/clusterflux-wasm-runtime/src/task_host_linker.rs")}`; -const nodeRuntimeSurface = `${nodeLib}\n${sharedWasmtimeRuntime}`; -const nodeLifecycleSurface = `${nodeMain}\n${nodeIdentity}\n${nodeTaskReports}\n${nodeDebugAgent}`; -const nodeAssignmentRunner = `${read("crates/clusterflux-node/src/assignment_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/control_watcher.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/process_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/validation.rs")}`; -const coordinatorCore = read("crates/clusterflux-coordinator/src/lib.rs"); -const coordinatorService = `${read("crates/clusterflux-coordinator/src/service.rs")}\n${read("crates/clusterflux-coordinator/src/service/routing.rs")}`; -const coordinatorServiceTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); -const coordinatorServiceSurface = `${coordinatorService}\n${coordinatorServiceTests}`; -const cliLocalRunSmoke = read("scripts/cli-local-run-smoke.js"); -const liveSmoke = read("scripts/cli-happy-path-live-smoke.js"); -const wasmtimeSmoke = read("scripts/wasmtime-node-smoke.js"); -const debugCore = read("crates/clusterflux-core/src/debug.rs"); - -assert.strictEqual( - (nodeMain.match(/CoordinatorSession::connect/g) || []).length, - 1, - "node runtime should open one coordinator session in the local process-boundary runtime" -); - -for (const [name, pattern] of [ - ["enrollment exchange over session", /"type": "exchange_node_enrollment_grant"/], - ["persisted node identity is reused locally", /"type": "node_identity_reused"/], - ["heartbeat over session", /"type": "node_heartbeat"/], - ["node-originated requests use signed envelope", /"type": "signed_node"/], - ["capability report over session", /"type": "report_node_capabilities"/], - ["task assignment polling over session", /"type": "poll_task_assignment"/], - ["process start over session", /"type": "start_process"/], - ["reconnect over session", /"type": "reconnect_node"/], - ["debug command polling over session", /"type": "poll_debug_command"/], - ["log event over session", /"type": "report_task_log"/], - ["VFS metadata over session", /"type": "report_vfs_metadata"/], - ["task control polling over session", /"type": "poll_task_control"/], - ["completion over session", /"type": "task_completed"/], - ["cancellation uses same session", /poll_task_cancellation\(session, args, &task, node_private_key\)/], - ["request count is reported", /session\.requests\(\)/], -]) { - expect(nodeLifecycleSurface, name, pattern); -} - -expect(cliNode, "user-authorized node attach over Client session", /"type": "attach_node"/); -assert.doesNotMatch( - nodeIdentity, - /"type": "attach_node"/, - "a persisted node must authenticate with its signed identity instead of replaying Client attach" -); - -const runtimeAcceptanceSurface = `${cliLocalRunSmoke}\n${liveSmoke}\n${coordinatorServiceTests}\n${nodeLib}\n${nodeAssignmentRunner}`; -for (const [name, pattern] of [ - ["real CLI launches a coordinator main", /node_report\.run\.status, "main_launched"/], - ["real CLI observes coordinator-main task spawning", /task_spawn_host_import, true/], - ["real CLI keeps child task instances distinct", /every live task event must retain its unique instance identity/], - ["signed active Wasm parent spawns and joins child", /fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only\(\)/], - ["controlled native process abort is exercised", /fn abort_requested[\s\S]*poll_task_control/], - ["native lifecycle freeze and resume are exercised", /linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume/], - ["strict live run requires a usable partial freeze", /partial_freeze\.partially_frozen/], - ["strict live run requires hosted restart evidence", /serviceRestart\.executed === true/], -]) { - expect(runtimeAcceptanceSurface, name, pattern); -} - -for (const [name, pattern] of [ - ["cooperative cancellation and abort are distinct", /cancel_requested: false,[\s\S]*abort_requested: true/], - ["controlled runner polls abort while command runs", /fn abort_requested[\s\S]*poll_task_control/], - ["controlled runner creates a process group", /process\.process_group\(0\)/], - ["controlled runner freezes the native process group", /libc::kill\(process_group, libc::SIGSTOP\)/], - ["controlled runner resumes the native process group", /libc::kill\(process_group, libc::SIGCONT\)/], - ["controlled runner kills the process group", /libc::kill\(process_group, libc::SIGKILL\)/], - ["Wasm code can poll cooperative cancellation", /task_control_v1/], - ["matched Wasm probes remain at a quiescent boundary", /TaskHostOperation::DebugProbe[\s\S]*enter_quiescent_host_boundary[\s\S]*leave_quiescent_host_boundary/], - ["debug snapshots use the live Wasm task handle registry", /debug_handle_snapshot[\s\S]*task_handle_\{handle_id\}[\s\S]*state=active/], - ["native command status comes from the controlled runner", /set_command_status[\s\S]*frozen command pid[\s\S]*native command exited with status/], -]) { - expect(`${coordinatorServiceSurface}\n${nodeLifecycleSurface}\n${sharedWasmtimeRuntime}\n${nodeAssignmentRunner}`, name, pattern); -} - -for (const [name, pattern] of [ - ["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/], - ["reconnect preserves scoped enrolled node identity", /reconnect_node\([\s\S]*&TenantId::from\("tenant"\),[\s\S]*&ProjectId::from\("project"\),[\s\S]*&NodeId::from\("node"\),[\s\S]*None/], - ["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/], -]) { - expect(coordinatorCore, name, pattern); -} - -for (const [name, pattern] of [ - ["coordinator delivers cancellation to connected node", /fn service_delivers_cancellation_to_connected_node_and_records_terminal_state\(\)/], - ["node polls task control", /CoordinatorRequest::PollTaskControl/], - ["cancelled terminal state is recorded", /TaskTerminalState::Cancelled/], -]) { - expect(coordinatorServiceSurface, name, pattern); -} - -for (const [name, pattern] of [ - ["native lifecycle test exists", /fn linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume\(\)/], - ["native freeze succeeds when supported", /lifecycle\.freeze_for_debug_epoch\(\)\.unwrap\(\)/], - ["native resume succeeds", /lifecycle\.resume_after_debug_epoch\(\)/], - ["native cancel reaches lifecycle", /lifecycle\.cancel\(\)/], - ["unsupported freeze errors", /BackendError::DebugFreezeUnsupported/], - ["wasmtime runtime exposes freeze resume probe", /pub fn freeze_resume_i32_export_probe/], - ["wasmtime runtime captures Wasm frame locals", /debug_i32_export_snapshot[\s\S]*local_values/], - ["wasmtime runtime creates Wasm debug participant", /kind: DebugParticipantKind::WasmTask/], - ["wasmtime debug participant carries local values", /local_values: snapshot\.local_values\.clone\(\)/], - ["wasmtime runtime resumes after freeze", /epoch\.continue_all\(\)/], -]) { - expect(nodeRuntimeSurface, name, pattern); -} - -for (const [name, pattern] of [ - ["wasmtime smoke runs debug freeze resume mode", /--debug-freeze-resume/], - ["wasmtime smoke verifies frozen state", /debugReport\.frozen_state, "Frozen"/], - ["wasmtime smoke verifies resumed state", /debugReport\.resumed_state, "Running"/], - ["wasmtime smoke verifies frame local values", /debugReport\.local_values[\s\S]*wasm_local_0/], - ["wasmtime smoke proves node runtime reached wasm task", /node_runtime_reached_wasm_task/], - ["wasmtime smoke proves node captured locals", /node_runtime_captured_wasm_locals/], -]) { - expect(wasmtimeSmoke, name, pattern); -} - -for (const [name, pattern] of [ - ["debug model freezes wasm and command participants", /fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks\(\)/], - ["debug model rejects unsupported freeze", /fn debug_epoch_reports_failure_when_no_participant_can_freeze\(\)/], - ["debug model resumes frozen participants", /fn continue_resumes_every_frozen_participant\(\)/], - ["debug model includes captured locals", /local_values/], - ["wasm participants are modeled", /DebugParticipantKind::WasmTask/], - ["controlled native command participants are modeled", /DebugParticipantKind::ControlledNativeCommand/], -]) { - expect(debugCore, name, pattern); -} - -console.log("Node lifecycle contract smoke passed"); diff --git a/scripts/node-signing.js b/scripts/node-signing.js deleted file mode 100644 index 3069840..0000000 --- a/scripts/node-signing.js +++ /dev/null @@ -1,181 +0,0 @@ -const crypto = require("crypto"); -const identities = new Map(); - -function nodeIdentity(identityPurpose, node) { - const identityKey = `${identityPurpose}:${node}`; - const existing = identities.get(identityKey); - if (existing) return existing; - const { privateKey: privateKeyObject, publicKey } = - crypto.generateKeyPairSync("ed25519"); - const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); - const publicDer = publicKey.export({ - format: "der", - type: "spki", - }); - const privateSeed = Buffer.from(privateDer).subarray(-32); - const identity = { - privateKey: `ed25519:${privateSeed.toString("base64")}`, - publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, - privateKeyObject, - }; - identities.set(identityKey, identity); - return identity; -} - -function nodeIdentityFromPrivateKey(privateKey) { - if (typeof privateKey !== "string" || !privateKey.startsWith("ed25519:")) { - throw new Error("node private key must use ed25519: encoding"); - } - const seed = Buffer.from(privateKey.slice("ed25519:".length), "base64"); - if (seed.length !== 32) throw new Error("node private key must contain 32 bytes"); - const privateKeyObject = crypto.createPrivateKey({ - key: Buffer.concat([ - Buffer.from("302e020100300506032b657004220420", "hex"), - seed, - ]), - format: "der", - type: "pkcs8", - }); - const publicKeyObject = crypto.createPublicKey(privateKeyObject); - const publicDer = publicKeyObject.export({ format: "der", type: "spki" }); - return { - privateKey, - publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, - privateKeyObject, - }; -} - -function canonicalSignedRequest(value, topLevel = true) { - if (Array.isArray(value)) { - return value.map((entry) => canonicalSignedRequest(entry, false)); - } - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value) - .filter( - ([key, entry]) => - entry !== null && - (!topLevel || !["agent_signature", "node_signature"].includes(key)) - ) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([key, entry]) => [key, canonicalSignedRequest(entry, false)]) - ); - } - return value; -} - -function withWireDefaults(request) { - const value = { ...request }; - if (value.type === "report_node_capabilities") { - value.dependency_cache_digests ??= []; - } else if (value.type === "launch_task" || value.type === "launch_child_task") { - value.wait_for_node ??= false; - if (value.task_spec && typeof value.task_spec === "object") { - value.task_spec = { ...value.task_spec }; - value.task_spec.failure_policy ??= "fail_fast"; - } - } else if (value.type === "start_process") { - value.restart ??= false; - } else if (value.type === "report_debug_state") { - value.stack_frames ??= []; - value.local_values ??= []; - value.task_args ??= []; - value.handles ??= []; - value.recent_output ??= []; - } else if (value.type === "report_task_log") { - value.stdout_tail ??= ""; - value.stderr_tail ??= ""; - } else if (value.type === "task_completed") { - value.stdout_tail ??= ""; - value.stderr_tail ??= ""; - value.stdout_truncated ??= false; - value.stderr_truncated ??= false; - } - return value; -} - -function signedRequestPayloadDigest(request) { - return `sha256:${crypto - .createHash("sha256") - .update(JSON.stringify(canonicalSignedRequest(withWireDefaults(request)))) - .digest("hex")}`; -} - -function nodeSignatureMessage( - node, - requestKind, - payloadDigest, - nonce, - issuedAtEpochSeconds -) { - const parts = [ - "clusterflux-node-request-signature:v2", - node, - requestKind, - payloadDigest, - nonce, - String(issuedAtEpochSeconds), - ]; - return Buffer.concat( - parts.flatMap((part) => [ - Buffer.from(`${Buffer.byteLength(part)}:`), - Buffer.from(part), - Buffer.from("\n"), - ]) - ); -} - -function signedNodeProof(node, identity, requestKind, request, options = {}) { - const nonce = - options.nonce ?? - `${requestKind}-${process.pid}-${Date.now()}-${crypto - .randomBytes(8) - .toString("hex")}`; - const issuedAt = - options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000); - const signature = crypto.sign( - null, - nodeSignatureMessage( - node, - requestKind, - signedRequestPayloadDigest(request), - nonce, - issuedAt - ), - identity.privateKeyObject - ); - return { - nonce, - issued_at_epoch_seconds: issuedAt, - signature: `ed25519:${signature.toString("base64")}`, - }; -} - -function signedNodeHeartbeat(tenant, project, node, identity, options = {}) { - const request = { type: "node_heartbeat", tenant, project, node }; - return signedNodeProof(node, identity, "node_heartbeat", request, options); -} - -function signedNodeRequest(node, identity, requestKind, request, options = {}) { - return { - type: "signed_node", - node, - node_signature: signedNodeProof( - node, - identity, - requestKind, - request, - options - ), - request, - }; -} - -module.exports = { - nodeIdentity, - nodeIdentityFromPrivateKey, - signedNodeProof, - signedRequestPayloadDigest, - signedNodeHeartbeat, - signedNodeRequest, -}; diff --git a/scripts/operator-panel-smoke.js b/scripts/operator-panel-smoke.js deleted file mode 100755 index b817c42..0000000 --- a/scripts/operator-panel-smoke.js +++ /dev/null @@ -1,385 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const path = require("path"); -const { nodeIdentity } = require("./node-signing"); -const { - ensureRootlessPodman, - repo, - runFlagshipWorker, - send, - startFlagship, - waitForTaskEvent, - waitForJsonLine, - waitForNodeStatus, -} = require("./real-flagship-harness"); - -const panelNode = "panel-node"; -const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode); -const panelProject = path.join(repo, "tests/fixtures/runtime-conformance"); - -function widget(panel, id) { - const item = panel.widgets[id]; - assert(item, `missing panel widget ${id}`); - return item; -} - -const delay = (milliseconds) => - new Promise((resolve) => setTimeout(resolve, milliseconds)); - -async function waitForBreakpointHit(addr, process) { - for (let attempt = 0; attempt < 2400; attempt += 1) { - const status = await send(addr, { - type: "inspect_debug_breakpoints", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - }); - if (status.type !== "debug_breakpoints") { - const events = await send(addr, { - type: "list_task_events", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - }); - throw new Error( - `breakpoint state disappeared: ${JSON.stringify({ status, events })}` - ); - } - if (status.hit_epoch != null) return status; - await delay(25); - } - throw new Error(`timed out waiting for package breakpoint in ${process}`); -} - -async function waitForDebugEpochFrozen(addr, process, epoch) { - for (let attempt = 0; attempt < 2400; attempt += 1) { - const status = await send(addr, { - type: "inspect_debug_epoch", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - epoch, - }); - assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status)); - if (status.failed) { - throw new Error(status.failure_messages.join("; ")); - } - if (status.fully_frozen) return status; - await delay(25); - } - throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`); -} - -(async () => { - ensureRootlessPodman(); - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - let coordinatorStderr = ""; - let worker; - coordinator.stderr.on("data", (chunk) => { - coordinatorStderr += chunk.toString(); - }); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - const projectCreated = await send(addr, { - type: "create_project", - tenant: "tenant", - actor_user: "user", - project: "project", - name: "Operator panel smoke", - }); - assert.strictEqual(projectCreated.type, "project_created"); - - worker = await runFlagshipWorker( - addr, - panelNode, - panelNodeIdentity, - panelProject - ); - const workerReady = await worker.ready; - assert.strictEqual(workerReady.node_status, "ready"); - const workerCompletion = waitForNodeStatus(worker.child, "completed"); - const flagship = startFlagship(addr, panelProject); - const configuredBreakpoints = await send(addr, { - type: "set_debug_breakpoints", - tenant: "tenant", - project: "project", - actor_user: "user", - process: flagship.process, - probe_symbols: ["clusterflux.probe.package_release"], - }); - assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints"); - await waitForTaskEvent( - addr, - flagship.process, - (event) => event.task_definition === "prepare_source", - "prepare_source after breakpoint configuration" - ); - const breakpointHit = await waitForBreakpointHit(addr, flagship.process); - assert.strictEqual( - breakpointHit.hit_probe_symbol, - "clusterflux.probe.package_release" - ); - const frozenEpoch = await waitForDebugEpochFrozen( - addr, - flagship.process, - breakpointHit.hit_epoch - ); - assert(frozenEpoch.acknowledgements.length >= 2); - const compileEvent = await waitForTaskEvent( - addr, - flagship.process, - (event) => event.task_definition === "compile_linux", - "compile before the package breakpoint" - ); - const report = await workerCompletion; - assert.strictEqual(report.node_status, "completed"); - assert.strictEqual(report.coordinator_response.type, "task_recorded"); - const process = flagship.process; - - const rendered = await send(addr, { - type: "render_operator_panel", - tenant: "tenant", - project: "project", - process, - actor_user: "user", - max_download_bytes: 1024 * 1024, - stopped: false - }); - assert.strictEqual(rendered.type, "operator_panel"); - const panel = rendered.panel; - assert.strictEqual(panel.tenant, "tenant"); - assert.strictEqual(panel.project, "project"); - assert.strictEqual(panel.process, process); - assert.strictEqual(panel.program_ui_events_enabled, true); - - assert.deepStrictEqual(widget(panel, "process-status").kind, { - Text: { value: "running" } - }); - const taskProgress = widget(panel, "task-progress").kind.Progress; - assert(taskProgress.current >= 2); - assert.strictEqual(taskProgress.current, taskProgress.total); - const taskSummary = widget(panel, "task-summary").kind.Text.value; - assert.match( - taskSummary, - new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`) - ); - assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/); - const downloadWidget = widget(panel, "download-artifact").kind; - const artifact = downloadWidget.ArtifactDownload.artifact; - assert( - artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length), - "panel download must point at a real flagship artifact" - ); - assert(!JSON.stringify(downloadWidget).includes("url_path")); - assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest")); - assert.deepStrictEqual(widget(panel, "debug-process").kind, { - Button: { action: "debug-process" } - }); - assert.deepStrictEqual(widget(panel, "cancel-process").kind, { - Button: { action: "cancel-process" } - }); - assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, { - Button: { action: "restart-task" } - }); - assert(panel.control_plane_actions.includes("DebugProcess")); - assert(panel.control_plane_actions.includes("CancelProcess")); - const restartTarget = panel.control_plane_actions.find( - (action) => action.RestartTask - )?.RestartTask; - assert( - restartTarget && taskSummary.includes(`[${restartTarget}]`), - "panel restart must target the real flagship task instance" - ); - assert( - panel.control_plane_actions.some( - (action) => action.DownloadArtifact === artifact - ) - ); - assert(!JSON.stringify(panel).includes(" action.DownloadArtifact === artifact - ) - ); - - const frozenEvent = await send(addr, { - type: "submit_panel_event", - tenant: "tenant", - project: "project", - process, - widget_id: "debug-process", - kind: "ButtonClicked", - max_events: 10 - }); - assert.strictEqual(frozenEvent.type, "error"); - assert.match(frozenEvent.message, /program UI events are disabled/i); - - const crossTenant = await send(addr, { - type: "render_operator_panel", - tenant: "other", - project: "project", - process, - actor_user: "user", - max_download_bytes: 1024 * 1024, - stopped: false - }); - assert.strictEqual(crossTenant.type, "error"); - assert.match( - crossTenant.message, - /scope|tenant|project|requires an active virtual process/i - ); - assert(!crossTenant.message.includes(process)); - - const resumed = await send(addr, { - type: "resume_debug_epoch", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - epoch: breakpointHit.hit_epoch, - }); - assert.strictEqual(resumed.type, "debug_epoch"); - const cleanup = await send(addr, { - type: "abort_process", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - }); - assert.strictEqual(cleanup.type, "process_aborted"); - } catch (error) { - if (coordinatorStderr) { - error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; - } - throw error; - } finally { - worker?.child.kill("SIGTERM"); - coordinator.kill("SIGTERM"); - } - - console.log("Operator panel smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/podman-backend-smoke.js b/scripts/podman-backend-smoke.js deleted file mode 100755 index 438c2df..0000000 --- a/scripts/podman-backend-smoke.js +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const path = require("path"); -const { configurePodmanTestEnvironment } = require("./podman-test-env"); - -const repo = path.resolve(__dirname, ".."); -const baseImage = "docker.io/library/alpine:3.20"; - -// Nix's standalone Podman package may not install the distribution-level -// containers/image policy normally found under /etc. Use an isolated test HOME -// without replacing a policy supplied by the host. -configurePodmanTestEnvironment(repo); - -function run(command, args, options = {}) { - return cp.execFileSync(command, args, { - cwd: repo, - encoding: "utf8", - stdio: options.stdio || ["ignore", "pipe", "pipe"], - }); -} - -function incomplete(reason) { - const error = new Error(`Linux Podman backend incomplete: ${reason}`); - error.code = "CLUSTERFLUX_PODMAN_INCOMPLETE"; - throw error; -} - -function ensurePodmanBaseImage() { - try { - run("podman", ["--version"]); - } catch (error) { - incomplete(`podman command is unavailable (${error.message})`); - } - - let rootless; - try { - rootless = run("podman", ["info", "--format", "{{.Host.Security.Rootless}}"]).trim(); - } catch (error) { - incomplete(`podman info did not report rootless status (${error.message})`); - } - if (rootless !== "true") { - incomplete(`podman is not running in rootless mode (reported ${JSON.stringify(rootless)})`); - } - - try { - run("podman", ["image", "exists", baseImage]); - } catch (_) { - try { - run("podman", ["pull", baseImage], { stdio: "inherit" }); - } catch (error) { - incomplete(`unable to make ${baseImage} available (${error.message})`); - } - } -} - -try { - ensurePodmanBaseImage(); - - const stdout = run("cargo", [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-podman-smoke" - ]); - const report = JSON.parse(stdout.trim().split("\n").at(-1)); - - assert.strictEqual(report.podman_status, "completed"); - assert.strictEqual(report.status_code, 0); - assert.strictEqual(report.stdout, "podman-ok:node-local source\n"); - assert.strictEqual(report.large_bytes_uploaded, false); - assert.strictEqual(report.uses_full_repo_tarball, false); - assert.strictEqual(report.coordinator_routed_file_reads, false); - assert.strictEqual(report.staged_artifact.path, "/vfs/artifacts/podman-smoke.txt"); - - console.log("Podman backend smoke passed"); -} catch (error) { - if (error.code === "CLUSTERFLUX_PODMAN_INCOMPLETE") { - console.error(error.message); - process.exit(2); - } - throw error; -} diff --git a/scripts/podman-test-env.js b/scripts/podman-test-env.js deleted file mode 100644 index 2c0dc3b..0000000 --- a/scripts/podman-test-env.js +++ /dev/null @@ -1,41 +0,0 @@ -const fs = require("fs"); -const os = require("os"); -const path = require("path"); - -function configurePodmanTestEnvironment(repo) { - const originalHome = os.homedir(); - if ( - fs.existsSync(path.join(originalHome, ".config/containers/policy.json")) || - fs.existsSync("/etc/containers/policy.json") - ) { - return; - } - - const isolatedHome = path.join(repo, "scripts/containers-home"); - const policy = path.join(isolatedHome, ".config/containers/policy.json"); - fs.mkdirSync(path.dirname(policy), { recursive: true }); - if (!fs.existsSync(policy)) { - fs.writeFileSync( - policy, - `${JSON.stringify( - { default: [{ type: "insecureAcceptAnything" }] }, - null, - 2 - )}\n` - ); - } - - process.env.CARGO_HOME ||= path.join(originalHome, ".cargo"); - process.env.RUSTUP_HOME ||= path.join(originalHome, ".rustup"); - process.env.HOME = isolatedHome; - process.env.XDG_DATA_HOME = path.join( - os.tmpdir(), - `clusterflux-containers-data-${process.getuid?.() ?? "user"}` - ); - process.env.XDG_CACHE_HOME = path.join( - os.tmpdir(), - `clusterflux-containers-cache-${process.getuid?.() ?? "user"}` - ); -} - -module.exports = { configurePodmanTestEnvironment }; diff --git a/scripts/prepare-public-release.js b/scripts/prepare-public-release.js deleted file mode 100755 index 27fa754..0000000 --- a/scripts/prepare-public-release.js +++ /dev/null @@ -1,1283 +0,0 @@ -#!/usr/bin/env node - -const crypto = require("crypto"); -const cp = require("child_process"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const outputRoot = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/public-release") -); -const publicTree = path.join(outputRoot, "public-tree"); -const assetsDir = path.join(outputRoot, "assets"); -const stagingDir = path.join(outputRoot, "staging"); -const publicBuildTarget = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_BUILD_TARGET_DIR || - path.join(outputRoot, "cargo-target") -); -const hostedBuildTarget = path.resolve( - process.env.CLUSTERFLUX_HOSTED_BUILD_TARGET_DIR || - path.join(outputRoot, "hosted-cargo-target") -); -const candidateManifestPath = process.env.CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST - ? path.resolve(process.env.CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST) - : null; -const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; -const forgejoHost = "git.michelpaulissen.com"; -const args = new Set(process.argv.slice(2)); -const includeForgejoWorkflows = - args.has("--include-forgejo-workflows") || - /^(1|true|yes)$/i.test(process.env.CLUSTERFLUX_INCLUDE_FORGEJO_WORKFLOWS || ""); -const filteredTopLevel = ["private", "internal", "experiments", ".git", "target"]; -const filteredDirectoryNames = [".clusterflux"]; -const archiveIgnoredPathFallbacks = [ - "target", - ".clusterflux", - "vscode-extension/node_modules", - "scripts/containers-home", -]; -const publicRepoBranch = process.env.CLUSTERFLUX_PUBLIC_REPO_BRANCH || "main"; -const publicBinaries = [ - "clusterflux", - "clusterflux-coordinator", - "clusterflux-node", - "clusterflux-debug-dap", -]; -const hostedBinary = "clusterflux-hosted-service"; - -function commandOutput(command, args, options = {}) { - try { - const execOptions = { - cwd: repo, - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - ...options, - }; - execOptions.env = commandEnv(command, options.env); - return cp - .execFileSync(command, args, execOptions) - .trim(); - } catch (_) { - return null; - } -} - -function run(command, args, options = {}) { - const execOptions = { - cwd: repo, - stdio: "inherit", - ...options, - }; - execOptions.env = commandEnv(command, options.env); - cp.execFileSync(command, args, execOptions); -} - -function nonInteractiveGitEnv(extra = {}) { - return { - ...process.env, - GIT_TERMINAL_PROMPT: "0", - GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", - SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", - GIT_SSH_COMMAND: - process.env.GIT_SSH_COMMAND || - "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", - ...extra, - }; -} - -function commandEnv(command, env) { - if (command !== "git") { - return env; - } - return nonInteractiveGitEnv(env); -} - -function ensureDir(dir) { - fs.mkdirSync(dir, { recursive: true }); -} - -function filteredOutPatterns() { - return [ - "private/**", - "internal/**", - "experiments/**", - ".git", - "target", - "git-ignored source paths", - "root/*.md except README.md", - "**/.clusterflux/**", - ...(includeForgejoWorkflows ? [] : [".forgejo/**"]), - ]; -} - -function gitIgnoredSourcePaths() { - const output = commandOutput("git", [ - "ls-files", - "--others", - "--ignored", - "--exclude-standard", - "--directory", - "-z", - ]); - if (output === null) { - return archiveIgnoredPathFallbacks; - } - return [ - ...new Set([ - ...archiveIgnoredPathFallbacks, - ...output - .split("\0") - .filter(Boolean) - .map((relativePath) => - relativePath.replaceAll("\\", "/").replace(/\/$/, "") - ), - ]), - ]; -} - -function isGitIgnored(relativePath, ignoredSourcePaths) { - const normalized = relativePath.replaceAll(path.sep, "/"); - return ignoredSourcePaths.some( - (ignored) => normalized === ignored || normalized.startsWith(`${ignored}/`) - ); -} - -function isFilteredRootMarkdown(relativePath, entry) { - return ( - !relativePath.includes(path.sep) && - (entry.isFile() || entry.isSymbolicLink()) && - path.extname(entry.name).toLowerCase() === ".md" && - !["README.md", "SECURITY.md"].includes(entry.name) - ); -} - -function shouldFilter(entry, relativePath) { - const parts = relativePath.split(path.sep).filter(Boolean); - const topLevel = parts[0]; - if (filteredTopLevel.includes(topLevel)) return true; - if (parts.some((part) => filteredDirectoryNames.includes(part))) return true; - if (topLevel === ".forgejo" && !includeForgejoWorkflows) return true; - if (isFilteredRootMarkdown(relativePath, entry)) return true; - return false; -} - -function copyFilteredTree(src, dest, ignoredSourcePaths, relative = "") { - ensureDir(dest); - const entries = fs - .readdirSync(src, { withFileTypes: true }) - .sort((left, right) => left.name.localeCompare(right.name)); - - for (const entry of entries) { - const childRelative = relative ? path.join(relative, entry.name) : entry.name; - if ( - shouldFilter(entry, childRelative) || - isGitIgnored(childRelative, ignoredSourcePaths) - ) { - continue; - } - - const from = path.join(src, entry.name); - const to = path.join(dest, entry.name); - if (entry.isDirectory()) { - copyFilteredTree(from, to, ignoredSourcePaths, childRelative); - } else if (entry.isSymbolicLink()) { - fs.symlinkSync(fs.readlinkSync(from), to); - } else if (entry.isFile()) { - fs.copyFileSync(from, to); - fs.chmodSync(to, fs.statSync(from).mode & 0o777); - } - } -} - -function assertFilteredTree(ignoredSourcePaths) { - for (const excluded of ["private", "internal", "experiments"]) { - if (fs.existsSync(path.join(publicTree, excluded))) { - throw new Error(`${excluded}/ leaked into the public tree`); - } - } - for (const file of walkFiles(publicTree)) { - if (file.split(path.sep).includes(".clusterflux")) { - throw new Error(`generated .clusterflux view state leaked into the public tree: ${file}`); - } - } - if (!includeForgejoWorkflows && fs.existsSync(path.join(publicTree, ".forgejo"))) { - throw new Error(".forgejo/ leaked into the host-neutral public tree"); - } - if (!fs.existsSync(path.join(publicTree, "README.md"))) { - throw new Error("product README.md is missing from the public tree"); - } - for (const ignored of ignoredSourcePaths) { - if (fs.existsSync(path.join(publicTree, ...ignored.split("/")))) { - throw new Error(`Git-ignored source path leaked into the public tree: ${ignored}`); - } - } - for (const entry of fs.readdirSync(publicTree, { withFileTypes: true })) { - if (isFilteredRootMarkdown(entry.name, entry)) { - throw new Error(`internal root Markdown file leaked into the public tree: ${entry.name}`); - } - } -} - -function walkFiles(root, relative = "") { - const dir = path.join(root, relative); - const entries = fs - .readdirSync(dir, { withFileTypes: true }) - .sort((left, right) => left.name.localeCompare(right.name)); - const files = []; - for (const entry of entries) { - const childRelative = relative ? path.join(relative, entry.name) : entry.name; - if (entry.isDirectory()) { - files.push(...walkFiles(root, childRelative)); - } else if (entry.isFile()) { - files.push(childRelative); - } else if (entry.isSymbolicLink()) { - files.push(childRelative); - } - } - return files; -} - -function hashTree(root) { - const hash = crypto.createHash("sha256"); - for (const file of walkFiles(root)) { - const absolute = path.join(root, file); - const stat = fs.lstatSync(absolute); - hash.update(file.replaceAll(path.sep, "/")); - hash.update("\0"); - hash.update(String(stat.mode & 0o777)); - hash.update("\0"); - if (stat.isSymbolicLink()) { - hash.update("symlink"); - hash.update("\0"); - hash.update(fs.readlinkSync(absolute)); - } else { - hash.update(fs.readFileSync(absolute)); - } - hash.update("\0"); - } - return `sha256:${hash.digest("hex")}`; -} - -function sha256File(file) { - return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); -} - -function sha256Buffer(buffer) { - return crypto.createHash("sha256").update(buffer).digest("hex"); -} - -function sourceTreeDigest(sourceCommit) { - const archive = cp.execFileSync("git", ["archive", "--format=tar", sourceCommit], { - cwd: repo, - encoding: null, - maxBuffer: 128 * 1024 * 1024, - stdio: ["ignore", "pipe", "inherit"], - env: nonInteractiveGitEnv(), - }); - return `sha256:${sha256Buffer(archive)}`; -} - -function evidenceIdentity(value) { - const serialized = JSON.stringify(value === undefined ? null : value); - return `sha256:${sha256Buffer(Buffer.from(serialized, "utf8"))}`; -} - -function sanitizeEvidence(value, key = "") { - const secretKey = /(?:token|secret|password|cookie|authorization|private_key)/iu.test( - key - ); - if (secretKey && typeof value === "string") return "[redacted]"; - if (secretKey && Array.isArray(value)) return ["[redacted]"]; - if (Array.isArray(value)) { - return value.map((entry) => sanitizeEvidence(entry)); - } - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value).map(([childKey, childValue]) => [ - childKey, - sanitizeEvidence(childValue, childKey), - ]) - ); - } - return value; -} - -function renderFinalTranscript(result) { - const lines = [ - "Clusterflux final release transcript", - `Source commit: ${result.source_commit}`, - `Source tree digest: ${result.source_tree_digest}`, - `Public tree identity: ${result.public_tree_identity}`, - `Acceptance result recorded: ${result.timestamps.acceptance_result_recorded_at}`, - `Evidence packaged: ${result.timestamps.evidence_packaged_at}`, - `Deployment generation: ${result.deployment.system_generation}`, - `Deployment identity: ${result.deployment.identity}`, - `Configuration identity: ${result.configuration_identity}`, - "", - "Commands:", - ...result.release_commands.map((command) => `- ${command}`), - "", - "Binary digests:", - ...Object.entries(result.binary_digests).map( - ([name, digest]) => `- ${name}: ${digest}` - ), - "", - "Named scenarios:", - ...result.named_scenarios.map( - (scenario) => - `- ${scenario.id}: ${scenario.status} (${scenario.duration_ms} ms)` - ), - "", - "Strict requirement ledger:", - ...result.strict_requirement_ledger.map( - (requirement) => - `- ${requirement.id}: ${requirement.passed ? "passed" : "failed"}` - ), - "", - `Scenario skips: ${JSON.stringify(result.scenario_skips)}`, - `Acceptance result: ${result.acceptance_result}`, - "", - "Failure-injection and complete machine evidence:", - JSON.stringify(result, null, 2), - "", - ]; - return `${lines.join("\n")}\n`; -} - -function tarGz(output, cwd, inputs) { - run("tar", ["-czf", output, "-C", cwd, ...inputs]); -} - -function platformName() { - return `${os.platform()}-${os.arch()}`; -} - -function binaryName(name) { - return process.platform === "win32" ? `${name}.exe` : name; -} - -function buildPublicBinaries() { - run("cargo", ["build", "--locked", "--workspace", "--bins", "--release", "--jobs", "2"], { - cwd: publicTree, - env: { ...process.env, CARGO_TARGET_DIR: publicBuildTarget }, - }); -} - -function buildHostedBinary() { - run( - "cargo", - [ - "build", - "--locked", - "--manifest-path", - "private/hosted-policy/Cargo.toml", - "--bin", - hostedBinary, - "--release", - "--jobs", - "2", - ], - { - cwd: repo, - env: { ...process.env, CARGO_TARGET_DIR: hostedBuildTarget }, - } - ); -} - -function stageBinaryAssets(releaseName) { - const stageRoot = path.join(stagingDir, "binaries"); - const binDir = path.join(stageRoot, "bin"); - fs.rmSync(stageRoot, { recursive: true, force: true }); - ensureDir(binDir); - - for (const binary of publicBinaries) { - const fileName = binaryName(binary); - const built = path.join(publicBuildTarget, "release", fileName); - if (!fs.existsSync(built)) { - throw new Error(`expected release binary ${built}`); - } - const staged = path.join(binDir, fileName); - fs.copyFileSync(built, staged); - fs.chmodSync(staged, 0o755); - } - - const archive = path.join( - assetsDir, - `clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz` - ); - tarGz(archive, stageRoot, ["."]); - return archive; -} - -function stageHostedBinaryAsset(releaseName) { - const stageRoot = path.join(stagingDir, "hosted-binary"); - const binDir = path.join(stageRoot, "bin"); - fs.rmSync(stageRoot, { recursive: true, force: true }); - ensureDir(binDir); - const fileName = binaryName(hostedBinary); - const built = path.join(hostedBuildTarget, "release", fileName); - if (!fs.existsSync(built)) { - throw new Error(`expected hosted release binary ${built}`); - } - const staged = path.join(binDir, fileName); - fs.copyFileSync(built, staged); - fs.chmodSync(staged, 0o755); - const archive = path.join( - assetsDir, - `clusterflux-hosted-${releaseName}-${platformName()}.tar.gz` - ); - tarGz(archive, stageRoot, ["."]); - return archive; -} - -function publicBinaryDigests() { - return Object.fromEntries( - publicBinaries.map((binary) => { - const fileName = binaryName(binary); - const file = path.join(publicBuildTarget, "release", fileName); - if (!fs.existsSync(file)) throw new Error(`missing built release binary ${file}`); - return [fileName, `sha256:${sha256File(file)}`]; - }) - ); -} - -function candidateBinaryDigests() { - const fileName = binaryName(hostedBinary); - const file = path.join(hostedBuildTarget, "release", fileName); - if (!fs.existsSync(file)) throw new Error(`missing hosted release binary ${file}`); - return { - ...publicBinaryDigests(), - [fileName]: `sha256:${sha256File(file)}`, - }; -} - -function reuseCandidateBinaries(candidate, releaseName) { - assertCandidateManifest(candidate); - const asset = candidate.assets.find((entry) => - entry.name.startsWith("clusterflux-public-binaries-") - ); - if (!asset || !fs.existsSync(asset.file)) { - throw new Error("release candidate binary archive is missing"); - } - if (sha256File(asset.file) !== asset.sha256) { - throw new Error("release candidate binary archive digest changed after candidate creation"); - } - const archive = path.join( - assetsDir, - `clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz` - ); - fs.copyFileSync(asset.file, archive); - const extractRoot = path.join(stagingDir, "candidate-binaries"); - fs.rmSync(extractRoot, { recursive: true, force: true }); - ensureDir(extractRoot); - run("tar", ["-xzf", archive, "-C", extractRoot]); - for (const name of publicBinaries.map(binaryName)) { - const digest = candidate.binary_digests[name]; - const binary = path.join(extractRoot, "bin", name); - if (!fs.existsSync(binary) || `sha256:${sha256File(binary)}` !== digest) { - throw new Error(`release candidate binary ${name} does not match ${digest}`); - } - } - return archive; -} - -function reuseCandidateHostedBinary(candidate, releaseName) { - assertCandidateManifest(candidate); - const asset = candidate.assets.find((entry) => - entry.name.startsWith("clusterflux-hosted-") - ); - if (!asset || !fs.existsSync(asset.file)) { - throw new Error("release candidate hosted binary archive is missing"); - } - if (sha256File(asset.file) !== asset.sha256) { - throw new Error("release candidate hosted archive digest changed after creation"); - } - const archive = path.join( - assetsDir, - `clusterflux-hosted-${releaseName}-${platformName()}.tar.gz` - ); - fs.copyFileSync(asset.file, archive); - const extractRoot = path.join(stagingDir, "candidate-hosted-binary"); - fs.rmSync(extractRoot, { recursive: true, force: true }); - ensureDir(extractRoot); - run("tar", ["-xzf", archive, "-C", extractRoot]); - const name = binaryName(hostedBinary); - const binary = path.join(extractRoot, "bin", name); - const digest = candidate.binary_digests[name]; - if (!fs.existsSync(binary) || `sha256:${sha256File(binary)}` !== digest) { - throw new Error(`release candidate hosted binary ${name} does not match ${digest}`); - } - return archive; -} - -function reuseCandidateExtension(candidate) { - assertCandidateManifest(candidate); - const asset = candidate.assets.find((entry) => entry.name.endsWith(".vsix")); - if (!asset || !fs.existsSync(asset.file)) { - throw new Error("release candidate VSIX is missing"); - } - const digest = sha256File(asset.file); - if ( - digest !== asset.sha256 || - digest !== candidate.release_candidate.extension_sha256 - ) { - throw new Error("release candidate VSIX digest changed after candidate creation"); - } - const archive = path.join(assetsDir, asset.name); - fs.copyFileSync(asset.file, archive); - return archive; -} - -function assertCandidateManifest(candidate) { - if (candidate.kind !== "clusterflux-public-release") { - throw new Error("release candidate manifest has the wrong kind"); - } - if (candidate.release_candidate?.mode !== "built-once") { - throw new Error("release finalization requires a built-once candidate manifest"); - } - if (!candidate.binary_digests || !Object.keys(candidate.binary_digests).length) { - throw new Error("release candidate manifest omitted binary digests"); - } - if (!candidate.binary_digests[binaryName(hostedBinary)]) { - throw new Error("release candidate omitted the hosted service binary digest"); - } - if (!candidate.release_candidate.hosted_binary_archive_sha256) { - throw new Error("release candidate omitted the hosted service archive digest"); - } - if (!candidate.release_candidate.extension_sha256) { - throw new Error("release candidate omitted the VSIX digest"); - } -} - -function stageEvidenceAsset( - releaseName, - sourceCommit, - sourceDigest, - publicTreeIdentity, - binaryDigests, - candidateBinding, - candidateConfigurationIdentity, - candidateProxyConfigurationIdentity -) { - const stage = path.join(stagingDir, "evidence"); - fs.rmSync(stage, { recursive: true, force: true }); - ensureDir(stage); - const evidenceRoot = path.join(repo, "target/acceptance"); - const included = []; - for (const name of ["public-environment.json", "wasmtime-assignment.json"]) { - const source = path.join(evidenceRoot, name); - if (!fs.existsSync(source)) continue; - fs.copyFileSync(source, path.join(stage, name)); - included.push(name); - } - const acceptancePath = path.resolve( - process.env.CLUSTERFLUX_FINAL_RESULT_PATH || - path.join(evidenceRoot, "cli-happy-path-live.json") - ); - const legacyIncompleteEvidenceEnv = [ - "CLUSTERFLUX", - "ALLOW", - "INCOMPLETE", - "RELEASE", - "EVIDENCE", - ].join("_"); - if (process.env[legacyIncompleteEvidenceEnv]) { - throw new Error( - "the legacy incomplete-evidence environment switch is unsupported; use CLUSTERFLUX_RELEASE_STAGE=candidate or final" - ); - } - const releaseStage = - process.env.CLUSTERFLUX_RELEASE_STAGE || - (candidateManifestPath ? "final" : "candidate"); - if (!new Set(["candidate", "final"]).has(releaseStage)) { - throw new Error("CLUSTERFLUX_RELEASE_STAGE must be candidate or final"); - } - const allowIncomplete = releaseStage === "candidate"; - let finalEvidence = { - complete: false, - result: null, - transcript: null, - acceptance_result_recorded_at: null, - deployment_identity: null, - configuration_identity: null, - }; - - if (fs.existsSync(acceptancePath)) { - const liveResult = JSON.parse(fs.readFileSync(acceptancePath, "utf8")); - if (liveResult.source_commit === sourceCommit) { - if (liveResult.acceptance_result !== "passed") { - throw new Error("final live acceptance result is not passed"); - } - if (liveResult.source_tree_digest !== sourceDigest) { - throw new Error("final live acceptance source-tree digest does not match the candidate"); - } - if (liveResult.public_tree_identity !== publicTreeIdentity) { - throw new Error("final live acceptance public-tree identity does not match the candidate"); - } - if ( - JSON.stringify(liveResult.binary_digests) !== JSON.stringify(binaryDigests) || - JSON.stringify(liveResult.release_binding?.binary_digests) !== - JSON.stringify(binaryDigests) - ) { - throw new Error("final live acceptance binaries do not match the exact candidate"); - } - if (liveResult.release_binding?.source_commit !== sourceCommit) { - throw new Error("final live acceptance release binding has the wrong source commit"); - } - if (!Array.isArray(liveResult.scenario_skips) || liveResult.scenario_skips.length) { - throw new Error("final live acceptance result contains unresolved scenario skips"); - } - if ( - !Array.isArray(liveResult.named_scenarios) || - !Array.isArray(liveResult.strict_requirement_ledger) || - liveResult.named_scenarios.length !== - liveResult.strict_requirement_ledger.length || - liveResult.named_scenarios.length < 29 || - liveResult.named_scenarios.some( - (scenario) => - scenario.status !== "passed" || - !Number.isFinite(scenario.duration_ms) || - scenario.duration_ms <= 0 - ) || - new Set(liveResult.named_scenarios.map((scenario) => scenario.id)) - .size !== liveResult.named_scenarios.length - ) { - throw new Error( - "every named final live check must be unique, measured, and passed" - ); - } - if ( - !liveResult.release_binding?.deployment || - !liveResult.release_binding?.configuration || - !liveResult.release_binding?.proxy_configuration - ) { - throw new Error( - "final live evidence must bind deployment, runtime configuration, and proxy configuration identities" - ); - } - if ( - !Array.isArray(liveResult.strict_requirement_ledger) || - !liveResult.strict_requirement_ledger.length || - liveResult.strict_requirement_ledger.some( - (requirement) => - requirement.passed !== true || - !Number.isFinite(requirement.duration_ms) || - requirement.duration_ms <= 0 || - !requirement.evidence - ) || - liveResult.strict_requirement_ledger.some( - (requirement, index) => - requirement.id !== liveResult.named_scenarios[index].id - ) - ) { - throw new Error("the strict final requirement ledger must be complete and passed"); - } - const deploymentGeneration = - process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || - liveResult.release_binding?.deployment?.system_generation || - null; - if (!deploymentGeneration && !allowIncomplete) { - throw new Error( - "CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION is required for final evidence" - ); - } - const recordedAt = fs.statSync(acceptancePath).mtime.toISOString(); - const packagedAt = new Date().toISOString(); - const sanitized = sanitizeEvidence(liveResult); - const finalResult = { - ...sanitized, - kind: "clusterflux-final-release-result", - source_commit: sourceCommit, - source_tree_digest: sourceDigest, - public_tree_identity: publicTreeIdentity, - binary_digests: binaryDigests, - release_candidate: candidateBinding, - deployment: { - system_generation: deploymentGeneration, - identity: evidenceIdentity(liveResult.release_binding?.deployment), - }, - configuration_identity: evidenceIdentity( - liveResult.release_binding?.configuration - ), - timestamps: { - acceptance_result_recorded_at: recordedAt, - evidence_packaged_at: packagedAt, - }, - release_commands: [ - "nix develop -c node scripts/release-quality-gates.js", - "node scripts/cli-happy-path-live-smoke.js (strict production-shaped environment)", - ], - }; - const resultName = "FINAL_RELEASE_RESULT.json"; - const transcriptName = "FINAL_RELEASE_TRANSCRIPT.txt"; - const resultPath = path.join(stage, resultName); - const transcriptPath = path.join(stage, transcriptName); - fs.writeFileSync(resultPath, `${JSON.stringify(finalResult, null, 2)}\n`); - fs.writeFileSync(transcriptPath, renderFinalTranscript(finalResult)); - included.push(resultName, transcriptName); - finalEvidence = { - complete: true, - result: { - name: resultName, - sha256: sha256File(resultPath), - }, - transcript: { - name: transcriptName, - sha256: sha256File(transcriptPath), - }, - acceptance_result_recorded_at: recordedAt, - deployment_identity: finalResult.deployment.identity, - configuration_identity: finalResult.configuration_identity, - }; - } else if (!allowIncomplete) { - throw new Error( - `final live acceptance commit ${liveResult.source_commit} does not match ${sourceCommit}` - ); - } - } else if (!allowIncomplete) { - throw new Error(`missing final live acceptance result: ${acceptancePath}`); - } - - const binding = { - kind: "clusterflux-release-evidence-binding", - source_commit: sourceCommit, - source_tree_digest: sourceDigest, - public_tree_identity: publicTreeIdentity, - binary_digests: binaryDigests, - release_candidate: candidateBinding, - candidate_configuration_identity: candidateConfigurationIdentity, - candidate_proxy_configuration_identity: candidateProxyConfigurationIdentity, - configuration_generation: - process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, - final_evidence: finalEvidence, - included_evidence: included.map((name) => ({ - name, - sha256: sha256File(path.join(stage, name)), - })), - }; - fs.writeFileSync( - path.join(stage, "EVIDENCE_BINDING.json"), - `${JSON.stringify(binding, null, 2)}\n` - ); - const archive = path.join( - assetsDir, - `clusterflux-public-evidence-${releaseName}.tar.gz` - ); - tarGz(archive, stage, ["."]); - return { archive, finalEvidence }; -} - -function stageSourceAsset(releaseName) { - const archive = path.join(assetsDir, `clusterflux-public-source-${releaseName}.tar.gz`); - tarGz(archive, publicTree, ["."]); - return archive; -} - -function stageExtensionAsset() { - const extensionRoot = path.join(publicTree, "vscode-extension"); - const packageJson = JSON.parse( - fs.readFileSync(path.join(extensionRoot, "package.json"), "utf8") - ); - const archive = path.join( - assetsDir, - `${packageJson.name}-${packageJson.version}.vsix` - ); - run("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund"], { - cwd: extensionRoot, - }); - run( - path.join(extensionRoot, "node_modules", ".bin", "vsce"), - [ - "package", - "--allow-missing-repository", - "--skip-license", - "--out", - archive, - ], - { cwd: extensionRoot } - ); - run("node", ["scripts/vscode-extension-smoke.js"], { cwd: publicTree }); - run("node", ["scripts/vscode-f5-smoke.js"], { cwd: publicTree }); - return archive; -} - -function resolverInstructions() { - if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS) { - return process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS.trim(); - } - if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY) { - return [ - "Add the controlled hosts entry supplied for this release:", - "", - "```", - process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY.trim(), - "```", - ].join("\n"); - } - if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP) { - return [ - "Add this controlled hosts entry for the release:", - "", - "```", - `${process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP} clusterflux.michelpaulissen.com`, - "```", - ].join("\n"); - } - return [ - "`clusterflux.michelpaulissen.com` should resolve through public DNS. If it", - "does not resolve yet, wait for DNS propagation or use the fallback hosts", - "entry supplied with the invitation.", - ].join("\n"); -} - -function writeGettingStartedAsset(releaseName, publicTreeIdentity, resolution) { - const file = path.join(assetsDir, `CLUSTERFLUX_GETTING_STARTED-${releaseName}.md`); - fs.writeFileSync( - file, - `# Clusterflux Getting Started - -Use the public repository and release downloads with the hosted coordinator at -\`https://clusterflux.michelpaulissen.com\`. - -## DNS - -${resolution} - -## Install - -1. Download the binary archive for your platform from the manually published GitHub release. -2. Check the archive against \`SHA256SUMS\`. -3. Extract it and put \`bin/\` on your \`PATH\`. -4. Install \`clusterflux-vscode-*.vsix\` when you want the VS Code debugger: - -\`\`\`bash -code --install-extension clusterflux-vscode-*.vsix -\`\`\` - -## Sign in and run - -\`\`\`bash -clusterflux login --browser -clusterflux auth status -clusterflux project list -clusterflux bundle inspect --project examples/hello-build -\`\`\` - -The CLI opens the server-provided Authentik authorization URL. State, nonce, -PKCE, provider code exchange, and userinfo remain on the hosted service. The CLI -stores only the resulting scoped Clusterflux session. - -Create a node enrollment grant, attach the node, and start the worker: - -\`\`\`bash -clusterflux node enroll --project-id --json -clusterflux node attach --project-id --node workstation \\ - --enrollment-grant "$ENROLLMENT_GRANT" -clusterflux-node --coordinator https://clusterflux.michelpaulissen.com \\ - --tenant --project-id --node workstation \\ - --project-root "$PWD" --worker --emit-ready -\`\`\` - -Then run the workflow: - -\`\`\`bash -clusterflux run --project examples/hello-build build -\`\`\` - -Public tree identity: \`${publicTreeIdentity}\` -Release name: \`${releaseName}\` -`, - "utf8" - ); - return file; -} - -function writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution) { - const file = path.join(assetsDir, `CLUSTERFLUX_RELEASE_NOTES-${releaseName}.md`); - const publicRepo = - process.env.CLUSTERFLUX_PUBLIC_REPO_URL || - "https://git.michelpaulissen.com/michel/clusterflux-public"; - const releaseUrl = - process.env.CLUSTERFLUX_GITHUB_RELEASE_URL || - "the manually published GitHub release"; - fs.writeFileSync( - file, - `# Clusterflux Release Notes - -Use this release with the public Clusterflux repository and hosted coordinator. - -## Links - -- Public repository: ${publicRepo} -- Release downloads: ${releaseUrl} -- Hosted coordinator: ${defaultHostedCoordinatorEndpoint} -- Public tree identity: ${publicTreeIdentity} -- Release name: ${releaseName} - -## DNS - -${resolution} - -## First run - -1. Download the archive for your platform and \`SHA256SUMS\`. -2. Extract the archive and put \`bin/\` on your \`PATH\`. -3. Optionally install the VS Code extension archive. -4. Run \`clusterflux login --browser\`. -5. Run \`clusterflux node enroll\`, attach your node, and start the worker. -6. Run \`clusterflux run --project examples/hello-build build\`. -`, - "utf8" - ); - return file; -} - -function writeSha256Sums(assets) { - const sumsPath = path.join(assetsDir, "SHA256SUMS"); - const lines = assets - .map((asset) => `${sha256File(asset)} ${path.basename(asset)}`) - .sort() - .join("\n"); - fs.writeFileSync(sumsPath, `${lines}\n`); - return sumsPath; -} - -function boolEnv(name) { - return /^(1|true|yes)$/i.test(process.env[name] || ""); -} - -function writePublicTreeProvenance(sourceCommit, releaseName) { - const provenance = { - kind: "clusterflux-filtered-public-tree", - source_commit: sourceCommit, - release_name: releaseName, - filtered_out: filteredOutPatterns(), - public_export: { - host_neutral: !includeForgejoWorkflows, - include_forgejo_workflows: includeForgejoWorkflows, - }, - forgejo_host: forgejoHost, - default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, - }; - fs.writeFileSync( - path.join(publicTree, "CLUSTERFLUX_PUBLIC_TREE.json"), - `${JSON.stringify(provenance, null, 2)}\n` - ); -} - -function publishPublicTree(releaseName, sourceCommit, publicTreeIdentity) { - const remote = - process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || - process.env.CLUSTERFLUX_PUBLIC_REPO_URL || - null; - const enabled = boolEnv("CLUSTERFLUX_PUBLISH_PUBLIC_TREE"); - const result = { - enabled, - remote, - branch: publicRepoBranch, - commit: null, - pushed: false, - }; - - if (!enabled) { - return result; - } - if (!remote) { - throw new Error( - "CLUSTERFLUX_PUBLISH_PUBLIC_TREE requires CLUSTERFLUX_PUBLIC_REPO_REMOTE or CLUSTERFLUX_PUBLIC_REPO_URL" - ); - } - if (!remote.includes(forgejoHost)) { - throw new Error(`public repo remote must point at ${forgejoHost}: ${remote}`); - } - - run("git", ["init"], { cwd: publicTree }); - run("git", ["config", "user.name", "Clusterflux release"], { - cwd: publicTree, - }); - run("git", ["config", "user.email", "release@clusterflux.invalid"], { - cwd: publicTree, - }); - run("git", ["add", "."], { cwd: publicTree }); - const tree = commandOutput("git", ["write-tree"], { cwd: publicTree }); - run("git", ["remote", "add", "public", remote], { cwd: publicTree }); - run( - "git", - [ - "fetch", - "public", - `refs/heads/${publicRepoBranch}:refs/remotes/public/${publicRepoBranch}`, - ], - { cwd: publicTree } - ); - const parent = commandOutput( - "git", - ["rev-parse", `refs/remotes/public/${publicRepoBranch}`], - { cwd: publicTree } - ); - result.commit = commandOutput( - "git", - [ - "commit-tree", - tree, - "-p", - parent, - "-m", - `Public release ${releaseName}`, - "-m", - `Source commit: ${sourceCommit}`, - "-m", - `Public tree identity: ${publicTreeIdentity}`, - ], - { cwd: publicTree } - ); - run("git", ["update-ref", `refs/heads/${publicRepoBranch}`, result.commit], { - cwd: publicTree, - }); - run("git", ["push", "public", `${result.commit}:${publicRepoBranch}`], { - cwd: publicTree, - }); - result.pushed = true; - return result; -} - -function main() { - const sourceCommit = commandOutput("git", ["rev-parse", "HEAD"]); - if (!sourceCommit || !/^[0-9a-f]{40}$/u.test(sourceCommit)) { - throw new Error("the public release requires an exact Git HEAD object id"); - } - const acceptanceCommit = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT?.trim(); - if (acceptanceCommit && acceptanceCommit !== sourceCommit) { - throw new Error( - `CLUSTERFLUX_ACCEPTANCE_COMMIT ${acceptanceCommit} does not match source HEAD ${sourceCommit}` - ); - } - const shortCommit = sourceCommit.slice(0, 12); - const releaseName = process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || `release-${shortCommit}`; - const sourceDigest = sourceTreeDigest(sourceCommit); - const candidate = candidateManifestPath - ? JSON.parse(fs.readFileSync(candidateManifestPath, "utf8")) - : null; - if (candidate) { - for (const asset of candidate.assets ?? []) { - asset.file = path.isAbsolute(asset.file) - ? asset.file - : path.resolve(path.dirname(candidateManifestPath), asset.file); - } - } - const releaseStage = - process.env.CLUSTERFLUX_RELEASE_STAGE || - (candidateManifestPath ? "final" : "candidate"); - if (releaseStage === "final" && !candidate) { - throw new Error("final release stage requires CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST"); - } - if (releaseStage === "candidate" && candidate) { - throw new Error("candidate release stage cannot consume a prior candidate manifest"); - } - if ( - releaseStage === "candidate" && - (!process.env.CLUSTERFLUX_CANDIDATE_CONFIGURATION_IDENTITY || - !process.env.CLUSTERFLUX_CANDIDATE_PROXY_CONFIGURATION_IDENTITY) - ) { - throw new Error( - "candidate release stage requires configuration and proxy configuration identities" - ); - } - if (candidate) { - assertCandidateManifest(candidate); - if (path.dirname(candidateManifestPath) === outputRoot) { - throw new Error("candidate and finalized release must use different output directories"); - } - if (candidate.source_commit !== sourceCommit) { - throw new Error("release candidate source commit does not match Git HEAD"); - } - if (candidate.source_tree_digest !== sourceDigest) { - throw new Error("release candidate source tree digest does not match Git HEAD"); - } - if (candidate.release_name !== releaseName) { - throw new Error("release candidate name does not match the finalized release name"); - } - } - const sourceStatus = commandOutput("git", ["status", "--short"]); - if (sourceStatus === null) { - throw new Error("the public release must be prepared from a Git checkout"); - } - if (sourceStatus !== "") { - throw new Error("the public release must be prepared from a clean source tree"); - } - const sourceTreeClean = true; - - fs.rmSync(outputRoot, { recursive: true, force: true }); - ensureDir(publicTree); - ensureDir(assetsDir); - ensureDir(stagingDir); - - const ignoredSourcePaths = gitIgnoredSourcePaths(); - copyFilteredTree(repo, publicTree, ignoredSourcePaths); - assertFilteredTree(ignoredSourcePaths); - writePublicTreeProvenance(sourceCommit, releaseName); - const publicTreeIdentity = hashTree(publicTree); - if (candidate && candidate.public_tree_identity !== publicTreeIdentity) { - throw new Error("release candidate public tree identity changed before finalization"); - } - const sourceArchive = stageSourceAsset(releaseName); - - run("scripts/check-old-name.sh", [], { cwd: publicTree }); - run("node", ["scripts/check-docs.js"], { cwd: publicTree }); - run("scripts/check-code-size.sh", [], { cwd: publicTree }); - const publicTreePublish = candidate - ? candidate.public_tree_publish - : publishPublicTree(releaseName, sourceCommit, publicTreeIdentity); - let binaryArchive; - let hostedBinaryArchive; - let extensionArchive; - let binaryDigests; - let candidateBinding; - if (candidate) { - binaryArchive = reuseCandidateBinaries(candidate, releaseName); - hostedBinaryArchive = reuseCandidateHostedBinary(candidate, releaseName); - extensionArchive = reuseCandidateExtension(candidate); - binaryDigests = candidate.binary_digests; - candidateBinding = { - mode: "finalized-exact", - manifest: path.relative(outputRoot, candidateManifestPath).split(path.sep).join("/"), - manifest_sha256: sha256File(candidateManifestPath), - binary_archive_sha256: sha256File(binaryArchive), - hosted_binary_archive_sha256: sha256File(hostedBinaryArchive), - extension_sha256: sha256File(extensionArchive), - }; - } else { - buildPublicBinaries(); - buildHostedBinary(); - binaryArchive = stageBinaryAssets(releaseName); - hostedBinaryArchive = stageHostedBinaryAsset(releaseName); - extensionArchive = stageExtensionAsset(); - binaryDigests = candidateBinaryDigests(); - candidateBinding = { - mode: "built-once", - binary_archive_sha256: sha256File(binaryArchive), - hosted_binary_archive_sha256: sha256File(hostedBinaryArchive), - extension_sha256: sha256File(extensionArchive), - }; - } - const candidateConfigurationIdentity = - candidate?.candidate_configuration_identity || - process.env.CLUSTERFLUX_CANDIDATE_CONFIGURATION_IDENTITY || - null; - const candidateProxyConfigurationIdentity = - candidate?.candidate_proxy_configuration_identity || - process.env.CLUSTERFLUX_CANDIDATE_PROXY_CONFIGURATION_IDENTITY || - null; - const evidence = stageEvidenceAsset( - releaseName, - sourceCommit, - sourceDigest, - publicTreeIdentity, - binaryDigests, - candidateBinding, - candidateConfigurationIdentity, - candidateProxyConfigurationIdentity - ); - const evidenceArchive = evidence.archive; - const resolution = resolverInstructions(); - const gettingStarted = writeGettingStartedAsset( - releaseName, - publicTreeIdentity, - resolution - ); - const invite = writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution); - const assets = [ - sourceArchive, - binaryArchive, - hostedBinaryArchive, - evidenceArchive, - extensionArchive, - gettingStarted, - invite, - ]; - const sha256Sums = writeSha256Sums(assets); - - const manifest = { - kind: "clusterflux-public-release", - release_name: releaseName, - source_commit: sourceCommit, - source_tree_digest: sourceDigest, - source_tree_clean: sourceTreeClean, - public_tree_identity: publicTreeIdentity, - public_tree: publicTree, - filtered_out: filteredOutPatterns(), - public_export: { - host_neutral: !includeForgejoWorkflows, - include_forgejo_workflows: includeForgejoWorkflows, - }, - forgejo_host: forgejoHost, - public_repo_url: - process.env.CLUSTERFLUX_PUBLIC_REPO_URL || - candidate?.public_repo_url || - publicTreePublish.remote, - public_repo_remote: - process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || candidate?.public_repo_remote || null, - public_tree_publish: publicTreePublish, - github_release_url: process.env.CLUSTERFLUX_GITHUB_RELEASE_URL || null, - forgejo_release_url: null, - default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, - dns_publication_state: - process.env.CLUSTERFLUX_DNS_PUBLICATION_STATE || "published", - resolver_override: - process.env.CLUSTERFLUX_RESOLVER_OVERRIDE || "none-required-public-dns", - platform: platformName(), - binary_digests: binaryDigests, - release_candidate: candidateBinding, - candidate_configuration_identity: candidateConfigurationIdentity, - candidate_proxy_configuration_identity: candidateProxyConfigurationIdentity, - configuration_generation: - process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, - final_evidence: evidence.finalEvidence, - tool_versions: { - node: process.version, - rustc: commandOutput("rustc", ["--version"]) || null, - cargo: commandOutput("cargo", ["--version"]) || null, - tar: commandOutput("tar", ["--version"]) || null, - }, - commands: [ - "scripts/check-old-name.sh", - "node scripts/check-docs.js", - "scripts/check-code-size.sh", - ...(publicTreePublish.enabled - ? [`git push public HEAD:${publicRepoBranch}`] - : []), - ...(candidate - ? ["finalize exact public and hosted binaries from CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST"] - : [ - "cargo build --locked --workspace --bins --release --jobs 2", - "cargo build --locked --manifest-path private/hosted-policy/Cargo.toml --bin clusterflux-hosted-service --release --jobs 2", - ]), - ], - assets: [...assets, sha256Sums].map((asset) => ({ - file: path.relative(outputRoot, asset).split(path.sep).join("/"), - name: path.basename(asset), - sha256: sha256File(asset), - })), - notes: - releaseStage === "final" - ? [ - "Publish manually to GitHub from the prepared manual release directory.", - "The manifest embeds the exact production-shaped validation evidence and candidate binding.", - "Forgejo release publication is not launch authority.", - ] - : [ - "This is a validation candidate, not a published release.", - "GitHub publication remains manual after final production-shaped validation.", - "Forgejo release publication is not launch authority.", - ], - }; - - const manifestPath = path.join(outputRoot, "public-release-manifest.json"); - fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - console.log(JSON.stringify({ manifest: manifestPath, assets: assetsDir }, null, 2)); -} - -main(); diff --git a/scripts/private-repository-gate.sh b/scripts/private-repository-gate.sh deleted file mode 100755 index 842457e..0000000 --- a/scripts/private-repository-gate.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -cd "$repo" - -scripts/check-old-name.sh -node scripts/check-docs.js -scripts/check-code-size.sh -cargo fmt --all --check -cargo clippy --workspace --all-targets -- -D warnings -cargo test --workspace -cargo test --locked --manifest-path private/hosted-policy/Cargo.toml -node scripts/vscode-extension-smoke.js diff --git a/scripts/public-local-demo-matrix-smoke.js b/scripts/public-local-demo-matrix-smoke.js deleted file mode 100755 index 8cae940..0000000 --- a/scripts/public-local-demo-matrix-smoke.js +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); - -function read(relativePath) { - return fs.readFileSync(path.join(repo, relativePath), "utf8"); -} - -function expect(source, name, pattern) { - assert.match(source, pattern, `missing public local demo evidence: ${name}`); -} - -const publicAcceptance = read("scripts/acceptance-public.sh"); -const publicSplit = read("scripts/verify-public-split.sh"); -const cliInstall = read("scripts/cli-install-smoke.js"); -const flagship = read("scripts/flagship-demo-smoke.js"); -const cliLocalRun = read("scripts/cli-local-run-smoke.js"); -const nodeAttach = read("scripts/node-attach-smoke.js"); -const vscodeExtension = read("scripts/vscode-extension-smoke.js"); -const vscodeF5 = read("scripts/vscode-f5-smoke.js"); -const artifactDownload = read("scripts/artifact-download-smoke.js"); -const artifactExport = read("scripts/artifact-export-smoke.js"); - -for (const script of [publicAcceptance, publicSplit]) { - for (const smoke of [ - "scripts/cli-install-smoke.js", - "scripts/flagship-demo-smoke.js", - "scripts/cli-local-run-smoke.js", - "scripts/node-attach-smoke.js", - "scripts/vscode-extension-smoke.js", - "scripts/vscode-f5-smoke.js", - "scripts/artifact-download-smoke.js", - "scripts/artifact-export-smoke.js", - ]) { - assert(script.includes(`node ${smoke}`), `public demo gate must run ${smoke}`); - } -} - -expect(cliInstall, "CLI install from project path", /cargo[\s\S]*install[\s\S]*crates\/clusterflux-cli/); -expect(cliInstall, "CLI install smoke targets runtime fixture", /const project = path\.join\(repo, "tests\/fixtures\/runtime-conformance"\)/); -expect(cliInstall, "installed CLI inspects flagship project", /installedBin[\s\S]*\["bundle", "inspect", "--project", project, "--json"\]/); - -expect(flagship, "flagship project source is conventional Rust workflow", /examples\/hello-build[\s\S]*src\/lib\.rs/); -expect(flagship, "flagship source avoids implementation details", /const forbidden/); -expect(flagship, "flagship builds a real bundle", /clusterflux-cli[\s\S]*build[\s\S]*hello/); - -expect(cliLocalRun, "local run starts node process", /cli_process_started_node_process[\s\S]*true/); -expect(cliLocalRun, "local run records real task events", /events\.events\.length >= 4[\s\S]*prepare_source[\s\S]*compile_linux[\s\S]*package_release/); -expect(cliLocalRun, "local run records artifact metadata", /events\.events\.some\(\(event\) => event\.artifact_path\)/); - -expect(nodeAttach, "Linux node attach creates enrollment grant", /create_node_enrollment_grant/); -expect(nodeAttach, "Linux node attach uses enrollment exchange", /used_enrollment_exchange[\s\S]*true/); -expect(nodeAttach, "attached Linux node proves signed enrollment", /used_enrollment_exchange[\s\S]*signedNodeHeartbeat/); - -expect(vscodeExtension, "extension contributes debugger", /contributes\.debuggers[\s\S]*type === "clusterflux"/); -expect(vscodeF5, "F5 uses local-services backend", /runtimeBackend[\s\S]*local-services/); -expect(vscodeF5, "F5 exposes real coordinator main thread", /threads\.find\(\(thread\) => thread\.name\.includes\("build coordinator main"\)\)[\s\S]*real coordinator-main entrypoint thread/); -expect(vscodeF5, "F5 does not fabricate a terminal event at the entry probe", /coordinator_task_events[\s\S]*value === 0[\s\S]*must not fabricate a terminal task event/); - -expect(artifactDownload, "artifact download creates scoped link", /create_artifact_download_link/); -expect(artifactDownload, "artifact download opens stream", /open_artifact_download_stream/); -expect(artifactExport, "artifact export targets receiver node", /export_artifact_to_node[\s\S]*node-export-receiver/); -expect(artifactExport, "artifact export rejects coordinator bulk relay", /coordinator_bulk_relay_allowed[\s\S]*false/); - -console.log("Public local demo matrix smoke passed"); diff --git a/scripts/public-release-preflight.js b/scripts/public-release-preflight.js deleted file mode 100755 index 53749b9..0000000 --- a/scripts/public-release-preflight.js +++ /dev/null @@ -1,451 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const releaseRoot = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/public-release") -); -const acceptanceRoot = path.join(repo, "target/acceptance"); -const manifestPath = - process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || - path.join(releaseRoot, "public-release-manifest.json"); -const reportPath = - process.env.CLUSTERFLUX_PUBLIC_RELEASE_PREFLIGHT_REPORT || - path.join(acceptanceRoot, "public-release-preflight.json"); - -function commandOutput(command, args, options = {}) { - try { - return cp - .execFileSync(command, args, { - cwd: repo, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - ...options, - }) - .trim(); - } catch (_) { - return null; - } -} - -function nonInteractiveEnv(extra = {}) { - return { - ...process.env, - GIT_TERMINAL_PROMPT: "0", - GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", - SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", - GIT_SSH_COMMAND: - process.env.GIT_SSH_COMMAND || - "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", - ...extra, - }; -} - -function expectedSourceCommit() { - const head = commandOutput("git", ["rev-parse", "HEAD"]); - assert(head && /^[0-9a-f]{40}$/u.test(head), "preflight requires an exact Git HEAD"); - const asserted = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT; - if (asserted) { - assert.strictEqual(asserted, head, "acceptance commit must match Git HEAD"); - } - return head; -} - -function readJson(file) { - return JSON.parse(fs.readFileSync(file, "utf8")); -} - -function sha256File(file) { - return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); -} - -function sha256Buffer(buffer) { - return crypto.createHash("sha256").update(buffer).digest("hex"); -} - -function readArchiveMember(archive, member) { - return cp.execFileSync("tar", ["-xOzf", archive, `./${member}`], { - cwd: repo, - encoding: null, - maxBuffer: 128 * 1024 * 1024, - stdio: ["ignore", "pipe", "pipe"], - }); -} - -function parseSha256Sums(file) { - const sums = new Map(); - for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) { - if (!line.trim()) continue; - const match = /^([0-9a-f]{64})\s+(.+)$/.exec(line.trim()); - assert(match, `malformed SHA256SUMS line: ${line}`); - sums.set(path.basename(match[2]), match[1]); - } - return sums; -} - -function remoteHead(remote) { - const output = commandOutput("git", ["ls-remote", remote, "HEAD", "refs/heads/main"], { - env: nonInteractiveEnv(), - timeout: Number(process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE_TIMEOUT_MS || 30000), - }); - if (!output) return null; - const lines = output.split(/\r?\n/).filter(Boolean); - const head = lines.find((line) => line.endsWith("\tHEAD")) || lines[0]; - return head && head.split(/\s+/)[0]; -} - -function publicTreeAlreadyPushed(manifest) { - return ( - (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || - process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" - ); -} - -function publicTreePushSource(manifest) { - return manifest.public_tree_publish && manifest.public_tree_publish.pushed === true - ? "manifest" - : "external-env"; -} - -function publicRepoRemoteForManifest(manifest) { - return ( - manifest.public_repo_url || - manifest.public_repo_remote || - process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || - process.env.CLUSTERFLUX_PUBLIC_REPO_URL || - null - ); -} - -function publicTreeCommitForManifest(manifest) { - return ( - (manifest.public_tree_publish && manifest.public_tree_publish.commit) || - process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || - process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || - null - ); -} - -function envState(name) { - return process.env[name] ? "set" : "unset"; -} - -function staleEvidence(file, currentSourceCommit) { - if (!fs.existsSync(file)) { - return { file, status: "missing", source_commit: null, release_name: null }; - } - const evidence = readJson(file); - if (!evidence.source_commit) { - return { - file, - status: "unversioned", - source_commit: null, - release_name: evidence.release_name || null, - }; - } - return { - file, - status: evidence.source_commit === currentSourceCommit ? "current" : "stale", - source_commit: evidence.source_commit, - release_name: evidence.release_name || null, - }; -} - -assert(fs.existsSync(manifestPath), `missing public release manifest: ${manifestPath}`); -const manifest = readJson(manifestPath); -for (const asset of manifest.assets ?? []) { - asset.file = path.isAbsolute(asset.file) - ? asset.file - : path.resolve(path.dirname(manifestPath), asset.file); -} -const currentSourceCommit = expectedSourceCommit(); -const currentTreeStatus = commandOutput("git", ["status", "--short"]) || ""; -assert.strictEqual( - currentTreeStatus, - "", - "public release preflight requires a clean source tree" -); -assert.strictEqual(manifest.kind, "clusterflux-public-release"); -assert.strictEqual( - manifest.source_commit, - currentSourceCommit, - "public release manifest must be regenerated for the current acceptance commit" -); -assert.strictEqual(manifest.source_tree_clean, true, "public release prep must start clean"); -assert.strictEqual( - publicTreeAlreadyPushed(manifest), - true, - "filtered public tree must be pushed to Forgejo before release publication" -); - -const publicRepoRemote = publicRepoRemoteForManifest(manifest); -assert(publicRepoRemote, "manifest must record public repository URL or remote"); -const publicTreeCommit = publicTreeCommitForManifest(manifest); -const remoteMain = remoteHead(publicRepoRemote); -assert(remoteMain, "Forgejo public repository main branch must be readable"); -if (publicTreeCommit) { - assert.strictEqual( - remoteMain, - publicTreeCommit, - "Forgejo public repository main branch must match the prepared public tree commit" - ); -} - -assert(Array.isArray(manifest.assets) && manifest.assets.length > 0, "manifest assets missing"); -const checksumAsset = manifest.assets.find((asset) => asset.name === "SHA256SUMS"); -assert(checksumAsset, "manifest must include SHA256SUMS"); -assert(fs.existsSync(checksumAsset.file), `missing checksum asset: ${checksumAsset.file}`); -const checksums = parseSha256Sums(checksumAsset.file); -const assets = manifest.assets.map((asset) => { - assert(fs.existsSync(asset.file), `missing release asset: ${asset.file}`); - const actual = sha256File(asset.file); - const expected = checksums.get(asset.name); - if (asset.name !== "SHA256SUMS") { - assert.strictEqual(actual, expected, `checksum mismatch for ${asset.name}`); - } - return { - name: asset.name, - file: asset.file, - bytes: fs.statSync(asset.file).size, - sha256: actual, - }; -}); -const binaryAsset = manifest.assets.find((asset) => - asset.name.startsWith("clusterflux-public-binaries-") -); -const hostedBinaryAsset = manifest.assets.find((asset) => - asset.name.startsWith("clusterflux-hosted-") -); -assert(binaryAsset, "manifest must include the exact candidate binary archive"); -assert(hostedBinaryAsset, "manifest must include the exact hosted binary archive"); -for (const [name, digest] of Object.entries(manifest.binary_digests)) { - const archive = name === "clusterflux-hosted-service" ? hostedBinaryAsset : binaryAsset; - const binary = readArchiveMember(archive.file, `bin/${name}`); - assert.strictEqual( - `sha256:${sha256Buffer(binary)}`, - digest, - `candidate archive binary ${name} does not match the manifest` - ); -} - -assert( - manifest.final_evidence && manifest.final_evidence.complete === true, - "release publication requires complete final result and transcript evidence" -); -const evidenceAsset = manifest.assets.find((asset) => - asset.name.startsWith("clusterflux-public-evidence-") -); -assert(evidenceAsset, "manifest must include the final evidence archive"); -const binding = JSON.parse( - readArchiveMember(evidenceAsset.file, "EVIDENCE_BINDING.json").toString("utf8") -); -assert.strictEqual(binding.source_commit, currentSourceCommit); -assert.strictEqual(binding.source_tree_digest, manifest.source_tree_digest); -assert.strictEqual(binding.public_tree_identity, manifest.public_tree_identity); -assert.deepStrictEqual(binding.binary_digests, manifest.binary_digests); -assert(binding.final_evidence && binding.final_evidence.complete === true); -assert(Array.isArray(binding.included_evidence)); -const includedEvidence = new Map( - binding.included_evidence.map((entry) => [entry.name, entry.sha256]) -); -for (const required of ["FINAL_RELEASE_RESULT.json", "FINAL_RELEASE_TRANSCRIPT.txt"]) { - assert(includedEvidence.has(required), `final evidence archive is missing ${required}`); - const contents = readArchiveMember(evidenceAsset.file, required); - assert.strictEqual( - sha256Buffer(contents), - includedEvidence.get(required), - `final evidence digest mismatch for ${required}` - ); -} -const finalResult = JSON.parse( - readArchiveMember(evidenceAsset.file, "FINAL_RELEASE_RESULT.json").toString("utf8") -); -assert.strictEqual(finalResult.source_commit, currentSourceCommit); -assert.strictEqual(finalResult.source_tree_digest, manifest.source_tree_digest); -assert.strictEqual(finalResult.public_tree_identity, manifest.public_tree_identity); -assert.deepStrictEqual(finalResult.binary_digests, manifest.binary_digests); -assert.strictEqual( - manifest.release_candidate?.mode, - "finalized-exact", - "publication requires exact candidate finalization" -); -assert.strictEqual(finalResult.release_binding?.source_commit, currentSourceCommit); -assert.strictEqual( - finalResult.release_binding?.source_tree_digest, - manifest.source_tree_digest -); -assert.strictEqual( - finalResult.release_binding?.public_tree_identity, - manifest.public_tree_identity -); -assert.deepStrictEqual( - finalResult.release_binding?.binary_digests, - manifest.binary_digests -); -assert.deepStrictEqual( - finalResult.release_candidate, - manifest.release_candidate -); -assert.strictEqual(finalResult.acceptance_result, "passed"); -assert.deepStrictEqual(finalResult.scenario_skips, []); -assert( - finalResult.release_binding?.deployment && - finalResult.release_binding?.configuration && - finalResult.release_binding?.proxy_configuration, - "final evidence must bind deployment, runtime configuration, and proxy configuration identities" -); -if (manifest.candidate_configuration_identity) { - assert.strictEqual( - finalResult.release_binding.configuration.evidence_identity, - manifest.candidate_configuration_identity, - "live configuration identity differs from the candidate binding" - ); -} -if (manifest.candidate_proxy_configuration_identity) { - assert.strictEqual( - finalResult.release_binding.proxy_configuration.evidence_identity, - manifest.candidate_proxy_configuration_identity, - "live proxy configuration identity differs from the candidate binding" - ); -} -assert( - Array.isArray(finalResult.named_scenarios) && - finalResult.named_scenarios.length >= 29 && - finalResult.named_scenarios.length === - finalResult.strict_requirement_ledger.length && - finalResult.named_scenarios.every( - (scenario) => - scenario.status === "passed" && - Number.isFinite(scenario.duration_ms) && - scenario.duration_ms > 0 - ), - "all named final checks must be present, measured, and passed" -); -assert( - Array.isArray(finalResult.strict_requirement_ledger) && - finalResult.strict_requirement_ledger.length > 0 && - finalResult.strict_requirement_ledger.every( - (requirement) => - requirement.passed === true && - Number.isFinite(requirement.duration_ms) && - requirement.duration_ms > 0 && - requirement.evidence - ), - "the strict final requirement ledger must be present and passed" -); -const finalTranscript = readArchiveMember( - evidenceAsset.file, - "FINAL_RELEASE_TRANSCRIPT.txt" -).toString("utf8"); -for (const identity of [ - currentSourceCommit, - manifest.source_tree_digest, - manifest.public_tree_identity, - ...Object.values(manifest.binary_digests), -]) { - assert(finalTranscript.includes(identity), `final transcript is missing identity ${identity}`); -} - -const evidence = [ - staleEvidence( - path.join(acceptanceRoot, "public-release-forgejo-release.json"), - currentSourceCommit - ), - staleEvidence( - path.join(acceptanceRoot, "public-release-service.json"), - currentSourceCommit - ), - staleEvidence( - path.join(acceptanceRoot, "hosted-client-compat.json"), - currentSourceCommit - ), - staleEvidence( - path.join(acceptanceRoot, "core-coordinator-compat.json"), - currentSourceCommit - ), - staleEvidence( - path.join(acceptanceRoot, "public-release-e2e.json"), - currentSourceCommit - ), - staleEvidence( - path.join(acceptanceRoot, "public-release-final.json"), - currentSourceCommit - ), -]; - -const report = { - kind: "clusterflux-public-release-preflight", - source_commit: currentSourceCommit, - release_name: manifest.release_name, - public_tree_commit: publicTreeCommit || remoteMain, - public_tree_push_source: publicTreePushSource(manifest), - public_repo_url: publicRepoRemote, - public_repo_remote_head: remoteMain, - source_tree_clean: currentTreeStatus === "", - local_assets_ready: true, - assets, - final_evidence: { - complete: true, - binding, - }, - evidence, - external_gates: { - forgejo_release_publication: { - status: envState("CLUSTERFLUX_FORGEJO_TOKEN") === "set" ? "ready" : "pending", - env: { - CLUSTERFLUX_FORGEJO_TOKEN: envState("CLUSTERFLUX_FORGEJO_TOKEN"), - }, - }, - live_service_smoke: { - status: - envState("CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR") === "set" && - envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" - ? "ready" - : "pending", - env: { - CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR: envState( - "CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR" - ), - CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( - "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" - ), - }, - }, - public_release_e2e: { - status: - envState("CLUSTERFLUX_PUBLIC_RELEASE_E2E") === "set" && - process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E === "1" && - envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" - ? "ready" - : "pending", - env: { - CLUSTERFLUX_PUBLIC_RELEASE_E2E: - process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E || "unset", - CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( - "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" - ), - }, - }, - final_evidence: { - status: - envState("CLUSTERFLUX_PUBLIC_RELEASE_FINAL") === "set" && - process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL === "1" - ? "ready" - : "pending", - env: { - CLUSTERFLUX_PUBLIC_RELEASE_FINAL: - process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL || "unset", - }, - }, - }, -}; - -fs.mkdirSync(path.dirname(reportPath), { recursive: true }); -fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); -console.log(JSON.stringify({ report: reportPath, release_name: report.release_name }, null, 2)); diff --git a/scripts/public-repository-gate.sh b/scripts/public-repository-gate.sh deleted file mode 100755 index 333585d..0000000 --- a/scripts/public-repository-gate.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -cd "$repo" - -scripts/verify-public-split.sh diff --git a/scripts/publish-public-release.js b/scripts/publish-public-release.js deleted file mode 100755 index 8fd76e4..0000000 --- a/scripts/publish-public-release.js +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env node - -const fs = require("fs"); -const https = require("https"); -const path = require("path"); -const cp = require("child_process"); - -const repo = path.resolve(__dirname, ".."); -const releaseRoot = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/public-release") -); -const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); -const reportPath = path.join( - repo, - "target/acceptance/public-release-forgejo-release.json" -); -const forgejoUrl = ( - process.env.CLUSTERFLUX_FORGEJO_URL || "https://git.michelpaulissen.com" -).replace(/\/+$/, ""); -const token = process.env.CLUSTERFLUX_FORGEJO_TOKEN; -let owner = process.env.CLUSTERFLUX_PUBLIC_REPO_OWNER; -let repoName = process.env.CLUSTERFLUX_PUBLIC_REPO_NAME; - -function requireEnv(name, value) { - if (!value) { - throw new Error(`${name} is required`); - } -} - -function commandOutput(command, args) { - try { - return cp - .execFileSync(command, args, { - cwd: repo, - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }) - .trim(); - } catch (_) { - return null; - } -} - -function expectedSourceCommit() { - return ( - process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || - commandOutput("git", ["rev-parse", "HEAD"]) || - "unknown" - ); -} - -function parseForgejoRepoIdentity(remote) { - if (!remote) { - return null; - } - - let pathname = remote; - try { - pathname = new URL(remote).pathname; - } catch (_) { - const scpLike = /^[^@/]+@[^:]+:(.+)$/.exec(remote); - if (scpLike) { - pathname = scpLike[1]; - } - } - - const parts = pathname - .replace(/^\/+/, "") - .replace(/\.git$/, "") - .split("/") - .filter(Boolean); - if (parts.length < 2) { - return null; - } - return { - owner: parts[parts.length - 2], - repoName: parts[parts.length - 1], - }; -} - -function resolveRepoIdentity(manifest) { - if (owner && repoName) { - return; - } - - const inferred = parseForgejoRepoIdentity( - manifest.public_repo_url || - manifest.public_repo_remote || - process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE - ); - owner = owner || (inferred && inferred.owner); - repoName = repoName || (inferred && inferred.repoName); - if (!owner || !repoName) { - throw new Error( - "CLUSTERFLUX_PUBLIC_REPO_OWNER and CLUSTERFLUX_PUBLIC_REPO_NAME are required when the manifest does not contain a parseable Forgejo repository URL" - ); - } -} - -function publicTreeAlreadyPushed(manifest) { - return ( - (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || - process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" - ); -} - -function publicTreeCommit(manifest) { - return ( - (manifest.public_tree_publish && manifest.public_tree_publish.commit) || - process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || - process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || - null - ); -} - -function apiPath(pathname) { - return `/api/v1${pathname}`; -} - -function request(method, pathname, { body, headers = {} } = {}) { - const url = new URL(apiPath(pathname), forgejoUrl); - const payload = - body === undefined - ? null - : Buffer.isBuffer(body) - ? body - : Buffer.from(JSON.stringify(body)); - const requestHeaders = { - Accept: "application/json", - Authorization: `token ${token}`, - ...headers, - }; - if (payload) { - requestHeaders["Content-Length"] = payload.length; - if (!requestHeaders["Content-Type"]) { - requestHeaders["Content-Type"] = "application/json"; - } - } - - return new Promise((resolve, reject) => { - const req = https.request( - url, - { - method, - headers: requestHeaders, - }, - (res) => { - const chunks = []; - res.on("data", (chunk) => chunks.push(chunk)); - res.on("end", () => { - const text = Buffer.concat(chunks).toString("utf8"); - let parsed = null; - if (text.trim()) { - try { - parsed = JSON.parse(text); - } catch (_) { - parsed = text; - } - } - if (res.statusCode < 200 || res.statusCode >= 300) { - reject( - new Error( - `${method} ${url.pathname} failed with ${res.statusCode}: ${text}` - ) - ); - return; - } - resolve({ status: res.statusCode, body: parsed }); - }); - } - ); - req.on("error", reject); - if (payload) req.write(payload); - req.end(); - }); -} - -function multipartFile(fieldName, file) { - const boundary = `clusterflux-${Date.now()}-${Math.random().toString(16).slice(2)}`; - const name = path.basename(file); - const header = Buffer.from( - `--${boundary}\r\n` + - `Content-Disposition: form-data; name="${fieldName}"; filename="${name}"\r\n` + - "Content-Type: application/octet-stream\r\n\r\n" - ); - const footer = Buffer.from(`\r\n--${boundary}--\r\n`); - return { - body: Buffer.concat([header, fs.readFileSync(file), footer]), - contentType: `multipart/form-data; boundary=${boundary}`, - }; -} - -async function existingRelease(tagName) { - const releases = await request( - "GET", - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases?limit=100` - ); - return (releases.body || []).find((release) => release.tag_name === tagName) || null; -} - -async function createOrReuseRelease(manifest) { - const tagName = manifest.release_name; - const found = await existingRelease(tagName); - if (found) { - return { release: found, created: false }; - } - const targetCommitish = - publicTreeCommit(manifest) || - "main"; - const body = [ - "Clusterflux public release.", - "", - `Default hosted coordinator endpoint: ${manifest.default_hosted_coordinator_endpoint}`, - `DNS publication state: ${manifest.dns_publication_state}`, - `Resolver override: ${manifest.resolver_override}`, - `Public tree identity: ${manifest.public_tree_identity}`, - `Source commit: ${manifest.source_commit}`, - ].join("\n"); - const created = await request( - "POST", - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases`, - { - body: { - tag_name: tagName, - target_commitish: targetCommitish, - name: tagName, - body, - draft: false, - prerelease: false, - }, - } - ); - return { release: created.body, created: true }; -} - -async function loadRelease(releaseId) { - const response = await request( - "GET", - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${releaseId}` - ); - return response.body; -} - -async function uploadAsset(release, asset) { - const { body, contentType } = multipartFile("attachment", asset.file); - const response = await request( - "POST", - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, - { - body, - headers: { - "Content-Type": contentType, - }, - } - ); - return response.body; -} - -function existingAssetByName(release, name) { - return Array.isArray(release.assets) - ? release.assets.find((asset) => asset.name === name) || null - : null; -} - -async function main() { - requireEnv("CLUSTERFLUX_FORGEJO_TOKEN", token); - if (!fs.existsSync(manifestPath)) { - throw new Error(`missing public release manifest: ${manifestPath}`); - } - - const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); - for (const asset of manifest.assets ?? []) { - asset.file = path.isAbsolute(asset.file) - ? asset.file - : path.resolve(path.dirname(manifestPath), asset.file); - } - resolveRepoIdentity(manifest); - if (manifest.kind !== "clusterflux-public-release") { - throw new Error(`unexpected public release manifest kind: ${manifest.kind}`); - } - if (manifest.source_commit !== expectedSourceCommit()) { - throw new Error( - "public release manifest is stale; regenerate it for the current acceptance commit before publishing the Forgejo Release" - ); - } - if ( - !publicTreeAlreadyPushed(manifest) - ) { - throw new Error( - "public tree must be pushed before publishing the Forgejo Release; run prepare-public-release.js with CLUSTERFLUX_PUBLISH_PUBLIC_TREE=1" - ); - } - if (!Array.isArray(manifest.assets) || manifest.assets.length === 0) { - throw new Error("public release manifest has no assets"); - } - - const { release: releaseResult, created } = await createOrReuseRelease(manifest); - const release = await loadRelease(releaseResult.id); - const uploaded = []; - const reused = []; - for (const asset of manifest.assets) { - if (!fs.existsSync(asset.file)) { - throw new Error(`missing release asset: ${asset.file}`); - } - const existing = existingAssetByName(release, asset.name); - if (existing) { - reused.push(existing); - continue; - } - uploaded.push(await uploadAsset(release, asset)); - } - - fs.mkdirSync(path.dirname(reportPath), { recursive: true }); - const report = { - kind: "clusterflux-public-release-forgejo-release", - forgejo_url: forgejoUrl, - owner, - repo: repoName, - release_id: release.id, - release_name: release.name || manifest.release_name, - tag_name: release.tag_name || manifest.release_name, - release_created: created, - default_hosted_coordinator_endpoint: manifest.default_hosted_coordinator_endpoint, - public_tree_identity: manifest.public_tree_identity, - public_tree_commit: publicTreeCommit(manifest), - source_commit: manifest.source_commit, - uploaded_assets: uploaded.map((asset) => ({ - id: asset.id, - name: asset.name, - size: asset.size, - browser_download_url: asset.browser_download_url, - })), - reused_assets: reused.map((asset) => ({ - id: asset.id, - name: asset.name, - size: asset.size, - browser_download_url: asset.browser_download_url, - })), - }; - fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); - console.log( - JSON.stringify( - { report: reportPath, uploaded: uploaded.length, reused: reused.length }, - null, - 2 - ) - ); -} - -main().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/quic-smoke.js b/scripts/quic-smoke.js deleted file mode 100755 index 5ad4634..0000000 --- a/scripts/quic-smoke.js +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); - -const repo = path.resolve(__dirname, ".."); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function rendezvousRequest(overrides = {}) { - return { - type: "request_rendezvous", - scope: { - tenant: "tenant", - project: "project", - process: "vp-quic", - object: { Artifact: "quic-artifact" }, - authorization_subject: "node-a-to-node-b", - }, - source: { - node: "node-a", - advertised_addr: "node-a.mesh.invalid:4433", - public_key_fingerprint: "sha256:node-a-public-key", - }, - destination: { - node: "node-b", - advertised_addr: "node-b.mesh.invalid:4433", - public_key_fingerprint: "sha256:node-b-public-key", - }, - direct_connectivity: true, - failure_reason: "", - ...overrides, - }; -} - -(async () => { - const output = cp.execFileSync( - "cargo", - ["run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-quic-smoke"], - { cwd: repo, encoding: "utf8" } - ); - const report = JSON.parse(output.trim().split("\n").at(-1)); - - assert.strictEqual(report.kind, "clusterflux_quic_smoke"); - assert.strictEqual(report.transport, "NativeQuic"); - assert.strictEqual(report.rust_native_quic, true); - assert.strictEqual(report.authenticated_direct_connection, true); - assert.strictEqual(report.coordinator_assisted_rendezvous, true); - assert.strictEqual(report.coordinator_bulk_relay_allowed, false); - assert.strictEqual(report.source_node, "node-a"); - assert.strictEqual(report.destination_node, "node-b"); - assert.strictEqual(report.scope.tenant, "tenant"); - assert.strictEqual(report.scope.project, "project"); - assert.strictEqual(report.scope.process, "vp-quic"); - assert.deepStrictEqual(report.scope.object, { Artifact: "quic-artifact" }); - assert.ok(report.authorization_digest.startsWith("sha256:")); - assert.ok(report.request_bytes > 0); - assert.strictEqual(report.server_received_request_bytes, report.request_bytes); - assert.ok(report.payload_bytes > 0); - - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback", - ], - { cwd: repo } - ); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - - const plan = await send(addr, rendezvousRequest()); - assert.strictEqual(plan.type, "rendezvous_plan"); - assert.strictEqual(plan.charged_rendezvous_attempts, 1); - assert.strictEqual(plan.plan.transport, "NativeQuic"); - assert.strictEqual(plan.plan.scope.tenant, "tenant"); - assert.strictEqual(plan.plan.scope.project, "project"); - assert.strictEqual(plan.plan.source.node, "node-a"); - assert.strictEqual(plan.plan.destination.node, "node-b"); - assert.strictEqual(plan.plan.coordinator_assisted_rendezvous, true); - assert.strictEqual(plan.plan.coordinator_bulk_relay_allowed, false); - assert.ok(plan.plan.authorization_digest.startsWith("sha256:")); - - const failed = await send( - addr, - rendezvousRequest({ - direct_connectivity: false, - failure_reason: "nat traversal failed", - }) - ); - assert.strictEqual(failed.type, "error"); - assert.match(failed.message, /nat traversal failed/); - assert.match(failed.message, /coordinator bulk relay is disabled/); - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("QUIC smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/real-flagship-harness.js b/scripts/real-flagship-harness.js deleted file mode 100644 index 1320f23..0000000 --- a/scripts/real-flagship-harness.js +++ /dev/null @@ -1,285 +0,0 @@ -const assert = require("assert"); -const cp = require("child_process"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { configurePodmanTestEnvironment } = require("./podman-test-env"); - -const repo = path.resolve(__dirname, ".."); -const project = path.join(repo, "examples/hello-build"); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.stderr?.on("data", (chunk) => { - stderr += chunk.toString(); - if (stderr.length > 4096) stderr = stderr.slice(-4096); - }); - child.once("exit", (code) => { - const detail = stderr.trim(); - reject(new Error( - `process exited before JSON line with code ${code}${detail ? `: ${detail}` : ""}` - )); - }); - }); -} - -function waitForNodeStatus(child, expectedStatus) { - return new Promise((resolve, reject) => { - let buffer = ""; - let stderr = ""; - const cleanup = () => { - child.stdout.off("data", onData); - child.stderr?.off("data", onStderr); - child.off("exit", onExit); - }; - const onData = (chunk) => { - buffer += chunk.toString(); - while (buffer.includes("\n")) { - const newline = buffer.indexOf("\n"); - const line = buffer.slice(0, newline).trim(); - buffer = buffer.slice(newline + 1); - if (!line) continue; - let event; - try { - event = JSON.parse(line); - } catch (error) { - cleanup(); - reject(error); - return; - } - if (event.node_status === expectedStatus) { - cleanup(); - resolve(event); - return; - } - } - }; - const onStderr = (chunk) => { - stderr += chunk.toString(); - if (stderr.length > 4096) stderr = stderr.slice(-4096); - }; - const onExit = (code) => { - cleanup(); - const detail = stderr.trim(); - reject( - new Error( - `worker exited before node status ${expectedStatus} with code ${code}${ - detail ? `: ${detail}` : "" - }` - ) - ); - }; - child.stdout.on("data", onData); - child.stderr?.on("data", onStderr); - child.once("exit", onExit); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function configureContainerPolicyHome() { - configurePodmanTestEnvironment(repo); -} - -function commandWithPodman(program, args) { - if (cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status === 0) { - return { program, args }; - } - if (cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0) { - return { - program: "nix", - args: ["shell", "nixpkgs#podman", "--command", program, ...args], - }; - } - throw new Error( - "real flagship smoke requires rootless Podman (or Nix to provide it)" - ); -} - -function ensureRootlessPodman() { - configureContainerPolicyHome(); - const invocation = commandWithPodman("podman", [ - "info", - "--format", - "{{.Host.Security.Rootless}}", - ]); - const rootless = cp.execFileSync(invocation.program, invocation.args, { - cwd: repo, - env: process.env, - encoding: "utf8", - }).trim(); - assert.strictEqual(rootless, "true", "flagship worker must use rootless Podman"); -} - -async function runFlagshipWorker(addr, node, identity, projectRoot = project) { - const enrollment = await send(addr, { - type: "create_node_enrollment_grant", - tenant: "tenant", - project: "project", - actor_user: "user", - ttl_seconds: 60, - }); - assert.strictEqual(enrollment.type, "node_enrollment_grant_created"); - const cargoArgs = [ - "run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node", "--", - "--coordinator", `${addr.host}:${addr.port}`, - "--tenant", "tenant", - "--project-id", "project", - "--node", node, - "--enrollment-grant", enrollment.grant, - "--worker", - "--emit-ready", - "--project-root", projectRoot, - "--assignment-poll-ms", "25", - ]; - const invocation = commandWithPodman("cargo", cargoArgs); - const child = cp.spawn(invocation.program, invocation.args, { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, - }, - }); - return { child, ready: waitForJsonLine(child) }; -} - -const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); - -async function waitForTaskEvent(addr, process, predicate, description) { - for (let attempt = 0; attempt < 2400; attempt += 1) { - const response = await send(addr, { - type: "list_task_events", - tenant: "tenant", - project: "project", - actor_user: "user", - process, - }); - assert.strictEqual(response.type, "task_events", JSON.stringify(response)); - const event = response.events.find(predicate); - if (event) return event; - await delay(25); - } - throw new Error(`timed out waiting for real Wasm task ${description}`); -} - -function startFlagship(addr, projectRoot = project) { - const report = JSON.parse( - cp.execFileSync( - "cargo", - [ - "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", - "run", "build", - "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, - "--project", projectRoot, - "--json", - ], - { cwd: repo, env: process.env, encoding: "utf8" } - ) - ); - assert.strictEqual(report.command, "run"); - assert.strictEqual(report.status, "main_launched", JSON.stringify(report)); - assert.strictEqual(report.entry, "build"); - assert.strictEqual(report.task_launch.type, "main_launched"); - assert.strictEqual(report.task_launch.task_instance, report.task_instance); - assert.strictEqual(report.task_launch.task_definition, report.task_definition); - assert.strictEqual(report.worker_placement_requested, true); - assert.match(report.bundle_digest, /^sha256:[0-9a-f]{64}$/); - assert.match(report.entry_export, /^clusterflux_entry_v1_/); - const virtualProcess = report.process; - return { report, process: virtualProcess }; -} - -async function launchFlagship(addr) { - const { report, process: virtualProcess } = startFlagship(addr); - const compileEvent = await waitForTaskEvent( - addr, - virtualProcess, - (event) => event.task_definition === "compile", - "compile" - ); - const sourceEvent = await waitForTaskEvent( - addr, - virtualProcess, - (event) => event.task_definition === "snapshot_current_project", - "snapshot_current_project" - ); - const packageEvent = compileEvent; - const buildEvent = await waitForTaskEvent( - addr, - virtualProcess, - (event) => event.task === report.task_instance && event.executor === "coordinator_main", - "coordinator build main" - ); - for (const event of [sourceEvent, compileEvent, buildEvent]) { - assert.strictEqual(event.terminal_state, "completed", JSON.stringify(event)); - } - return { - report, - process: virtualProcess, - compileEvent, - sourceEvent, - packageEvent, - buildEvent, - }; -} - -function flagshipNodeCapabilities() { - return { - os: "Linux", - arch: process.arch, - capabilities: [ - "Command", - "Containers", - "RootlessPodman", - "SourceFilesystem", - "VfsArtifacts", - ], - environment_backends: ["Container"], - source_providers: ["filesystem"], - }; -} - -module.exports = { - ensureRootlessPodman, - flagshipNodeCapabilities, - launchFlagship, - project, - repo, - runFlagshipWorker, - send, - startFlagship, - waitForTaskEvent, - waitForJsonLine, - waitForNodeStatus, -}; diff --git a/scripts/recovery-build-smoke.js b/scripts/recovery-build-smoke.js deleted file mode 100755 index 202d788..0000000 --- a/scripts/recovery-build-smoke.js +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); -const { DapClient } = require("./dap-client"); -const { configurePodmanTestEnvironment } = require("./podman-test-env"); - -const repo = path.resolve(__dirname, ".."); -const project = path.join(repo, "examples/recovery-build"); -const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs")); -const original = fs.readFileSync(sourcePath, "utf8"); -const failing = "\"exit 23\".to_owned()"; -const replacement = - "\"printf 'recovered\\n' > /clusterflux/output/recovering.txt\".to_owned()"; -assert(original.includes(failing), "recovery source must contain the real failing command"); -configurePodmanTestEnvironment(repo); - -(async () => { - const client = new DapClient({ - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_DAP_TIMEOUT_MS: - process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || "120000", - }, - }); - let restored = false; - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true, - }); - await client.response(initialize, "initialize"); - - const launch = client.send("launch", { - entry: "build", - project, - runtimeBackend: "local-services", - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const configured = client.send("configurationDone"); - await client.response(configured, "configurationDone"); - const failed = await client.waitFor( - (message) => - message.type === "event" && - message.event === "stopped" && - message.body.reason === "exception" - ); - assert.strictEqual(failed.body.allThreadsStopped, false); - - const threadRequest = client.send("threads"); - const threads = (await client.response(threadRequest, "threads")).body.threads; - const laneThreads = threads - .filter((thread) => /build[_ ]lane/.test(thread.name)) - .sort((left, right) => left.id - right.id); - assert.strictEqual( - laneThreads.length, - 1, - "only the failed recovering lane should remain a live DAP thread" - ); - const recoveringThread = laneThreads[0]; - const views = JSON.parse( - fs.readFileSync(path.join(project, ".clusterflux/views.json"), "utf8") - ); - const nodeReport = JSON.parse( - views.inspector.find((item) => item.label === "Node report").value - ); - const laneSnapshots = nodeReport.task_snapshots.snapshots.filter( - (snapshot) => snapshot.task_definition === "build_lane" - ); - assert.strictEqual(laneSnapshots.length, 2); - assert.notStrictEqual(laneSnapshots[0].task, laneSnapshots[1].task); - assert(laneSnapshots.some((snapshot) => snapshot.state === "completed")); - assert( - laneSnapshots.some( - (snapshot) => snapshot.state === "failed_awaiting_action" - ) - ); - const startedIds = new Set( - client.messages - .filter( - (message) => - message.type === "event" && - message.event === "thread" && - message.body.reason === "started" - ) - .map((message) => message.body.threadId) - ); - const laneStartedIds = [...startedIds] - .filter((id) => id !== 1) - .sort((left, right) => left - right) - .slice(-2); - assert.strictEqual(laneStartedIds.length, 2); - assert.notStrictEqual(laneStartedIds[0], laneStartedIds[1]); - assert(startedIds.has(recoveringThread.id)); - - fs.writeFileSync(sourcePath, original.replace(failing, replacement)); - const restart = client.send("restartFrame", { - threadId: recoveringThread.id, - }); - await client.response(restart, "restartFrame"); - const terminated = await client.waitFor( - (message) => - message.type === "event" && - message.event === "terminated" - ); - assert(terminated); - - const exitedIds = new Set( - client.messages - .filter( - (message) => - message.type === "event" && - message.event === "thread" && - message.body.reason === "exited" - ) - .map((message) => message.body.threadId) - ); - assert(exitedIds.has(laneStartedIds[0])); - assert(exitedIds.has(recoveringThread.id)); - - fs.writeFileSync(sourcePath, original); - restored = true; - await client.close(); - console.log("Recovery build DAP restart smoke passed"); - } finally { - if (!restored) fs.writeFileSync(sourcePath, original); - if (client.child.exitCode === null) client.child.kill("SIGKILL"); - } -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/release-quality-gates.js b/scripts/release-quality-gates.js deleted file mode 100755 index 5ec6343..0000000 --- a/scripts/release-quality-gates.js +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const os = require("os"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const manifestPath = path.resolve( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || - path.join( - process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || - path.join(repo, "target/release-candidate"), - "public-release-manifest.json" - ) -); -const evidencePath = path.resolve( - process.env.CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH || - path.join(repo, "target/acceptance/quality-gates.json") -); -const transcriptPath = path.resolve( - process.env.CLUSTERFLUX_QUALITY_GATE_TRANSCRIPT_PATH || - path.join(repo, "target/acceptance/quality-gates-transcript.txt") -); - -function readJson(file) { - return JSON.parse(fs.readFileSync(file, "utf8")); -} - -function sha256(file) { - return crypto - .createHash("sha256") - .update(fs.readFileSync(file)) - .digest("hex"); -} - -function commandText(command, args) { - return [command, ...args] - .map((value) => - /^[A-Za-z0-9_./:=+-]+$/.test(value) - ? value - : JSON.stringify(value) - ) - .join(" "); -} - -function main() { - assert(fs.existsSync(manifestPath), `missing release manifest ${manifestPath}`); - const manifest = readJson(manifestPath); - fs.mkdirSync(path.dirname(evidencePath), { recursive: true }); - fs.mkdirSync(path.dirname(transcriptPath), { recursive: true }); - const transcript = fs.openSync(transcriptPath, "w"); - const evidence = { - kind: "clusterflux-release-quality-gates", - source_commit: manifest.source_commit, - source_tree_digest: manifest.source_tree_digest, - public_tree_identity: manifest.public_tree_identity, - checks: {}, - }; - - const runGroup = (id, commands, extra = {}) => { - const startedAt = Date.now(); - const commandEvidence = []; - let status = "passed"; - for (const command of commands) { - const args = command.args || []; - const cwd = command.cwd || repo; - const text = commandText(command.command, args); - fs.writeSync(transcript, `\n[${id}] ${text}\n`); - const commandStartedAt = Date.now(); - const result = cp.spawnSync(command.command, args, { - cwd, - env: { ...process.env, ...(command.env || {}) }, - stdio: ["ignore", transcript, transcript], - }); - const durationMs = Math.max(1, Date.now() - commandStartedAt); - commandEvidence.push({ - command: text, - cwd, - exit_status: result.status, - duration_ms: durationMs, - }); - if (result.error || result.status !== 0) { - status = "failed"; - evidence.checks[id] = { - status, - duration_ms: Math.max(1, Date.now() - startedAt), - commands: commandEvidence, - error: result.error?.message || `command exited ${result.status}`, - ...extra, - }; - fs.writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); - throw new Error(`${id} failed: ${text}`); - } - } - evidence.checks[id] = { - status, - duration_ms: Math.max(1, Date.now() - startedAt), - commands: commandEvidence, - ...extra, - }; - }; - - try { - runGroup("repository_structure", [ - { command: path.join(repo, "scripts/check-old-name.sh") }, - { command: "node", args: [path.join(repo, "scripts/check-docs.js")] }, - { command: path.join(repo, "scripts/check-code-size.sh") }, - ]); - runGroup("formatting", [ - { command: "cargo", args: ["fmt", "--all", "--check"] }, - { - command: "cargo", - args: [ - "fmt", - "--all", - "--manifest-path", - "private/hosted-policy/Cargo.toml", - "--check", - ], - }, - ]); - runGroup("clippy_warnings_denied", [ - { - command: "cargo", - args: ["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"], - }, - { - command: "cargo", - args: [ - "clippy", - "--manifest-path", - "private/hosted-policy/Cargo.toml", - "--all-targets", - "--", - "-D", - "warnings", - ], - }, - ]); - runGroup("public_workspace_tests", [ - { - command: "cargo", - args: ["test", "--workspace", "--all-targets"], - }, - { - command: "cargo", - args: ["build", "--workspace", "--all-targets"], - }, - ]); - runGroup("private_hosted_policy_locked_tests", [ - { - command: "cargo", - args: [ - "test", - "--locked", - "--manifest-path", - "private/hosted-policy/Cargo.toml", - "--all-targets", - ], - }, - ]); - runGroup("process_lifecycle_regressions", [ - { - command: "cargo", - args: [ - "test", - "-p", - "clusterflux-coordinator", - "completed_main_", - ], - }, - { - command: "cargo", - args: [ - "test", - "-p", - "clusterflux-coordinator", - "failed_main_aborts_unfinished_children_and_clears_process_debug_state", - ], - }, - { - command: "cargo", - args: [ - "test", - "-p", - "clusterflux-coordinator", - "service_cancels_whole_process_and_blocks_new_task_launches", - ], - }, - ]); - runGroup("wasm_example_builds", [ - { - command: "cargo", - args: [ - "build", - "-p", - "hello-build", - "--target", - "wasm32-unknown-unknown", - ], - }, - { - command: "cargo", - args: [ - "build", - "-p", - "recovery-build", - "--target", - "wasm32-unknown-unknown", - ], - }, - { - command: "cargo", - args: [ - "build", - "-p", - "runtime-conformance", - "--target", - "wasm32-unknown-unknown", - ], - }, - ]); - runGroup("filtered_public_tree_build_and_tests", [ - { command: path.join(repo, "scripts/verify-public-split.sh") }, - ]); - - const extensionAsset = manifest.assets.find((asset) => - asset.name.endsWith(".vsix") - ); - assert(extensionAsset, "candidate manifest omitted its VSIX"); - const extensionFile = path.resolve( - path.dirname(manifestPath), - extensionAsset.file - ); - assert(fs.existsSync(extensionFile), `missing candidate VSIX ${extensionFile}`); - const extensionDigest = sha256(extensionFile); - assert.strictEqual(extensionDigest, extensionAsset.sha256); - assert.strictEqual( - extensionDigest, - manifest.release_candidate.extension_sha256 - ); - const extractedVsix = fs.mkdtempSync( - path.join(os.tmpdir(), "clusterflux-tested-vsix-") - ); - try { - runGroup( - "vscode_extension_candidate", - [ - { - command: "npm", - args: ["ci", "--ignore-scripts"], - cwd: path.join(repo, "vscode-extension"), - }, - { - command: "unzip", - args: ["-q", extensionFile, "-d", extractedVsix], - }, - { - command: "node", - args: [path.join(repo, "scripts/vscode-extension-smoke.js")], - env: { - CLUSTERFLUX_VSCODE_EXTENSION_ROOT: path.join( - extractedVsix, - "extension" - ), - }, - }, - ], - { - candidate_vsix: { - file: extensionFile, - sha256: extensionDigest, - }, - } - ); - } finally { - fs.rmSync(extractedVsix, { recursive: true, force: true }); - } - - const publicCheckIds = [ - "repository_structure", - "formatting", - "clippy_warnings_denied", - "public_workspace_tests", - "process_lifecycle_regressions", - "wasm_example_builds", - "filtered_public_tree_build_and_tests", - "vscode_extension_candidate", - ]; - evidence.public = { - status: "passed", - duration_ms: publicCheckIds.reduce( - (total, id) => total + evidence.checks[id].duration_ms, - 0 - ), - independent_filtered_tree: true, - }; - evidence.private = { - status: "passed", - duration_ms: - evidence.checks.formatting.duration_ms + - evidence.checks.clippy_warnings_denied.duration_ms + - evidence.checks.private_hosted_policy_locked_tests.duration_ms, - }; - evidence.status = "passed"; - evidence.transcript = transcriptPath; - fs.writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); - } finally { - fs.closeSync(transcript); - } - console.log(`Release quality gates passed: ${evidencePath}`); -} - -try { - main(); -} catch (error) { - console.error(error.stack || error.message); - process.exit(1); -} diff --git a/scripts/release-source-scan.sh b/scripts/release-source-scan.sh deleted file mode 100755 index d8bf013..0000000 --- a/scripts/release-source-scan.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo" - -release_paths=( - Cargo.toml - Cargo.lock - README.md - crates - examples - private - scripts - vscode-extension -) - -existing_release_paths=() -for path in "${release_paths[@]}"; do - if [[ -e "$path" ]]; then - existing_release_paths+=("$path") - fi -done - -prose_scan_paths=( - README.md - crates - examples - private - scripts - vscode-extension -) - -existing_prose_scan_paths=() -for path in "${prose_scan_paths[@]}"; do - if [[ -e "$path" ]]; then - existing_prose_scan_paths+=("$path") - fi -done - -scan_globs=( - --glob '!**/target/**' - --glob '!**/node_modules/**' - --glob '!scripts/release-source-scan.sh' - --glob '!scripts/rename-to-clusterflux.sh' - --glob '!scripts/check-docs.js' -) - -placeholder_pattern='debugger-gate|experiments/debugger-gate|CLUSTERFLUX-DEMO|device-code-placeholder|artifact://demo|vp-local-demo' -if rg -n "${scan_globs[@]}" "$placeholder_pattern" "${existing_release_paths[@]}"; then - echo "release source scan failed: stale experiment/demo placeholder reference found" >&2 - exit 1 -fi - -demo_setup_pattern='undocumented manual state|hidden setup|demo-only credentials?|hard-coded local paths?' -if rg -n "${scan_globs[@]}" "$demo_setup_pattern" "${existing_prose_scan_paths[@]}"; then - echo "release source scan failed: demo requires hidden setup or demo-only state" >&2 - exit 1 -fi - -hidden_local_pattern='file://|/home/[[:alnum:]_.-]+/|/Users/[[:alnum:]_.-]+/|C:\\Users\\|https?://(localhost|127\.0\.0\.1)[^[:space:]]*/artifacts/' -if rg -n "${scan_globs[@]}" "$hidden_local_pattern" "${existing_release_paths[@]}"; then - echo "release source scan failed: hidden local path or local artifact URL found" >&2 - exit 1 -fi - -public_wording_pattern='reddit|hacker news|lobsters|launch forum|traffic source|free tier' -if rg -n "${scan_globs[@]}" "$public_wording_pattern" "${existing_prose_scan_paths[@]}"; then - echo "release source scan failed: public-facing launch-forum/free-tier wording found" >&2 - exit 1 -fi - -echo "Release source scan passed" diff --git a/scripts/resource-metering-contract-smoke.js b/scripts/resource-metering-contract-smoke.js deleted file mode 100644 index 62e593d..0000000 --- a/scripts/resource-metering-contract-smoke.js +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); - -function read(relativePath) { - return fs.readFileSync(path.join(repo, relativePath), "utf8"); -} - -function maybeRead(segments) { - const fullPath = path.join(repo, ...segments); - if (!fs.existsSync(fullPath)) return null; - return fs.readFileSync(fullPath, "utf8"); -} - -function expect(source, name, pattern) { - assert.match(source, pattern, `missing resource metering evidence: ${name}`); -} - -const coreLimits = read("crates/clusterflux-core/src/limits.rs"); -// Phase 3 keeps the protocol dispatch in service.rs and the metered operation -// implementations in focused service modules. Read the complete relevant -// boundary so this contract follows the refactor instead of one mega-file. -const coordinatorService = [ - read("crates/clusterflux-coordinator/src/service.rs"), - read("crates/clusterflux-coordinator/src/service/routing.rs"), - read("crates/clusterflux-coordinator/src/service/processes.rs"), - read("crates/clusterflux-coordinator/src/service/process_launch.rs"), - read("crates/clusterflux-coordinator/src/service/artifacts.rs"), -].join("\n"); -const coordinatorQuota = read("crates/clusterflux-coordinator/src/service/quota.rs"); -const coordinatorLogs = read("crates/clusterflux-coordinator/src/service/logs.rs"); -const coordinatorDebug = read("crates/clusterflux-coordinator/src/service/debug.rs"); -const coordinatorTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); -const wasmRuntime = read("crates/clusterflux-wasm-runtime/src/lib.rs"); -const wasmRuntimeTests = read("crates/clusterflux-wasm-runtime/src/tests.rs"); -const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); -const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); -const quicSmoke = read("scripts/quic-smoke.js"); - -for (const [name, pattern] of [ - ["API call limit kind", /\bApiCall,/], - ["spawn limit kind", /\bSpawn,/], - ["log bytes limit kind", /\bLogBytes,/], - ["metadata bytes limit kind", /\bMetadataBytes,/], - ["debug read bytes limit kind", /\bDebugReadBytes,/], - ["UI event limit kind", /\bUiEvent,/], - ["rendezvous attempt limit kind", /\bRendezvousAttempt,/], - ["artifact download bytes limit kind", /\bArtifactDownloadBytes,/], - [ - "preflight can check without consuming", - /pub fn can_charge\([\s\S]*used\.saturating_add\(amount\) > limit/, - ], - [ - "charge goes through preflight", - /pub fn charge\([\s\S]*self\.can_charge\(limits, kind, amount\)\?/, - ], -]) { - expect(coreLimits, name, pattern); -} - -for (const [name, pattern] of [ - [ - "Wasm stores enforce a concrete memory limit", - /StoreLimitsBuilder::new\(\)[\s\S]*memory_size\(runtime\.memory_bytes\)/, - ], - [ - "Wasm compute uses a refillable fuel token bucket", - /struct FuelTokenBucket[\s\S]*fractional_fuel_numerator[\s\S]*fn refill_after/, - ], - ["Wasmtime fuel consumption is enabled", /config\.consume_fuel\(true\)/], - ["Wasmtime epoch interruption is enabled", /config\.epoch_interruption\(true\)/], -]) { - expect(wasmRuntime, name, pattern); -} - -for (const [name, pattern] of [ - [ - "fuel refill preserves fractional credit", - /frequent_refills_preserve_fractional_credit/, - ], - [ - "linear memory growth is bounded per store", - /wasm_linear_memory_growth_is_bounded_per_store/, - ], - [ - "CPU-bound Wasm is interrupted without a host call", - /epoch_interruption_aborts_cpu_bound_wasm_without_a_host_call/, - ], -]) { - expect(`${wasmRuntime}\n${wasmRuntimeTests}`, name, pattern); -} - -for (const [name, pattern] of [ - [ - "rendezvous charges before transport planning", - /handle_request_rendezvous[\s\S]*charge_rendezvous_attempt\([\s\S]*&scope\.tenant[\s\S]*&scope\.project[\s\S]*now_epoch_seconds[\s\S]*plan_authenticated_direct_bulk_transfer/, - ], - [ - "artifact link creation preflights downloadable bytes before link creation", - /handle_create_artifact_download_link[\s\S]*downloadable_size[\s\S]*can_charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*downloadable_size[\s\S]*create_download_link/, - ], - [ - "artifact delivery charges scoped bytes before advancing its offset", - /handle_open_artifact_download_stream[\s\S]*stream_download_chunk\([\s\S]*charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*streamed_bytes[\s\S]*delivered_offset = end/, - ], -]) { - expect(coordinatorService, name, pattern); -} - -for (const [name, pattern] of [ - [ - "quota keys include tenant/project resource kind and window", - /struct ProjectQuotaScope[\s\S]*tenant: TenantId[\s\S]*project: ProjectId[\s\S]*struct MeterKey[\s\S]*kind: LimitKind[\s\S]*window: u64/, - ], - [ - "quota module discards expired windows for an accessed scope and kind", - /fn meter_mut[\s\S]*self\.meters\.retain[\s\S]*existing\.scope != key\.scope[\s\S]*existing\.kind != kind[\s\S]*existing\.window == key\.window/, - ], - [ - "quota module charges rendezvous attempts through the scoped window meter", - /fn charge_rendezvous_attempt[\s\S]*self\.charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::RendezvousAttempt/, - ], - [ - "quota module charges authenticated API calls through the scoped window meter", - /fn charge_api_call[\s\S]*self\.charge\(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds\)/, - ], - [ - "quota module preflights and charges log bytes through the scoped window meter", - /fn can_charge_log_bytes[\s\S]*LimitKind::LogBytes[\s\S]*fn charge_log_bytes[\s\S]*LimitKind::LogBytes/, - ], - [ - "quota module preflights artifact download bytes through the scoped meter", - /fn can_charge_download[\s\S]*self\.can_charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::ArtifactDownloadBytes/, - ], - [ - "quota status reports current scoped window usage", - /fn project_status[\s\S]*for kind in LimitKind::ALL[\s\S]*self\.used\(tenant, project, kind, now_epoch_seconds\)/, - ], -]) { - expect(coordinatorQuota, name, pattern); -} - -for (const [source, name, pattern] of [ - [ - coordinatorService, - "authenticated API calls are charged after session authorization and before dispatch", - /authenticate_cli_session[\s\S]*authorize_authenticated_user_operation[\s\S]*charge_api_call[\s\S]*match request/, - ], - [ - coordinatorLogs, - "signed node log ingestion preflights and charges bytes before accepting the report", - /handle_report_task_log[\s\S]*authorize_node_for_process_or_termination[\s\S]*can_charge_log_bytes[\s\S]*charge_log_bytes[\s\S]*TaskLogRecorded/, - ], - [ - coordinatorDebug, - "debug reads charge the scoped debug-read budget before audit state is recorded", - /record_debug_audit_event[\s\S]*charge_debug_read[\s\S]*DebugAuditEvent/, - ], - [ - coordinatorTests, - "tests prove API-call and log-byte quota enforcement and project isolation", - /authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch[\s\S]*project-a[\s\S]*project-b[\s\S]*signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes[\s\S]*LogBytes/, - ], -]) { - expect(source, name, pattern); -} - -for (const [name, source, patterns] of [ - [ - "rendezvous smoke", - quicSmoke, - [/charged_rendezvous_attempts, 1/, /coordinator bulk relay is disabled/], - ], - [ - "artifact download smoke", - artifactDownloadSmoke, - [ - /downloaded\.response\.charged_download_bytes,[\s\S]*packageEvent\.artifact_size_bytes/, - /retaining_node_reverse_stream/, - /revoked/, - ], - ], - [ - "operator panel smoke", - operatorPanelSmoke, - [ - /type: "submit_panel_event"/, - /max_events: 1/, - /used_events, 1/, - /rate limit/i, - /max_download_bytes: 1/, - /exceeds download limit/, - ], - ], -]) { - for (const pattern of patterns) { - expect(source, name, pattern); - } -} - -const privateHostedLibSource = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); -const privateHostedTests = maybeRead(["private", "hosted-policy", "src", "tests.rs"]); -const privateHostedLib = privateHostedLibSource - ? [privateHostedLibSource, privateHostedTests].filter(Boolean).join("\n") - : null; -if (privateHostedLib) { - for (const [name, pattern] of [ - [ - "private hosted configuration owns exact control-plane limits and quota windows", - /community_tier_resource_limits[\s\S]*LimitKind::ApiCall[\s\S]*LimitKind::ArtifactDownloadBytes[\s\S]*community_tier_quota_configuration[\s\S]*CoordinatorQuotaConfiguration::new/, - ], - [ - "hosted zero-capability policy rejects native execution capabilities", - /hosted_zero_capability_wasm_rejects_filesystem_and_command_capabilities[\s\S]*Capability::Command[\s\S]*Capability::Network[\s\S]*Capability::ArbitrarySyscalls/, - ], - ]) { - expect(privateHostedLib, name, pattern); - } - assert.doesNotMatch( - privateHostedLib, - /HostedFuel|HostedMemoryBytes|HostedWallClockMs|HostedStateBytes/, - "decorative hosted Wasm quota kinds must not return", - ); -} - -console.log("Resource metering contract smoke passed"); diff --git a/scripts/scheduler-placement-smoke.js b/scripts/scheduler-placement-smoke.js deleted file mode 100755 index 2050403..0000000 --- a/scripts/scheduler-placement-smoke.js +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); - -const repo = path.resolve(__dirname, ".."); -const digest = (value) => - `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; -const environmentDigest = digest("scheduler-linux-container"); -const dependencyDigest = digest("scheduler-toolchain-dependencies"); -const sourceDigest = digest("scheduler-source-tree"); - -function buildFlagshipBundle() { - const output = cp.execFileSync( - "cargo", - [ - "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", - "build", "--project", "tests/fixtures/runtime-conformance", "--json", - ], - { cwd: repo, encoding: "utf8" } - ); - const report = JSON.parse(output); - const directory = path.resolve(repo, report.bundle_artifact.directory); - const manifest = JSON.parse(fs.readFileSync(path.join(directory, "manifest.json"), "utf8")); - const entrypoints = JSON.parse( - fs.readFileSync(path.join(directory, manifest.entrypoints), "utf8") - ); - const entrypoint = entrypoints.find((candidate) => candidate.name === "build"); - assert(entrypoint, "flagship bundle omitted build entrypoint"); - const taskDescriptors = JSON.parse( - fs.readFileSync(path.join(directory, manifest.task_descriptors), "utf8") - ); - const prepareSource = taskDescriptors.find( - (candidate) => candidate.name === "prepare_source" - ); - assert(prepareSource, "flagship bundle omitted prepare_source task"); - return { - digest: manifest.bundle_digest, - taskExport: prepareSource.export, - wasmModuleBase64: fs.readFileSync(path.join(directory, "module.wasm")).toString("base64"), - }; -} - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function linuxCapabilities() { - return { - os: "Linux", - arch: "x86_64", - capabilities: [ - "Command", - "Containers", - "RootlessPodman", - "SourceFilesystem", - "VfsArtifacts" - ], - environment_backends: ["Container"], - source_providers: ["filesystem"] - }; -} - -function gitCapabilities() { - const capabilities = linuxCapabilities(); - capabilities.capabilities = [...capabilities.capabilities, "SourceGit"].sort(); - capabilities.source_providers = [...capabilities.source_providers, "git"].sort(); - return capabilities; -} - -async function attachNode(addr, node) { - const identity = nodeIdentity("scheduler-placement-smoke", node); - const attached = await send(addr, { - type: "attach_node", - tenant: "tenant", - project: "project", - node, - public_key: identity.publicKey - }); - assert.strictEqual(attached.type, "node_attached"); - assert.strictEqual(attached.node, node); - return identity; -} - -async function reportNode(addr, node, identity, locality) { - const recorded = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node, - capabilities: locality.capabilities || linuxCapabilities(), - cached_environment_digests: locality.cached_environment_digests, - dependency_cache_digests: locality.dependency_cache_digests, - source_snapshots: locality.source_snapshots, - artifact_locations: locality.artifact_locations, - direct_connectivity: locality.direct_connectivity !== false, - online: true - })); - assert.strictEqual( - recorded.type, - "node_capabilities_recorded", - JSON.stringify(recorded) - ); - assert.strictEqual(recorded.node, node); - return recorded; -} - -(async () => { - const bundle = buildFlagshipBundle(); - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - let coordinatorStderr = ""; - coordinator.stderr.on("data", (chunk) => { - coordinatorStderr += chunk.toString(); - }); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const coldNode = await attachNode(addr, "cold-node"); - const warmNode = await attachNode(addr, "warm-node"); - - const cold = await reportNode(addr, "cold-node", coldNode, { - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: false - }); - assert.strictEqual(cold.node_descriptors, 1); - - const warm = await reportNode(addr, "warm-node", warmNode, { - cached_environment_digests: [environmentDigest], - dependency_cache_digests: [dependencyDigest], - source_snapshots: [sourceDigest], - artifact_locations: ["toolchain-cache"] - }); - assert.strictEqual(warm.node_descriptors, 2); - const reportedNodes = new Set([cold.node, warm.node]); - assert.strictEqual(reportedNodes.size, 2); - assert(reportedNodes.has("cold-node")); - assert(reportedNodes.has("warm-node")); - - const inspected = await send(addr, { - type: "list_node_descriptors", - tenant: "tenant", - project: "project", - actor_user: "operator" - }); - assert.strictEqual(inspected.type, "node_descriptors"); - assert.strictEqual(inspected.actor, "operator"); - assert.strictEqual(inspected.descriptors.length, 2); - const warmDescriptor = inspected.descriptors.find( - (descriptor) => descriptor.id === "warm-node" - ); - assert(warmDescriptor, "warm node descriptor must be visible to inspector state"); - assert(warmDescriptor.capabilities.capabilities.includes("Command")); - assert(warmDescriptor.capabilities.capabilities.includes("RootlessPodman")); - assert(warmDescriptor.cached_environments.includes(environmentDigest)); - assert(warmDescriptor.dependency_caches.includes(dependencyDigest)); - assert(warmDescriptor.source_snapshots.includes(sourceDigest)); - assert(warmDescriptor.artifact_locations.includes("toolchain-cache")); - - const crossScopeInspection = await send(addr, { - type: "list_node_descriptors", - tenant: "other-tenant", - project: "project", - actor_user: "operator" - }); - assert.strictEqual(crossScopeInspection.type, "node_descriptors"); - assert.strictEqual(crossScopeInspection.descriptors.length, 0); - - const crossTenantReport = await send(addr, signedNodeRequest("warm-node", warmNode, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "other-tenant", - project: "project", - node: "warm-node", - capabilities: linuxCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: true, - online: true - })); - assert.strictEqual(crossTenantReport.type, "error"); - assert.match( - crossTenantReport.message, - /tenant\/project scope|node identity is not enrolled/ - ); - - const placement = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: { - os: "Linux", - arch: null, - capabilities: ["Containers", "RootlessPodman"] - }, - environment_digest: environmentDigest, - required_capabilities: ["Command"], - dependency_cache: dependencyDigest, - source_snapshot: sourceDigest, - required_artifacts: ["toolchain-cache"], - prefer_node: null - }); - assert.strictEqual(placement.type, "task_placement"); - assert.strictEqual(placement.placement.node, "warm-node"); - assert.ok(placement.placement.score > 0); - assert.ok(placement.placement.reasons.includes("warm environment cache")); - assert.ok(placement.placement.reasons.includes("warm dependency cache")); - assert.ok(placement.placement.reasons.includes("source snapshot already local")); - assert.ok( - placement.placement.reasons.includes("1 required artifact(s) already local") - ); - - const impossible = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: null, - environment_digest: null, - required_capabilities: ["WindowsCommandDev"], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - prefer_node: null - }); - assert.strictEqual(impossible.type, "error"); - assert.match(impossible.message, /WindowsCommandDev/); - - const started = await send(addr, { - type: "start_process", - tenant: "tenant", - project: "project", - actor_user: "operator", - process: "vp-wait-for-git" - }); - assert.strictEqual(started.type, "process_started"); - - const queued = await send(addr, { - type: "launch_task", - tenant: "tenant", - project: "project", - actor_user: "operator", - task_spec: { - tenant: "tenant", - project: "project", - process: "vp-wait-for-git", - task_definition: "prepare_source", - task_instance: "prepare_source-1", - dispatch: { - kind: "coordinator_node_wasm", - export: bundle.taskExport, - abi: "task_v1", - }, - environment_id: null, - environment: null, - environment_digest: null, - required_capabilities: ["SourceFilesystem", "SourceGit"], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [], - vfs_epoch: started.epoch, - bundle_digest: bundle.digest, - }, - wait_for_node: true, - artifact_path: "/vfs/artifacts/git-status.txt", - wasm_module_base64: bundle.wasmModuleBase64, - }); - assert.strictEqual(queued.type, "error"); - assert.match(queued.message, /external callers may launch only EntrypointV1/); - - const gitNode = await attachNode(addr, "git-node"); - const gitRecorded = await reportNode(addr, "git-node", gitNode, { - capabilities: gitCapabilities(), - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: false - }); - assert.strictEqual(gitRecorded.type, "node_capabilities_recorded"); - - const pendingAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { - type: "poll_task_assignment", - tenant: "tenant", - project: "project", - node: "git-node" - })); - assert.strictEqual(pendingAssignment.type, "task_assignment"); - assert.strictEqual(pendingAssignment.assignment, null); - - const emptyAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { - type: "poll_task_assignment", - tenant: "tenant", - project: "project", - node: "git-node" - })); - assert.strictEqual(emptyAssignment.type, "task_assignment"); - assert.strictEqual(emptyAssignment.assignment, null); - - await reportNode(addr, "warm-node", warmNode, { - cached_environment_digests: [], - dependency_cache_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: false - }); - const disconnectedTransfer = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: null, - environment_digest: null, - required_capabilities: ["Command"], - dependency_cache: null, - source_snapshot: sourceDigest, - required_artifacts: ["toolchain-cache"], - prefer_node: null - }); - assert.strictEqual(disconnectedTransfer.type, "error"); - assert.match(disconnectedTransfer.message, /source snapshot unavailable/); - assert.match(disconnectedTransfer.message, /required artifact\(s\) unavailable/); - assert.match(disconnectedTransfer.message, /direct connectivity unavailable/); - } catch (error) { - if (coordinatorStderr) { - error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; - } - throw error; - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("Scheduler placement smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/sdk-spawn-runtime-smoke.js b/scripts/sdk-spawn-runtime-smoke.js deleted file mode 100755 index 48fa978..0000000 --- a/scripts/sdk-spawn-runtime-smoke.js +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const sdk = fs.readFileSync(path.join(repo, "crates/clusterflux-sdk/src/lib.rs"), "utf8"); -const productRuntime = fs.readFileSync( - path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), - "utf8" -); - -assert.match(sdk, /pub struct RuntimeSpawnEvent/); -assert.match(sdk, /pub debugger_visible: bool/); -assert.match(sdk, /fn register_runtime_thread/); -assert.match(sdk, /register_runtime_thread\(id, self\.name, self\.env\)/); -assert.match(sdk, /pub fn debugger_visible\(&self\) -> bool/); -assert.match(sdk, /ProductRuntimeConfig::from_env\(\)/); -assert.match(sdk, /start_remote_task\(\s*config,\s*task_id/); -assert.match(sdk, /start_guest_host_task/); -assert.match(sdk, /join_remote_task\(remote\)/); -assert.doesNotMatch(productRuntime, /"type": "launch_task"/); -assert.match(productRuntime, /native SDK task spawning requires coordinator EntrypointV1 execution/); -assert.match(productRuntime, /task_start_v1/); -assert.match(productRuntime, /task_join_v1/); -assert.match(productRuntime, /command_run_v1/); -assert.match(productRuntime, /remote_completion_observed/); -assert.match(productRuntime, /TaskJoinState::Pending/); -assert.doesNotMatch( - productRuntime, - /entry\s*\(/, - "product runtime must not invoke the submitted local Rust closure" -); - -cp.execFileSync( - "cargo", - [ - "test", - "-p", - "clusterflux-sdk", - "spawn_task_start_registers_debugger_visible_runtime_thread", - ], - { cwd: repo, stdio: "inherit" } -); - -console.log("SDK spawn runtime smoke passed"); diff --git a/scripts/self-hosted-coordinator-smoke.js b/scripts/self-hosted-coordinator-smoke.js deleted file mode 100644 index 2938eac..0000000 --- a/scripts/self-hosted-coordinator-smoke.js +++ /dev/null @@ -1,666 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); - -const repo = path.resolve(__dirname, ".."); -const teamArtifactDigest = `sha256:${"a".repeat(64)}`; -const teamEnvironmentDigest = `sha256:${crypto - .createHash("sha256") - .update("env-team-linux") - .digest("hex")}`; -const teamDependencyCacheDigest = `sha256:${crypto - .createHash("sha256") - .update("cargo-cache") - .digest("hex")}`; -const teamSourceDigest = `sha256:${crypto - .createHash("sha256") - .update("source-team") - .digest("hex")}`; -const coordinatorTaskModule = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); -const coordinatorTaskBundleDigest = `sha256:${crypto - .createHash("sha256") - .update(coordinatorTaskModule) - .digest("hex")}`; -const adminToken = "self-hosted-smoke-admin-token"; -const clientSessionSecret = "self-hosted-smoke-client-session-secret"; -const releaseManifestPath = - process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || - path.join(repo, "target/public-release/public-release-manifest.json"); - -function sha256(value) { - return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; -} - -function digestFromParts(parts) { - const hash = crypto.createHash("sha256"); - for (const value of parts) { - const bytes = Buffer.from(String(value)); - const length = Buffer.alloc(8); - length.writeBigUInt64BE(BigInt(bytes.length)); - hash.update(length); - hash.update(bytes); - } - return `sha256:${hash.digest("hex")}`; -} - -function adminRequest(token, operation, tenant, actorUser, targetTenant, nonce) { - const issuedAtEpochSeconds = Math.floor(Date.now() / 1000); - return { - type: operation, - tenant, - actor_user: actorUser, - ...(operation === "suspend_tenant" ? { target_tenant: targetTenant } : {}), - admin_proof: digestFromParts([ - "clusterflux-admin-request-proof:v1", - sha256(token), - operation, - tenant, - actorUser, - targetTenant, - nonce, - issuedAtEpochSeconds, - ]), - admin_nonce: nonce, - issued_at_epoch_seconds: issuedAtEpochSeconds, - }; -} - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function authenticated(request, sessionSecret = clientSessionSecret) { - return { - type: "authenticated", - session_secret: sessionSecret, - request, - }; -} - -function linuxNodeCapabilities() { - return { - os: "Linux", - arch: "x86_64", - capabilities: ["Command", "RootlessPodman", "VfsArtifacts", "QuicDirect"], - environment_backends: ["Container"], - source_providers: ["filesystem", "git"] - }; -} - -function nodeIdentity(node) { - void node; - const { privateKey: privateKeyObject, publicKey } = - crypto.generateKeyPairSync("ed25519"); - const publicDer = publicKey.export({ - format: "der", - type: "spki", - }); - return { - publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, - privateKeyObject, - }; -} - -function signedRequestPayloadDigest(request) { - const canonicalize = (value, topLevel = true) => { - if (Array.isArray(value)) return value.map((entry) => canonicalize(entry, false)); - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value) - .filter( - ([key, entry]) => - entry !== null && - (!topLevel || !["agent_signature", "node_signature"].includes(key)) - ) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([key, entry]) => [key, canonicalize(entry, false)]) - ); - } - return value; - }; - return `sha256:${crypto - .createHash("sha256") - .update(JSON.stringify(canonicalize(request))) - .digest("hex")}`; -} - -function nodeSignatureMessage( - node, - requestKind, - payloadDigest, - nonce, - issuedAtEpochSeconds -) { - const parts = [ - "clusterflux-node-request-signature:v2", - node, - requestKind, - payloadDigest, - nonce, - String(issuedAtEpochSeconds), - ]; - return Buffer.concat( - parts.flatMap((part) => [ - Buffer.from(`${Buffer.byteLength(part)}:`), - Buffer.from(part), - Buffer.from("\n"), - ]) - ); -} - -function signedNodeHeartbeat(tenant, project, node, identity) { - const request = { type: "node_heartbeat", tenant, project, node }; - const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`; - const issuedAt = Math.floor(Date.now() / 1000); - const signature = crypto.sign( - null, - nodeSignatureMessage( - node, - "node_heartbeat", - signedRequestPayloadDigest(request), - nonce, - issuedAt - ), - identity.privateKeyObject - ); - return { - nonce, - issued_at_epoch_seconds: issuedAt, - signature: `ed25519:${signature.toString("base64")}`, - }; -} - -function signedNodeRequest(node, identity, requestKind, request) { - return { - type: "signed_node", - node, - node_signature: signedNodeHeartbeatForKind(node, identity, requestKind, request), - request, - }; -} - -function signedNodeHeartbeatForKind(node, identity, requestKind, request) { - const nonce = `${requestKind}-${process.pid}-${Date.now()}`; - const issuedAt = Math.floor(Date.now() / 1000); - const signature = crypto.sign( - null, - nodeSignatureMessage( - node, - requestKind, - signedRequestPayloadDigest(request), - nonce, - issuedAt - ), - identity.privateKeyObject - ); - return { - nonce, - issued_at_epoch_seconds: issuedAt, - signature: `ed25519:${signature.toString("base64")}`, - }; -} - -function commandOutput(command, args) { - try { - return cp - .execFileSync(command, args, { - cwd: repo, - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }) - .trim(); - } catch (_) { - return null; - } -} - -function runJson(command, args, options = {}) { - return JSON.parse( - cp.execFileSync(command, args, { - cwd: options.cwd || repo, - encoding: "utf8", - input: options.input, - }) - ); -} - -function expectedSourceCommit() { - return ( - process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || - commandOutput("git", ["rev-parse", "HEAD"]) - ); -} - -function readReleaseManifest() { - if (!fs.existsSync(releaseManifestPath)) return null; - const manifest = JSON.parse(fs.readFileSync(releaseManifestPath, "utf8")); - if (manifest.kind !== "clusterflux-public-release") return null; - const expectedCommit = expectedSourceCommit(); - if (expectedCommit && manifest.source_commit !== expectedCommit) return null; - return manifest; -} - -function releaseIdentity() { - const manifest = readReleaseManifest(); - return { - sourceCommit: manifest ? manifest.source_commit : expectedSourceCommit(), - releaseName: - (manifest && manifest.release_name) || - process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || - null, - }; -} - -async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilities()) { - const identity = nodeIdentity(node); - const grant = await send(addr, authenticated({ - type: "create_node_enrollment_grant", - ttl_seconds: 900, - })); - assert.strictEqual(grant.type, "node_enrollment_grant_created"); - - const attached = await send(addr, { - type: "exchange_node_enrollment_grant", - tenant: "team", - project: "self-hosted", - node, - public_key: identity.publicKey, - enrollment_grant: grant.grant, - }); - assert.strictEqual(attached.type, "node_enrollment_exchanged"); - - const heartbeat = await send(addr, { - type: "node_heartbeat", - tenant: "team", - project: "self-hosted", - node, - node_signature: signedNodeHeartbeat("team", "self-hosted", node, identity) - }); - assert.strictEqual(heartbeat.type, "node_heartbeat"); - - const reported = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "team", - project: "self-hosted", - node, - capabilities, - cached_environment_digests: [teamEnvironmentDigest], - dependency_cache_digests: [teamDependencyCacheDigest], - source_snapshots: [teamSourceDigest], - artifact_locations: [], - direct_connectivity: true, - online: true - })); - assert.strictEqual(reported.type, "node_capabilities_recorded"); - return identity; -} - -(async () => { - const release = releaseIdentity(); - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0" - ], - { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_ADMIN_TOKEN: adminToken, - CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET: clientSessionSecret, - CLUSTERFLUX_SELF_HOSTED_TENANT: "team", - CLUSTERFLUX_SELF_HOSTED_PROJECT: "self-hosted", - CLUSTERFLUX_SELF_HOSTED_USER: "developer", - }, - } - ); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual(ready.client_authority, "strict"); - assert.strictEqual(ready.self_hosted_session_bootstrapped, true); - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const forgedBodyAuthority = await send(addr, { - type: "start_process", - tenant: "victim-tenant", - project: "victim-project", - actor_user: "forged-user", - process: "vp-forged", - }); - assert.strictEqual(forgedBodyAuthority.type, "error"); - assert.match(forgedBodyAuthority.message, /body.*identity.*not authority/i); - const wrongSession = await send( - addr, - authenticated({ type: "auth_status" }, "wrong-session-secret") - ); - assert.strictEqual(wrongSession.type, "error"); - assert.match(wrongSession.message, /session.*not recognized|not recognized.*session/i); - const authStatus = await send(addr, authenticated({ type: "auth_status" })); - assert.strictEqual(authStatus.type, "auth_status"); - assert.strictEqual(authStatus.tenant, "team"); - assert.strictEqual(authStatus.project, "self-hosted"); - assert.strictEqual(authStatus.actor, "developer"); - - const missingAdminCredential = await send( - addr, - adminRequest("", "admin_status", "team", "forged-admin", "team", "admin-empty") - ); - assert.strictEqual(missingAdminCredential.type, "error"); - assert.match(missingAdminCredential.message, /admin.*proof.*invalid/i); - const wrongAdminCredential = await send( - addr, - adminRequest( - "wrong-token", - "admin_status", - "team", - "forged-admin", - "team", - "admin-wrong" - ) - ); - assert.strictEqual(wrongAdminCredential.type, "error"); - assert.match(wrongAdminCredential.message, /admin.*proof.*invalid/i); - const replayableAdminRequest = adminRequest( - adminToken, - "admin_status", - "team", - "self-hosted-admin", - "team", - "admin-replay" - ); - const directAdminStatus = await send(addr, replayableAdminRequest); - assert.strictEqual(directAdminStatus.type, "admin_status"); - const replayedAdminStatus = await send(addr, replayableAdminRequest); - assert.strictEqual(replayedAdminStatus.type, "error"); - assert.match(replayedAdminStatus.message, /nonce was already used/i); - const cargoTargetDir = path.resolve( - process.env.CARGO_TARGET_DIR || path.join(repo, "target") - ); - const cliBin = path.join( - cargoTargetDir, - "debug", - process.platform === "win32" ? "clusterflux.exe" : "clusterflux" - ); - const selfHostedCliProject = path.join( - repo, - "target/acceptance/self-hosted-cli-project" - ); - fs.rmSync(selfHostedCliProject, { recursive: true, force: true }); - fs.mkdirSync(selfHostedCliProject, { recursive: true }); - const cliConnect = runJson( - cliBin, - [ - "auth", - "connect-self-hosted", - "--coordinator", - ready.listen, - "--tenant", - "team", - "--project-id", - "self-hosted", - "--user", - "developer", - "--session-secret-stdin", - "--json", - ], - { cwd: selfHostedCliProject, input: `${clientSessionSecret}\n` } - ); - assert.strictEqual(cliConnect.status, "connected"); - assert.strictEqual(cliConnect.session_secret_read_from_stdin, true); - assert.strictEqual(cliConnect.session_secret_exposed_in_report, false); - assert.strictEqual(cliConnect.coordinator_response.type, "auth_status"); - const sessionPath = path.join( - selfHostedCliProject, - ".clusterflux/session.json" - ); - const storedSession = JSON.parse(fs.readFileSync(sessionPath, "utf8")); - assert.strictEqual(storedSession.kind, "self_hosted"); - assert.strictEqual(storedSession.session_secret, clientSessionSecret); - assert.strictEqual(storedSession.provider_tokens_exposed_to_cli, false); - assert.strictEqual(storedSession.provider_tokens_sent_to_nodes, false); - if (process.platform !== "win32") { - assert.strictEqual(fs.statSync(sessionPath).mode & 0o777, 0o600); - } - const cliAuthStatus = runJson( - cliBin, - ["auth", "status", "--json"], - { cwd: selfHostedCliProject } - ); - assert.strictEqual(cliAuthStatus.active_coordinator, ready.listen); - assert.strictEqual( - cliAuthStatus.coordinator_account_status.used_cli_session_credential, - true - ); - assert.strictEqual( - cliAuthStatus.coordinator_account_status.coordinator_response_type, - "auth_status" - ); - const cliAdminStatus = runJson(cliBin, [ - "admin", - "status", - "--coordinator", - ready.listen, - "--tenant", - "team", - "--user", - "self-hosted-admin", - "--admin-token", - adminToken, - "--json", - ]); - assert.strictEqual(cliAdminStatus.response.type, "admin_status"); - assert.strictEqual(cliAdminStatus.suspended, false); - const cliAdminSuspend = runJson(cliBin, [ - "admin", - "suspend-tenant", - "--coordinator", - ready.listen, - "--tenant", - "team", - "--user", - "self-hosted-admin", - "--target-tenant", - "admin-probe-tenant", - "--admin-token", - adminToken, - "--yes", - "--json", - ]); - assert.strictEqual(cliAdminSuspend.response.type, "tenant_suspended"); - assert.strictEqual(cliAdminSuspend.suspended, true); - const cliAdminProbeStatus = runJson(cliBin, [ - "admin", - "status", - "--coordinator", - ready.listen, - "--tenant", - "admin-probe-tenant", - "--user", - "self-hosted-admin", - "--admin-token", - adminToken, - "--json", - ]); - assert.strictEqual(cliAdminProbeStatus.suspended, true); - - const teamLinuxA = await attachTrustedNode(addr, "team-linux-a"); - const teamLinuxBCapabilities = linuxNodeCapabilities(); - teamLinuxBCapabilities.capabilities = teamLinuxBCapabilities.capabilities.filter( - (capability) => capability !== "VfsArtifacts" - ); - await attachTrustedNode(addr, "team-linux-b", teamLinuxBCapabilities); - - const placement = await send(addr, authenticated({ - type: "schedule_task", - environment: { - os: "Linux", - arch: "x86_64", - capabilities: ["Command", "QuicDirect"] - }, - environment_digest: teamEnvironmentDigest, - required_capabilities: ["VfsArtifacts"], - source_snapshot: teamSourceDigest, - required_artifacts: [], - prefer_node: "team-linux-a" - })); - assert.strictEqual(placement.type, "task_placement"); - assert.strictEqual(placement.placement.node, "team-linux-a"); - assert(placement.placement.reasons.includes("preferred node")); - assert(placement.placement.reasons.includes("warm environment cache")); - assert(placement.placement.reasons.includes("source snapshot already local")); - - const started = await send(addr, authenticated({ - type: "start_process", - process: "vp-team-build", - restart: false, - })); - assert.strictEqual(started.type, "process_started"); - - const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", { - type: "reconnect_node", - tenant: "team", - project: "self-hosted", - node: "team-linux-a", - process: "vp-team-build", - epoch: started.epoch - })); - assert.strictEqual(reconnected.type, "node_reconnected"); - - const launched = await send(addr, authenticated({ - type: "launch_task", - task_spec: { - tenant: "team", - project: "self-hosted", - process: "vp-team-build", - task_definition: "compile-linux", - task_instance: "compile-linux", - dispatch: { - kind: "coordinator_node_wasm", - export: "compile_linux", - abi: "task_v1", - }, - environment_id: null, - environment: null, - environment_digest: null, - required_capabilities: ["VfsArtifacts"], - dependency_cache: null, - source_snapshot: null, - required_artifacts: [], - args: [], - vfs_epoch: started.epoch, - bundle_digest: coordinatorTaskBundleDigest, - }, - wait_for_node: false, - artifact_path: "/vfs/artifacts/team-output.txt", - wasm_module_base64: coordinatorTaskModule.toString("base64"), - })); - assert.strictEqual(launched.type, "error", JSON.stringify(launched)); - assert.match(launched.message, /external callers may launch only EntrypointV1/); - - const reportPath = path.join(repo, "target/acceptance/core-coordinator-compat.json"); - fs.mkdirSync(path.dirname(reportPath), { recursive: true }); - fs.writeFileSync( - reportPath, - `${JSON.stringify( - { - kind: "clusterflux-core-coordinator-compatibility", - source_commit: release.sourceCommit, - release_name: release.releaseName, - coordinator_implementation: "standalone-core-coordinator", - coordinator_addr: ready.listen, - client_authority: ready.client_authority, - authenticated_session: authStatus.authenticated, - forged_body_authority_denied: forgedBodyAuthority.type, - wrong_session_denied: wrongSession.type, - ping: "pong", - nodes: ["team-linux-a", "team-linux-b"], - task_placement: placement.type, - process_started: started.type, - external_task_v1_denied: launched.type, - self_hosted_admin: { - missing_credential_denied: missingAdminCredential.type, - wrong_credential_denied: wrongAdminCredential.type, - nonce_bound_proof_succeeded: directAdminStatus.type, - replay_denied: replayedAdminStatus.type, - cli_status: cliAdminStatus.response.type, - cli_suspend: cliAdminSuspend.response.type, - suspended_state_observed: cliAdminProbeStatus.suspended, - }, - self_hosted_cli: { - connected: cliConnect.status, - secret_read_from_stdin: cliConnect.session_secret_read_from_stdin, - secret_exposed_in_report: cliConnect.session_secret_exposed_in_report, - session_file_mode: - process.platform === "win32" - ? "windows-best-effort" - : (fs.statSync(sessionPath).mode & 0o777).toString(8), - authenticated_status: - cliAuthStatus.coordinator_account_status.coordinator_response_type, - }, - }, - null, - 2 - )}\n` - ); - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("Self-hosted coordinator smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/source-preparation-smoke.js b/scripts/source-preparation-smoke.js deleted file mode 100755 index 07be55d..0000000 --- a/scripts/source-preparation-smoke.js +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); - -const repo = path.resolve(__dirname, ".."); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function sourceCapableNode(sourceProviders = ["git"]) { - return { - os: "Linux", - arch: "x86_64", - capabilities: ["Command", "SourceFilesystem", "SourceGit"], - environment_backends: [], - source_providers: sourceProviders - }; -} - -(async () => { - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - let coordinatorStderr = ""; - coordinator.stderr.on("data", (chunk) => { - coordinatorStderr += chunk.toString(); - }); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const pending = await send(addr, { - type: "request_source_preparation", - tenant: "tenant", - project: "project", - provider: "Git" - }); - assert.strictEqual(pending.type, "source_preparation"); - assert.strictEqual(pending.status.preparation.tenant, "tenant"); - assert.strictEqual(pending.status.preparation.project, "project"); - assert.strictEqual(pending.status.preparation.provider, "Git"); - assert.strictEqual( - pending.status.preparation.coordinator_requires_checkout_access, - false - ); - assert.deepStrictEqual(pending.status.preparation.required_capabilities, [ - "SourceGit" - ]); - assert.match(pending.status.disposition.Pending.reason, /waiting|node/i); - - const nodeIdentities = new Map(); - for (const node of ["source-cold", "source-ready"]) { - const identity = nodeIdentity("source-preparation-smoke", node); - nodeIdentities.set(node, identity); - const attached = await send(addr, { - type: "attach_node", - tenant: "tenant", - project: "project", - node, - public_key: identity.publicKey - }); - assert.strictEqual(attached.type, "node_attached"); - } - - const cold = await send(addr, signedNodeRequest("source-cold", nodeIdentities.get("source-cold"), "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: "source-cold", - capabilities: sourceCapableNode(), - cached_environment_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: true, - online: true - })); - assert.strictEqual(cold.type, "node_capabilities_recorded"); - - const readyReport = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: "source-ready", - capabilities: sourceCapableNode(), - cached_environment_digests: [], - source_snapshots: [], - artifact_locations: [], - direct_connectivity: true, - online: true - })); - assert.strictEqual(readyReport.type, "node_capabilities_recorded"); - - const assigned = await send(addr, { - type: "request_source_preparation", - tenant: "tenant", - project: "project", - provider: "Git" - }); - assert.strictEqual(assigned.type, "source_preparation"); - assert(["source-cold", "source-ready"].includes(assigned.status.disposition.Assigned.node)); - assert.strictEqual( - assigned.status.preparation.coordinator_requires_checkout_access, - false - ); - - const crossTenantCompletion = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { - type: "complete_source_preparation", - tenant: "other", - project: "project", - node: "source-ready", - provider: "Git", - source_snapshot: "sha256:source-prepared" - })); - assert.strictEqual(crossTenantCompletion.type, "error"); - assert.match( - crossTenantCompletion.message, - /tenant\/project scope|node identity is not enrolled/i - ); - - const completed = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { - type: "complete_source_preparation", - tenant: "tenant", - project: "project", - node: "source-ready", - provider: "Git", - source_snapshot: "sha256:source-prepared" - })); - assert.strictEqual(completed.type, "source_preparation_completed"); - assert.strictEqual(completed.node, "source-ready"); - assert.strictEqual(completed.provider, "Git"); - assert.strictEqual(completed.source_snapshot, "sha256:source-prepared"); - - const placement = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: null, - environment_digest: null, - required_capabilities: ["SourceGit"], - source_snapshot: "sha256:source-prepared", - required_artifacts: [], - prefer_node: null - }); - assert.strictEqual(placement.type, "task_placement"); - assert.strictEqual(placement.placement.node, "source-ready"); - assert( - placement.placement.reasons.includes("source snapshot already local"), - "completed source preparation must update node source locality" - ); - } catch (error) { - if (coordinatorStderr) { - error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; - } - throw error; - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("Source preparation smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/tenant-isolation-contract-smoke.js b/scripts/tenant-isolation-contract-smoke.js deleted file mode 100644 index 65a1c9d..0000000 --- a/scripts/tenant-isolation-contract-smoke.js +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); - -function read(relativePath) { - return fs.readFileSync(path.join(repo, relativePath), "utf8"); -} - -function maybeRead(segments) { - const fullPath = path.join(repo, ...segments); - if (!fs.existsSync(fullPath)) return null; - return fs.readFileSync(fullPath, "utf8"); -} - -function expect(source, name, pattern) { - assert.match(source, pattern, `missing tenant-isolation evidence: ${name}`); -} - -const auth = read("crates/clusterflux-core/src/auth.rs"); -const artifact = read("crates/clusterflux-core/src/artifact.rs"); -const operatorPanel = read("crates/clusterflux-core/src/operator_panel.rs"); -const source = read("crates/clusterflux-core/src/source.rs"); -const coordinatorService = [ - read("crates/clusterflux-coordinator/src/service.rs"), - read("crates/clusterflux-coordinator/src/service/routing.rs"), - read("crates/clusterflux-coordinator/src/service/nodes.rs"), - read("crates/clusterflux-coordinator/src/service/keys.rs"), - read("crates/clusterflux-coordinator/src/service/artifacts.rs"), - read("crates/clusterflux-coordinator/src/service/processes.rs"), - read("crates/clusterflux-coordinator/src/service/process_launch.rs"), - read("crates/clusterflux-coordinator/src/service/tests.rs"), -].join("\n"); -const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); -const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); -const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); -const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); -const liveSmoke = read("scripts/cli-happy-path-live-smoke.js"); - -for (const [name, pattern] of [ - ["auth contexts carry tenant and project", /pub struct AuthContext[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], - ["enrollment grants carry tenant and project", /pub struct EnrollmentGrant[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], - ["node credentials carry tenant and project", /pub struct NodeCredential[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], - ["same tenant/project denies tenant mismatch", /pub fn same_tenant_project[\s\S]*tenant mismatch/], - ["same tenant/project denies project mismatch", /pub fn same_tenant_project[\s\S]*project mismatch/], - ["auth unit denies cross-tenant access", /fn tenant_project_scope_denies_cross_tenant_access\(\)/], -]) { - expect(auth, name, pattern); -} - -for (const [name, pattern] of [ - ["artifact metadata carries tenant and project", /pub struct ArtifactMetadata[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], - ["download links carry tenant project process and actor", /pub struct DownloadLink[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId[\s\S]*pub actor: Actor/], - ["downloads authorize against scoped metadata", /let authz = same_tenant_project\(context, &scope\)/], - ["artifact test denies cross-tenant download", /fn cross_tenant_download_is_denied_even_with_known_artifact_id\(\)/], - ["artifact test denies cross-project download", /fn cross_project_download_is_denied_even_with_known_artifact_id\(\)/], -]) { - expect(artifact, name, pattern); -} - -for (const [name, pattern] of [ - ["panel state carries tenant project and process", /pub struct PanelState[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], - ["panel events carry tenant project and process", /pub struct PanelEvent[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], - ["panel events reject scope mismatch", /PanelError::ScopeMismatch/], - ["panel scope validation compares tenant project process", /self\.tenant != event\.tenant[\s\S]*self\.project != event\.project[\s\S]*self\.process != event\.process/], -]) { - expect(operatorPanel, name, pattern); -} - -expect( - source, - "source preparation carries tenant and project", - /pub struct SourcePreparation[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/ -); - -for (const [name, pattern] of [ - ["project creation rejects foreign tenant reuse", /project id is outside the signed-in tenant scope/], - ["project selection rejects foreign tenant", /project is outside the signed-in tenant scope/], - ["project listing uses tenant-scoped context", /CoordinatorRequest::ListProjects[\s\S]*list_projects\(&context\)/], - [ - "node capability reports resolve the full enrolled scope", - /NodeScopeKey::from_refs\(&tenant, &project, &node\)[\s\S]*node_identity\(&tenant, &project, &node\)/, - ], - ["operator panel stop state is tenant/project/process keyed", /type PanelStopKey = \(TenantId, ProjectId, ProcessId\)/], - ["download service hides cross-tenant artifact existence", /cross_tenant\.to_string\(\)\.contains\("does not exist"\)/], - ["download service hides cross-project artifact existence", /cross_project\.to_string\(\)\.contains\("does not exist"\)/], - ["node capability test rejects cross-scope report", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], - ["source preparation test rejects cross-scope completion", /fn service_rejects_source_preparation_completion_outside_node_scope\(\)/], -]) { - expect(coordinatorService, name, pattern); -} - -for (const [name, sourceText, patterns] of [ - [ - "artifact download smoke", - artifactDownloadSmoke, - [ - /const crossTenant = await send/, - /const crossProject = await send/, - /const crossTenantOpen = await send/, - /const crossProjectOpen = await send/, - /artifact does not exist/, - /token is invalid/, - ], - ], - [ - "operator panel smoke", - operatorPanelSmoke, - [/const crossTenant = await send/, /render_operator_panel/, /scope\|tenant\|project/], - ], - [ - "scheduler and capability smoke", - schedulerSmoke, - [ - /const crossScopeInspection = await send/, - /assert\.strictEqual\(crossScopeInspection\.descriptors\.length, 0\)/, - /const crossTenantReport = await send/, - /tenant\\\/project scope/, - ], - ], - [ - "source preparation smoke", - sourcePreparationSmoke, - [/const crossTenantCompletion = await send/, /complete_source_preparation/, /tenant\\\/project scope/i], - ], -]) { - for (const pattern of patterns) { - expect(sourceText, name, pattern); - } -} - -for (const [name, pattern] of [ - [ - "live same-ID collision refreshes first-tenant metadata before the second build", - /const firstCollisionHelloBuild = await runHostedHelloBuild[\s\S]*secondHelloBuild = await runHostedHelloBuild/, - ], - [ - "live same-ID collision re-reads the fresh first-tenant process", - /"--process",[\s\S]*firstCollisionHelloBuild\.process[\s\S]*candidate\.artifact === firstCollisionHelloBuild\.artifact/, - ], -]) { - expect(liveSmoke, name, pattern); -} - -const hostedLibRoot = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); -const hostedServiceSource = maybeRead([ - "private", - "hosted-policy", - "src", - "bin", - "clusterflux-hosted-service.rs", -]); -const hostedLib = hostedLibRoot; -const hostedSmoke = maybeRead([ - "private", - "hosted-policy", - "scripts", - "hosted-client-compat-smoke.js", -]); - -if (hostedLib && hostedServiceSource && hostedSmoke) { - for (const [name, pattern] of [ - ["hosted private layer is a compact identity broker", /pub struct HostedIdentityBroker[\s\S]*cli_session_ttl_seconds/], - ["hosted private layer has no parallel coordinator", /Runtime, persistence, nodes, processes,[\s\S]*remain owned by public CoordinatorService/], - ["hosted service delegates Client state to Core", /core_coordinator: CoordinatorService/], - ["hosted login creates project through Core", /issue_cli_session[\s\S]*AuthenticatedCoordinatorRequest::CreateProject/], - ]) { - expect(`${hostedLib}\n${hostedServiceSource}`, name, pattern); - } - - for (const forbidden of [ - "HostedCommunityControlPlane", - "HostedObservabilitySnapshot", - "agent_public_keys: BTreeMap", - "node_statuses: BTreeMap", - "process_statuses: BTreeMap", - "debug_sessions: BTreeMap", - ]) { - assert( - !hostedLib.includes(forbidden), - `private hosted policy must not reimplement Core state: ${forbidden}` - ); - } - - for (const [name, pattern] of [ - ["client-supplied identity is denied", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], - ["second OIDC subject receives a different tenant", /assert\.notStrictEqual\(victimLogin\.session\.tenant, session\.tenant\)/], - ["cross-tenant process events are denied", /const crossTenantTaskEventsDenied = await sendHostedControl[\s\S]*vp-victim[\s\S]*scope\|denied\|unauthorized/], - ]) { - expect(hostedSmoke, name, pattern); - } -} - -console.log("Tenant isolation contract smoke passed"); diff --git a/scripts/user-session-token-boundary-smoke.js b/scripts/user-session-token-boundary-smoke.js deleted file mode 100755 index d00a9da..0000000 --- a/scripts/user-session-token-boundary-smoke.js +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); - -function read(file) { - return fs.readFileSync(path.join(repo, file), "utf8"); -} - -function extractBalancedBlock(source, marker) { - const start = source.indexOf(marker); - assert(start >= 0, `missing marker ${marker}`); - const open = source.indexOf("{", start); - assert(open >= 0, `missing opening brace for ${marker}`); - let depth = 0; - for (let index = open; index < source.length; index += 1) { - const char = source[index]; - if (char === "{") depth += 1; - if (char === "}") { - depth -= 1; - if (depth === 0) return source.slice(start, index + 1); - } - } - throw new Error(`missing closing brace for ${marker}`); -} - -function extractEnumVariant(source, variant) { - const marker = ` ${variant} {`; - return extractBalancedBlock(source, marker); -} - -const forbiddenUserSessionCredentials = - /\b(BrowserSession|CliDeviceSession|BrowserLoginFlow|CliLoginFlow|authorization_url|verification_url|device_code|access_token|refresh_token|provider_token|provider_tokens|session_token|user_token|oauth_token)\b/i; - -function assertNoUserSessionCredential(surface, text) { - assert.doesNotMatch( - text, - forbiddenUserSessionCredentials, - `${surface} must not carry user OAuth/browser/session credentials` - ); -} - -const coordinatorService = `${read("crates/clusterflux-coordinator/src/service/protocol.rs")}\n${read("crates/clusterflux-coordinator/src/service/protocol/responses.rs")}`; -for (const variant of [ - "AdminStatus", - "SuspendTenant", - "AttachNode", - "RegisterAgentPublicKey", - "ListAgentPublicKeys", - "RotateAgentPublicKey", - "RevokeAgentPublicKey", - "NodeHeartbeat", - "ReportNodeCapabilities", - "RevokeNodeCredential", - "RequestRendezvous", - "RequestSourcePreparation", - "CompleteSourcePreparation", - "StartProcess", - "ReconnectNode", - "CancelTask", - "CancelProcess", - "PollTaskControl", - "RestartTask", - "DebugAttach", - "TaskCompleted", -]) { - assertNoUserSessionCredential( - `CoordinatorRequest::${variant}`, - extractEnumVariant(coordinatorService, variant) - ); -} - -const nodeRuntime = [ - read("crates/clusterflux-node/src/lib.rs"), - read("crates/clusterflux-node/src/command_runner.rs"), -].join("\n"); -for (const marker of [ - "pub struct LinuxCommandRunPlan", - "pub struct LinuxCommandTaskOutput", - "pub struct CapturedCommandLogs", - "pub struct VirtualThreadCommand", - "pub struct CommandOutput", -]) { - assertNoUserSessionCredential(marker, extractBalancedBlock(nodeRuntime, marker)); -} - -const coreExecution = read("crates/clusterflux-core/src/execution.rs"); -assertNoUserSessionCredential( - "CommandInvocation", - extractBalancedBlock(coreExecution, "pub struct CommandInvocation") -); - -const dapAdapter = read("crates/clusterflux-dap/src/variables.rs"); -assertNoUserSessionCredential( - "DAP variables response", - extractBalancedBlock(dapAdapter, "fn variables_response") -); - -const panel = read("crates/clusterflux-core/src/operator_panel.rs"); -assertNoUserSessionCredential( - "PanelEvent", - extractBalancedBlock(panel, "pub struct PanelEvent") -); - -const auth = read("crates/clusterflux-core/src/auth.rs"); -assert.match( - auth, - /task_credentials_do_not_contain_user_session/, - "core auth must keep the task credential user-session guard" -); -assert.match( - auth, - /CredentialKind::BrowserSession \| CredentialKind::CliDeviceSession/, - "task credential guard must reject browser and CLI sessions" -); - -console.log("User session token boundary smoke passed"); diff --git a/scripts/verify-public-split.sh b/scripts/verify-public-split.sh deleted file mode 100755 index f264477..0000000 --- a/scripts/verify-public-split.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -source_commit="$(git -C "$repo_root" rev-parse HEAD)" -export CLUSTERFLUX_ACCEPTANCE_COMMIT="$source_commit" -tmp_dir="$(mktemp -d)" -trap 'rm -rf "$tmp_dir"' EXIT - -tar \ - --exclude='./.git' \ - --exclude='./target' \ - --exclude='./private' \ - --exclude='./internal' \ - --exclude='./experiments' \ - --exclude='./.clusterflux' \ - --exclude='./vscode-extension/node_modules' \ - --exclude='./scripts/containers-home' \ - -C "$repo_root" \ - -cf - . | tar -C "$tmp_dir" -xf - - -if find "$tmp_dir" -path "$tmp_dir/private" -print -quit | grep -q .; then - echo "private directory leaked into public split" >&2 - exit 1 -fi - -if find "$tmp_dir" -path "$tmp_dir/experiments" -print -quit | grep -q .; then - echo "experiments directory leaked into public split" >&2 - exit 1 -fi - -if find "$tmp_dir" -path "$tmp_dir/internal" -print -quit | grep -q .; then - echo "internal directory leaked into public split" >&2 - exit 1 -fi - -(cd "$tmp_dir" && scripts/check-old-name.sh) -(cd "$tmp_dir" && CLUSTERFLUX_FILTERED_PUBLIC_TREE=1 node scripts/check-docs.js) -(cd "$tmp_dir" && scripts/check-code-size.sh) -(cd "$tmp_dir" && node scripts/resource-metering-contract-smoke.js) -(cd "$tmp_dir" && node scripts/hostile-input-contract-smoke.js) -(cd "$tmp_dir" && node scripts/tenant-isolation-contract-smoke.js) -(cd "$tmp_dir" && cargo fmt --all --check) -CARGO_TARGET_DIR="$tmp_dir/target" cargo test \ - --workspace \ - --all-targets \ - --manifest-path "$tmp_dir/Cargo.toml" -CARGO_TARGET_DIR="$tmp_dir/target" cargo build \ - --workspace \ - --all-targets \ - --manifest-path "$tmp_dir/Cargo.toml" -(cd "$tmp_dir" && node scripts/cli-install-smoke.js) -(cd "$tmp_dir" && node scripts/flagship-demo-smoke.js) -(cd "$tmp_dir" && node scripts/cli-local-run-smoke.js) -(cd "$tmp_dir" && node scripts/node-attach-smoke.js) -(cd "$tmp_dir" && node scripts/vscode-extension-smoke.js) -(cd "$tmp_dir" && node scripts/vscode-f5-smoke.js) -(cd "$tmp_dir" && node scripts/artifact-download-smoke.js) -(cd "$tmp_dir" && node scripts/artifact-export-smoke.js) - -public_digest="$( - find "$tmp_dir" \ - -path "$tmp_dir/target" -prune -o \ - -type f -print0 \ - | LC_ALL=C sort -z \ - | xargs -0 sha256sum \ - | sha256sum \ - | cut -d' ' -f1 -)" -printf 'Filtered public tree passed: commit=%s digest=sha256:%s\n' \ - "$source_commit" \ - "$public_digest" diff --git a/scripts/vscode-extension-smoke.js b/scripts/vscode-extension-smoke.js deleted file mode 100755 index 3a98e8d..0000000 --- a/scripts/vscode-extension-smoke.js +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env node - -const fs = require("fs"); -const os = require("os"); -const path = require("path"); -const assert = require("assert"); - -const extensionRoot = path.resolve( - process.env.CLUSTERFLUX_VSCODE_EXTENSION_ROOT || - path.join(__dirname, "../vscode-extension") -); -const extension = require(path.join(extensionRoot, "extension.js")); -const packageJson = require(path.join(extensionRoot, "package.json")); -const extensionSource = fs.readFileSync( - path.join(extensionRoot, "extension.js"), - "utf8" -); -const repo = path.resolve(__dirname, ".."); - -assert.strictEqual(packageJson.main, "./extension.js"); -assert(fs.existsSync(path.join(extensionRoot, packageJson.main))); -assert.deepStrictEqual(packageJson.dependencies || {}, {}); -const root = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-vscode-")); -fs.mkdirSync(path.join(root, "envs/linux"), { recursive: true }); -fs.writeFileSync(path.join(root, "envs/linux/Containerfile"), "FROM alpine\n"); -fs.mkdirSync(path.join(root, ".clusterflux"), { recursive: true }); -fs.writeFileSync( - extension.clusterfluxViewStatePath(root), - JSON.stringify({ - nodes: [{ id: "node-linux", status: "online", capabilities: "Command RootlessPodman" }], - processes: [{ id: "vp-build", status: "running", entry: "build" }], - logs: [{ task: "compile-linux", message: "stdout=12 stderr=0", bytes: 12 }], - artifacts: [{ path: "/vfs/artifacts/app.tar.zst", status: "retained", size: 12 }], - inspector: [{ label: "debug", value: "attached" }] - }) -); - -const envs = extension.discoverEnvironmentNames(root); -assert.deepStrictEqual(envs, ["linux"]); - -const diagnostics = extension.diagnoseEnvReferences( - 'let _ = env!("linux"); let _ = env!("windows");', - envs -); -assert.strictEqual(diagnostics.length, 1); -assert.strictEqual(diagnostics[0].name, "windows"); -assert.match(diagnostics[0].message, /envs\/windows\/Containerfile/); - -const inspectCommand = extension.bundleInspectCommand(root, "/repo"); -assert.strictEqual(inspectCommand.command, "cargo"); -assert.deepStrictEqual(inspectCommand.args.slice(0, 8), [ - "run", - "-q", - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - "bundle" -]); -assert(inspectCommand.args.includes("inspect")); -assert(inspectCommand.args.includes("--project")); -assert(inspectCommand.args.includes(root)); -assert(inspectCommand.args.includes("--json")); - -const refreshed = extension.refreshBundleBeforeLaunch(root, "/repo", (command, args, options) => { - assert.strictEqual(command, "cargo"); - assert(args.includes("bundle")); - assert.strictEqual(options.cwd, "/repo"); - return { - status: 0, - stdout: JSON.stringify({ metadata: { identity: "sha256:abc" } }), - stderr: "" - }; -}); -assert.strictEqual(refreshed.metadata.identity, "sha256:abc"); - -const launch = extension.resolveClusterfluxDebugConfiguration( - { uri: { fsPath: root } }, - {} -); -assert.deepStrictEqual(launch, { - name: "Clusterflux: Launch Virtual Process", - type: "clusterflux", - request: "launch", - entry: "build", - project: root, - runtimeBackend: "local-services" -}); -assert.strictEqual( - extension.clusterfluxProcessId("/workspace/app", "build"), - "vp-e4bd6ef50539" -); -assert.strictEqual( - extension.existingProcessRelationship( - { process: "vp-e4bd6ef50539", state: "running" }, - "vp-e4bd6ef50539" - ), - "same_launch_target" -); -assert.strictEqual( - extension.existingProcessRelationship( - { process: "vp-other", state: "running" }, - "vp-e4bd6ef50539" - ), - "different_launch_target" -); - -const liveProcesses = extension.loadLiveProcesses(root, "/repo", (_command, args) => { - assert.deepStrictEqual(args.slice(-3), ["process", "list", "--json"]); - return { - status: 0, - stdout: JSON.stringify({ - coordinator: "https://clusterflux.michelpaulissen.com", - tenant: "tenant-live", - project: "project-live", - user: "user-live", - processes: [{ process: "vp-live", state: "cancelling" }] - }), - stderr: "" - }; -}); -assert.deepStrictEqual(liveProcesses.processes, [ - { process: "vp-live", state: "cancelling" } -]); -assert.strictEqual(liveProcesses.project, "project-live"); - -const adapter = extension.debugAdapterExecutableSpec(root, repo); -assert.strictEqual(adapter.command, "cargo"); -assert.deepStrictEqual(adapter.args, [ - "run", - "-q", - "-p", - "clusterflux-dap", - "--bin", - "clusterflux-debug-dap" -]); -assert.deepStrictEqual(adapter.options, { cwd: repo }); - -const releasedAdapterPath = path.join(root, process.platform === "win32" ? "clusterflux-debug-dap.exe" : "clusterflux-debug-dap"); -fs.writeFileSync(releasedAdapterPath, ""); -const releasedAdapter = extension.debugAdapterExecutableSpec(root, repo); -assert.strictEqual(releasedAdapter.command, releasedAdapterPath); -assert.deepStrictEqual(releasedAdapter.args, []); -assert.deepStrictEqual(releasedAdapter.options, { cwd: root }); - -assert.throws( - () => - extension.refreshBundleBeforeLaunch(root, "/repo", () => ({ - status: 1, - stdout: "", - stderr: "missing environment linux" - })), - /missing environment linux/ -); - -assert( - packageJson.contributes.viewsContainers.activitybar.some( - (container) => container.id === "clusterflux" && container.title === "Clusterflux" - ), - "package.json must contribute a Clusterflux activity-bar container" -); -assert( - fs.existsSync(path.join(extensionRoot, "resources/clusterflux.svg")), - "Clusterflux activity-bar icon must exist" -); -const packageViewIds = packageJson.contributes.views.clusterflux.map((view) => view.id).sort(); -const descriptorViewIds = extension.clusterfluxViewDescriptors().map((view) => view.id).sort(); -assert.deepStrictEqual(descriptorViewIds, [ - "clusterflux.artifacts", - "clusterflux.inspector", - "clusterflux.logs", - "clusterflux.nodes", - "clusterflux.processes" -]); -assert.deepStrictEqual(packageViewIds, descriptorViewIds); -for (const viewId of descriptorViewIds) { - const items = extension.clusterfluxViewItems( - extension.loadClusterfluxViewState(root), - viewId - ); - assert( - items.length > 0 && !items[0].label.startsWith("No "), - `${viewId} should render state-backed items` - ); -} -assert.deepStrictEqual( - extension.clusterfluxViewItems(extension.loadClusterfluxViewState(root), "clusterflux.nodes")[0], - { - label: "node-linux", - description: "online Command RootlessPodman" - } -); - -assert( - packageJson.contributes.debuggers.some((debuggerContribution) => debuggerContribution.type === "clusterflux"), - "package.json must contribute the clusterflux debugger type" -); -for (const viewId of descriptorViewIds) { - assert( - packageJson.activationEvents.includes(`onView:${viewId}`), - `${viewId} should activate the extension when opened` - ); -} -const launchProperties = - packageJson.contributes.debuggers[0].configurationAttributes.launch.properties; -assert.strictEqual(launchProperties.runtimeBackend.default, "local-services"); -assert(launchProperties.runtimeBackend.enum.includes("live-services")); -assert.strictEqual(launchProperties.coordinatorEndpoint.default, undefined); -assert.strictEqual(launchProperties.tenant, undefined); -assert.strictEqual(launchProperties.projectId, undefined); -assert.strictEqual(launchProperties.actorUser, undefined); -assert( - packageJson.contributes.debuggers[0].configurationAttributes.attach, - "package.json must contribute an attach configuration" -); -for (const command of [ - "clusterflux.refreshProcesses", - "clusterflux.process.attach", - "clusterflux.process.cancel", - "clusterflux.process.abort" -]) { - assert( - packageJson.contributes.commands.some((entry) => entry.command === command), - `${command} must be contributed` - ); -} -assert.match(extensionSource, /registerDebugAdapterDescriptorFactory\("clusterflux"/); -assert.match(extensionSource, /clusterflux-debug-dap/); -assert.match(extensionSource, /\.clusterflux\/views\.json/); - -fs.rmSync(root, { recursive: true, force: true }); -console.log("VS Code extension smoke passed"); diff --git a/scripts/vscode-f5-smoke.js b/scripts/vscode-f5-smoke.js deleted file mode 100755 index d7b7893..0000000 --- a/scripts/vscode-f5-smoke.js +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const path = require("path"); - -const extension = require("../vscode-extension/extension"); - -class DapClient { - constructor(spec) { - this.child = cp.spawn(spec.command, spec.args, { - cwd: spec.options && spec.options.cwd, - env: spec.options && spec.options.env, - detached: process.platform !== "win32" - }); - this.seq = 1; - this.buffer = Buffer.alloc(0); - this.messages = []; - this.waiters = []; - this.stderr = ""; - - this.child.stdout.on("data", (chunk) => { - this.buffer = Buffer.concat([this.buffer, chunk]); - this.parse(); - }); - this.child.stderr.on("data", (chunk) => { - this.stderr += chunk.toString(); - }); - this.child.on("exit", () => this.flushWaiters()); - } - - send(command, args = {}) { - const seq = this.seq++; - const message = { seq, type: "request", command, arguments: args }; - const payload = Buffer.from(JSON.stringify(message)); - this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); - this.child.stdin.write(payload); - return seq; - } - - async response(seq, command) { - const message = await this.waitFor( - (item) => - item.type === "response" && - item.request_seq === seq && - item.command === command - ); - if (!message.success) { - throw new Error( - `DAP ${command} failed: ${message.message || JSON.stringify(message)}\n${this.stderr}` - ); - } - return message; - } - - terminate() { - if (this.child.exitCode !== null) return; - if (process.platform === "win32") { - this.child.kill("SIGKILL"); - return; - } - try { - process.kill(-this.child.pid, "SIGKILL"); - } catch (_) { - this.child.kill("SIGKILL"); - } - } - - waitFor(predicate, timeoutMs = 240000) { - const existing = this.messages.find(predicate); - if (existing) return Promise.resolve(existing); - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - this.terminate(); - reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); - }, timeoutMs); - this.waiters.push({ predicate, resolve, timer }); - }); - } - - parse() { - while (true) { - const headerEnd = this.buffer.indexOf("\r\n\r\n"); - if (headerEnd < 0) return; - const header = this.buffer.slice(0, headerEnd).toString(); - const match = header.match(/Content-Length: (\d+)/i); - if (!match) throw new Error(`bad DAP header: ${header}`); - const length = Number(match[1]); - const start = headerEnd + 4; - const end = start + length; - if (this.buffer.length < end) return; - const payload = this.buffer.slice(start, end).toString(); - this.buffer = this.buffer.slice(end); - this.messages.push(JSON.parse(payload)); - this.flushWaiters(); - } - } - - flushWaiters() { - for (const waiter of [...this.waiters]) { - const message = this.messages.find(waiter.predicate); - if (!message) continue; - clearTimeout(waiter.timer); - this.waiters.splice(this.waiters.indexOf(waiter), 1); - waiter.resolve(message); - } - } - - async close() { - if (this.child.exitCode !== null) return; - const seq = this.send("disconnect"); - await this.response(seq, "disconnect"); - this.child.stdin.end(); - } -} - -(async () => { - const repo = path.resolve(__dirname, ".."); - const project = path.join(repo, "examples/hello-build"); - const buildSourceLines = fs - .readFileSync(path.join(project, "src/lib.rs"), "utf8") - .split(/\r?\n/); - const sourceLine = (needle) => { - const index = buildSourceLines.findIndex((line) => line.includes(needle)); - assert(index >= 0, `flagship source must contain ${needle}`); - return index + 1; - }; - const buildMainLine = sourceLine("async fn build()"); - const launchConfig = extension.resolveClusterfluxDebugConfiguration( - { uri: { fsPath: project } }, - {} - ); - - assert.strictEqual(launchConfig.type, "clusterflux"); - assert.strictEqual(launchConfig.request, "launch"); - assert.strictEqual(launchConfig.entry, "build"); - assert.strictEqual(launchConfig.project, project); - assert.strictEqual(launchConfig.runtimeBackend, "local-services"); - - const inspection = extension.refreshBundleBeforeLaunch(project, repo); - assert.match(inspection.metadata.identity, /^sha256:/); - - // Keep the timed attach focused on the runtime boundary. A completely fresh - // public checkout may otherwise spend most of that window compiling the - // coordinator or node after the adapter has already started waiting. - cp.execFileSync( - "cargo", - [ - "build", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-node", - ], - { cwd: repo, stdio: "inherit" } - ); - const executableSuffix = process.platform === "win32" ? ".exe" : ""; - const adapterSpec = extension.debugAdapterExecutableSpec(repo); - adapterSpec.options = { - ...(adapterSpec.options || {}), - env: { - ...process.env, - CLUSTERFLUX_COORDINATOR_BIN: path.join( - repo, - "target", - "debug", - `clusterflux-coordinator${executableSuffix}` - ), - CLUSTERFLUX_NODE_BIN: path.join( - repo, - "target", - "debug", - `clusterflux-node${executableSuffix}` - ), - }, - }; - - const client = new DapClient(adapterSpec); - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true - }); - await client.response(initialize, "initialize"); - - const launch = client.send("launch", launchConfig); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const breakpoints = client.send("setBreakpoints", { - source: { path: path.join(project, "src/lib.rs") }, - breakpoints: [{ line: buildMainLine }] - }); - const breakpointResponse = await client.response(breakpoints, "setBreakpoints"); - assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, false); - assert.match( - breakpointResponse.body.breakpoints[0].message, - /Pending coordinator breakpoint installation/ - ); - - const configurationDone = client.send("configurationDone"); - await client.response(configurationDone, "configurationDone"); - const installedBreakpoint = await client.waitFor( - (message) => - message.type === "event" && - message.event === "breakpoint" && - message.body?.breakpoint?.verified === true - ); - assert.strictEqual(installedBreakpoint.body.breakpoint.line, buildMainLine); - const stopped = await client.waitFor( - (message) => message.type === "event" && message.event === "stopped" - ); - assert.strictEqual(stopped.body.allThreadsStopped, true); - assert.strictEqual(stopped.body.reason, "breakpoint"); - - const threadsRequest = client.send("threads"); - const threads = (await client.response(threadsRequest, "threads")).body.threads; - const mainThread = threads.find((thread) => thread.name.includes("build coordinator main")); - assert(mainThread, "F5 launch must expose the real coordinator-main entrypoint thread"); - - const stackRequest = client.send("stackTrace", { - threadId: mainThread.id, - startFrame: 0, - levels: 1 - }); - const stack = (await client.response(stackRequest, "stackTrace")).body.stackFrames; - assert.strictEqual(stack[0].line, buildMainLine); - assert.strictEqual(stack[0].source.path, path.join(project, "src/lib.rs")); - assert.strictEqual(stack[0].source.sourceReference || 0, 0); - - const sourceRequest = client.send("source", { source: stack[0].source }); - const source = (await client.response(sourceRequest, "source")).body; - assert.match(source.content, /compile/); - - const scopesRequest = client.send("scopes", { frameId: stack[0].id }); - const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; - const localsScope = scopes.find((scope) => scope.name === "Source Locals"); - const argsScope = scopes.find((scope) => scope.name === "Task Args and Handles"); - const runtimeScope = scopes.find((scope) => scope.name === "Clusterflux Runtime"); - const outputScope = scopes.find((scope) => scope.name === "Recent Output"); - assert(localsScope, "F5 launch must expose source locals scope"); - assert(argsScope, "F5 launch must expose task args and handles"); - assert(runtimeScope, "F5 launch must expose Clusterflux runtime state"); - assert(outputScope, "F5 launch must expose recent output state"); - - const localsRequest = client.send("variables", { - variablesReference: localsScope.variablesReference - }); - const locals = (await client.response(localsRequest, "variables")).body.variables; - assert( - locals.some( - (variable) => - variable.name === "unavailable-local-diagnostic" && - String(variable.value).includes("cannot be inspected") - ), - "source locals scope must report unavailable real Rust locals explicitly" - ); - - const runtimeRequest = client.send("variables", { - variablesReference: runtimeScope.variablesReference - }); - const runtime = (await client.response(runtimeRequest, "variables")).body.variables; - assert( - runtime.some( - (variable) => variable.name === "runtime_backend" && variable.value === "LocalServices" - ), - "extension-resolved F5 launch must use the real local-services backend" - ); - assert( - runtime.some( - (variable) => variable.name === "coordinator_task_events" && variable.value === 0 - ), - "a task frozen at its entry probe must not fabricate a terminal task event" - ); - assert( - runtime.some( - (variable) => - variable.name === "command_status" && - String(variable.value).includes( - "frozen through local services at executing Wasm probe" - ) - ) - ); - assert( - runtime.some( - (variable) => variable.name === "state" && variable.value === "Frozen" - ), - "F5 must expose the node-acknowledged frozen participant state" - ); - assert(runtime.some((variable) => variable.name === "command_spec")); - assert(runtime.some((variable) => variable.name === "stdout_tail")); - assert(runtime.some((variable) => variable.name === "stderr_tail")); - - const outputRequest = client.send("variables", { - variablesReference: outputScope.variablesReference - }); - const output = (await client.response(outputRequest, "variables")).body.variables; - assert(output.some((variable) => variable.name === "stdout_tail")); - assert(output.some((variable) => variable.name === "stderr_tail")); - - await client.close(); - } catch (error) { - client.terminate(); - throw error; - } - - console.log("VS Code F5 smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/wasmtime-assignment-smoke.js b/scripts/wasmtime-assignment-smoke.js deleted file mode 100755 index 9ec2683..0000000 --- a/scripts/wasmtime-assignment-smoke.js +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const sdkRuntime = fs.readFileSync( - path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), - "utf8" -); - -assert.doesNotMatch( - sdkRuntime, - /"type": "launch_task"/, - "native SDK code must not submit external TaskV1 work" -); -assert.match( - sdkRuntime, - /native SDK task spawning requires coordinator EntrypointV1 execution/ -); - -cp.execFileSync("node", ["scripts/cli-local-run-smoke.js"], { - cwd: repo, - stdio: "inherit", -}); -cp.execFileSync( - "cargo", - ["test", "-p", "clusterflux-node", "wasmtime_runtime_runs_named_task_export"], - { cwd: repo, stdio: "inherit" } -); -cp.execFileSync( - "cargo", - [ - "test", - "-p", - "clusterflux-coordinator", - "signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only", - ], - { cwd: repo, stdio: "inherit" } -); - -console.log("Wasmtime assignment smoke passed"); diff --git a/scripts/wasmtime-node-smoke.js b/scripts/wasmtime-node-smoke.js deleted file mode 100644 index 7cbb1ac..0000000 --- a/scripts/wasmtime-node-smoke.js +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const fs = require("fs"); -const path = require("path"); - -const repo = path.resolve(__dirname, ".."); -const wasmTarget = path.join( - repo, - "target", - "wasm32-unknown-unknown", - "release", - "runtime_conformance.wasm" -); - -cp.execFileSync( - "cargo", - [ - "build", - "--release", - "-p", - "runtime-conformance", - "--target", - "wasm32-unknown-unknown", - ], - { - cwd: repo, - stdio: "inherit", - env: { - ...process.env, - CARGO_PROFILE_RELEASE_OPT_LEVEL: "z", - CARGO_PROFILE_RELEASE_LTO: "thin", - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1", - }, - } -); - -assert(fs.existsSync(wasmTarget), `missing compiled Wasm module at ${wasmTarget}`); - -const output = cp.execFileSync( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-wasmtime-smoke", - "--", - wasmTarget, - "task_add_one", - "41", - "42", - ], - { cwd: repo, encoding: "utf8" } -); - -const report = JSON.parse(output); -assert.strictEqual(report.type, "wasmtime_task_smoke"); -assert.strictEqual(report.export, "task_add_one"); -assert.strictEqual(report.arg, 41); -assert.strictEqual(report.result, 42); - -const debugOutput = cp.execFileSync( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-wasmtime-smoke", - "--", - "--debug-freeze-resume", - wasmTarget, - "task_add_one", - "41", - "42", - ], - { cwd: repo, encoding: "utf8" } -); -const debugReport = JSON.parse(debugOutput); -assert.strictEqual(debugReport.type, "wasmtime_debug_freeze_resume_smoke"); -assert.strictEqual(debugReport.export, "task_add_one"); -assert.strictEqual(debugReport.task, "task_add_one"); -assert.strictEqual(debugReport.frozen_state, "Frozen"); -assert.strictEqual(debugReport.resumed_state, "Running"); -assert(debugReport.stack_frames.some((frame) => String(frame).includes("task_add_one"))); -assert( - debugReport.local_values.some( - ([name, value]) => name === "wasm_local_0" && String(value).includes("41") - ), - "Wasmtime debug snapshot must expose the real i32 argument as a frame local" -); -assert.strictEqual(debugReport.node_runtime_captured_wasm_locals, true); -assert.strictEqual(debugReport.result, 42); -assert.strictEqual(debugReport.node_runtime_reached_wasm_task, true); - -const hostCommandOutput = cp.execFileSync( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-wasmtime-smoke", - "--", - "--host-command", - wasmTarget, - "compile_linux", - ], - { cwd: repo, encoding: "utf8" } -); -const hostCommandReport = JSON.parse(hostCommandOutput); -assert.strictEqual(hostCommandReport.type, "wasmtime_host_command_smoke"); -assert.strictEqual(hostCommandReport.task, "compile_linux"); -assert.match(hostCommandReport.export, /^clusterflux_task_v1_[0-9a-f]{64}$/); -assert.strictEqual(hostCommandReport.program, "cc"); -assert.deepStrictEqual(hostCommandReport.args, [ - "-Os", - "-static", - "-s", - "fixture/hello-clusterflux.c", - "-o", - "/clusterflux/output/hello-clusterflux", -]); -assert.strictEqual(hostCommandReport.working_directory, "/workspace"); -assert.deepStrictEqual(hostCommandReport.environment_variables, { - SOURCE_DATE_EPOCH: "0", -}); -assert.strictEqual(hostCommandReport.timeout_ms, 180000); -assert.strictEqual(hostCommandReport.network, "disabled"); -assert.strictEqual(hostCommandReport.status_code, 0); -assert.strictEqual(hostCommandReport.stdout, ""); -assert.strictEqual(hostCommandReport.artifact_name, "hello-clusterflux"); -assert.strictEqual(hostCommandReport.artifact_size_bytes, "hello-clusterflux".length); -assert.match(hostCommandReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); -assert.strictEqual(hostCommandReport.node_host_import, "clusterflux.command_run_v1"); -assert.strictEqual(hostCommandReport.artifact_host_import, "clusterflux.vfs_operation_v1"); -assert.strictEqual(hostCommandReport.flagship_linux_build_task, true); -assert.strictEqual(hostCommandReport.node_executed_host_command, true); -assert.strictEqual(hostCommandReport.hosted_control_plane_ran_command, false); - -const artifactOutput = cp.execFileSync( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-wasmtime-smoke", - "--", - "--task-artifact", - wasmTarget, - "package_release", - ], - { cwd: repo, encoding: "utf8" } -); -const artifactReport = JSON.parse(artifactOutput); -assert.strictEqual(artifactReport.type, "wasmtime_task_artifact_smoke"); -assert.strictEqual(artifactReport.task, "package_release"); -assert.strictEqual(artifactReport.artifact_name, "release.tar"); -assert.strictEqual(artifactReport.artifact_size_bytes, "release.tar".length); -assert.match(artifactReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); -assert.strictEqual(artifactReport.host_import, "clusterflux.vfs_operation_v1"); -assert.strictEqual(artifactReport.host_issued_handle_returned, true); -assert.ok(artifactReport.artifact.id.endsWith(artifactReport.artifact_digest.slice("sha256:".length))); - -console.log("Wasmtime node smoke passed"); diff --git a/scripts/windows-best-effort-smoke.js b/scripts/windows-best-effort-smoke.js deleted file mode 100755 index 839582a..0000000 --- a/scripts/windows-best-effort-smoke.js +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); - -const repo = path.resolve(__dirname, ".."); -const digest = (value) => - `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; -const environmentDigest = digest("windows-command-dev-environment"); -const sourceDigest = digest("windows-best-effort-source"); -const windowsArtifactBytes = Buffer.from("windows-output-data"); -const windowsArtifactDigest = digest(windowsArtifactBytes); -const emptyWasm = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); -const emptyWasmDigest = digest(emptyWasm); -const nodeDaemon = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/daemon.rs"), "utf8"); -const taskReports = fs.readFileSync( - path.join(repo, "crates/clusterflux-node/src/task_reports.rs"), - "utf8" -); -const nodeLib = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/lib.rs"), "utf8"); -const windowsDev = fs.readFileSync( - path.join(repo, "crates/clusterflux-node/src/windows_dev.rs"), - "utf8" -); -const executionCore = fs.readFileSync( - path.join(repo, "crates/clusterflux-core/src/execution.rs"), - "utf8" -); -const dapSource = [ - fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/demo_backend.rs"), "utf8"), - fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/tests.rs"), "utf8"), -].join("\n"); -const windowsNode = "windows-node"; -const windowsIdentity = nodeIdentity("windows-best-effort-smoke", windowsNode); - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function windowsCapabilities() { - return { - os: "Windows", - arch: "x86_64", - capabilities: ["Command", "VfsArtifacts", "WindowsCommandDev"], - environment_backends: ["WindowsCommandDev"], - source_providers: ["filesystem"] - }; -} - -function assertWindowsBackendBoundary() { - assert.match(nodeDaemon, /CoordinatorSession::connect\(&args\.coordinator\)/); - assert.match(nodeDaemon, /record_completed_task\(/); - assert.match(taskReports, /"type": "task_completed"/); - assert.doesNotMatch(nodeDaemon, /WindowsCommandDev|windows-command-dev|cfg\(windows\)/); - - assert.match(nodeLib, /impl CommandBackend for LinuxRootlessPodmanBackend/); - assert.match(nodeLib, /mod windows_dev/); - assert.match(nodeLib, /pub use windows_dev::\{WindowsCommandDevBackend, WindowsSandboxStubBackend\}/); - assert.match(windowsDev, /impl CommandBackend for WindowsCommandDevBackend/); - assert.match(windowsDev, /impl CommandBackend for WindowsSandboxStubBackend/); - assert.doesNotMatch(windowsDev, /LinuxRootlessPodmanBackend/); - assert.match(executionCore, /\bLinuxRootlessPodman\b/); - assert.match(executionCore, /\bWindowsCommandDev\b/); - assert.match(executionCore, /\bStubbedWindowsSandbox\b/); - - assert.match(dapSource, /thread\(WINDOWS_THREAD, "compile-windows", "compile windows"/); - assert.match(dapSource, /fn launch_threads_include_windows_task_in_same_virtual_process\(\)/); - assert.match(dapSource, /fn windows_thread_runtime_variables_share_virtual_process\(\)/); -} - -(async () => { - assertWindowsBackendBoundary(); - - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - let coordinatorStderr = ""; - coordinator.stderr.on("data", (chunk) => { - coordinatorStderr += chunk.toString(); - }); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const attached = await send(addr, { - type: "attach_node", - tenant: "tenant", - project: "project", - node: windowsNode, - public_key: windowsIdentity.publicKey - }); - assert.strictEqual(attached.type, "node_attached"); - assert.strictEqual(attached.node, windowsNode); - - const recorded = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: windowsNode, - capabilities: windowsCapabilities(), - cached_environment_digests: [environmentDigest], - dependency_cache_digests: [], - source_snapshots: [sourceDigest], - artifact_locations: [], - direct_connectivity: true, - online: true - })); - assert.strictEqual(recorded.type, "node_capabilities_recorded"); - assert.strictEqual(recorded.node, windowsNode); - - const placement = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: { - os: "Windows", - arch: null, - capabilities: ["WindowsCommandDev"] - }, - environment_digest: environmentDigest, - required_capabilities: ["Command"], - dependency_cache: null, - source_snapshot: sourceDigest, - required_artifacts: [], - prefer_node: null - }); - assert.strictEqual(placement.type, "task_placement"); - assert.strictEqual(placement.placement.node, windowsNode); - assert.ok(placement.placement.reasons.includes("warm environment cache")); - assert.ok(placement.placement.reasons.includes("source snapshot already local")); - - const started = await send(addr, { - type: "start_process", - tenant: "tenant", - project: "project", - process: "vp-windows" - }); - assert.strictEqual(started.type, "process_started"); - assert.strictEqual(started.process, "vp-windows"); - - const reconnected = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "reconnect_node", { - type: "reconnect_node", - tenant: "tenant", - project: "project", - node: windowsNode, - process: "vp-windows", - epoch: started.epoch - })); - assert.strictEqual(reconnected.type, "node_reconnected"); - - const launched = await send(addr, { - type: "launch_task", - tenant: "tenant", - project: "project", - actor_user: "operator", - task_spec: { - tenant: "tenant", - project: "project", - process: "vp-windows", - task_definition: "windows-command-dev", - task_instance: "windows-command-dev", - dispatch: { - kind: "coordinator_node_wasm", - export: "windows_command_dev", - abi: "task_v1", - }, - environment_id: "windows", - environment: { - os: "Windows", - arch: null, - capabilities: ["WindowsCommandDev"], - }, - environment_digest: environmentDigest, - required_capabilities: ["Command", "WindowsCommandDev"], - dependency_cache: null, - source_snapshot: sourceDigest, - required_artifacts: [], - args: [{ SourceSnapshot: sourceDigest }], - vfs_epoch: started.epoch, - bundle_digest: emptyWasmDigest, - }, - wait_for_node: false, - artifact_path: "/vfs/artifacts/windows-output.txt", - wasm_module_base64: emptyWasm.toString("base64"), - }); - assert.strictEqual(launched.type, "error", JSON.stringify(launched)); - assert.match(launched.message, /external callers may launch only EntrypointV1/); - - const recordedTask = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "task_completed", { - type: "task_completed", - tenant: "tenant", - project: "project", - process: "vp-windows", - node: windowsNode, - task: "windows-command-dev", - status_code: 0, - stdout_bytes: 18, - stderr_bytes: 0, - stdout_tail: "", - stderr_tail: "", - stdout_truncated: false, - stderr_truncated: false, - artifact_path: "/vfs/artifacts/windows-output.txt", - artifact_digest: windowsArtifactDigest, - artifact_size_bytes: windowsArtifactBytes.length - })); - assert.strictEqual(recordedTask.type, "error"); - assert.match(recordedTask.message, /coordinator-issued task instance|issued active task|active task/); - - const events = await send(addr, { - type: "list_task_events", - tenant: "tenant", - project: "project", - actor_user: "user", - process: "vp-windows" - }); - assert.strictEqual(events.type, "task_events"); - assert.strictEqual(events.events.length, 0); - - const link = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact: "windows-output.txt", - max_bytes: 1024 - }); - assert.strictEqual(link.type, "error"); - assert.match(link.message, /does not exist|not found|unavailable/); - } catch (error) { - if (coordinatorStderr) { - error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; - } - throw error; - } finally { - coordinator.kill("SIGTERM"); - } - - console.log("Windows best-effort smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/scripts/windows-runner-smoke.js b/scripts/windows-runner-smoke.js deleted file mode 100755 index b7b3628..0000000 --- a/scripts/windows-runner-smoke.js +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env node - -const assert = require("assert"); -const cp = require("child_process"); -const crypto = require("crypto"); -const fs = require("fs"); -const net = require("net"); -const path = require("path"); -const { coordinatorWireRequest } = require("./coordinator-wire"); -const { nodeIdentity, signedNodeRequest } = require("./node-signing"); - -const repo = path.resolve(__dirname, ".."); -const forgejoWindowsNode = "forgejo-windows-node"; -const forgejoWindowsIdentity = nodeIdentity("windows-runner-smoke", forgejoWindowsNode); -const windowsEnvironmentDigest = `sha256:${crypto - .createHash("sha256") - .update("clusterflux/windows-command-dev/v1") - .digest("hex")}`; -const sourceSnapshot = `sha256:${crypto - .createHash("sha256") - .update("clusterflux/windows-runner/source/v1") - .digest("hex")}`; - -class DapClient { - constructor() { - this.child = cp.spawn( - "cargo", - ["run", "-q", "-p", "clusterflux-dap", "--bin", "clusterflux-debug-dap"], - { cwd: repo } - ); - this.seq = 1; - this.buffer = Buffer.alloc(0); - this.messages = []; - this.waiters = []; - this.stderr = ""; - - this.child.stdout.on("data", (chunk) => { - this.buffer = Buffer.concat([this.buffer, chunk]); - this.parse(); - }); - this.child.stderr.on("data", (chunk) => { - this.stderr += chunk.toString(); - }); - this.child.on("exit", () => this.flushWaiters()); - } - - send(command, args = {}) { - const seq = this.seq++; - const message = { seq, type: "request", command, arguments: args }; - const payload = Buffer.from(JSON.stringify(coordinatorWireRequest(message))); - this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); - this.child.stdin.write(payload); - return seq; - } - - async response(seq, command) { - const message = await this.waitFor( - (item) => - item.type === "response" && - item.request_seq === seq && - item.command === command - ); - if (!message.success) { - throw new Error(`DAP ${command} failed: ${message.message || JSON.stringify(coordinatorWireRequest(message))}`); - } - return message; - } - - waitFor(predicate, timeoutMs = 120000) { - const existing = this.messages.find(predicate); - if (existing) return Promise.resolve(existing); - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - this.child.kill("SIGKILL"); - reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); - }, timeoutMs); - this.waiters.push({ predicate, resolve, timer }); - }); - } - - parse() { - while (true) { - const headerEnd = this.buffer.indexOf("\r\n\r\n"); - if (headerEnd < 0) return; - const header = this.buffer.slice(0, headerEnd).toString(); - const match = header.match(/Content-Length: (\d+)/i); - if (!match) throw new Error(`bad DAP header: ${header}`); - const length = Number(match[1]); - const start = headerEnd + 4; - const end = start + length; - if (this.buffer.length < end) return; - const payload = this.buffer.slice(start, end).toString(); - this.buffer = this.buffer.slice(end); - this.messages.push(JSON.parse(payload)); - this.flushWaiters(); - } - } - - flushWaiters() { - for (const waiter of [...this.waiters]) { - const message = this.messages.find(waiter.predicate); - if (!message) continue; - clearTimeout(waiter.timer); - this.waiters.splice(this.waiters.indexOf(waiter), 1); - waiter.resolve(message); - } - } - - async close() { - if (this.child.exitCode !== null) return; - const seq = this.send("disconnect"); - await this.response(seq, "disconnect"); - this.child.stdin.end(); - } -} - -function waitForJsonLine(child) { - return new Promise((resolve, reject) => { - let buffer = ""; - child.stdout.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - try { - resolve(JSON.parse(buffer.slice(0, newline).trim())); - } catch (error) { - reject(error); - } - }); - child.once("exit", (code) => { - reject(new Error(`process exited before JSON line with code ${code}`)); - }); - }); -} - -function send(addr, message) { - return new Promise((resolve, reject) => { - const socket = net.connect(addr.port, addr.host, () => { - socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); - }); - let buffer = ""; - socket.on("data", (chunk) => { - buffer += chunk.toString(); - const newline = buffer.indexOf("\n"); - if (newline < 0) return; - socket.end(); - try { - resolve(JSON.parse(buffer.slice(0, newline))); - } catch (error) { - reject(error); - } - }); - socket.on("error", reject); - }); -} - -function runWindowsAttach(addr, grant) { - return new Promise((resolve, reject) => { - const child = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-cli", - "--bin", - "clusterflux", - "--", - "node", - "attach", - "--coordinator", - `${addr.host}:${addr.port}`, - "--tenant", - "tenant", - "--project-id", - "project", - "--node", - forgejoWindowsNode, - "--public-key", - forgejoWindowsIdentity.publicKey, - "--enrollment-grant", - grant, - "--cap", - "windows-command-dev", - "--json" - ], - { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, - }, - } - ); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - child.on("exit", (code) => { - if (code !== 0) { - reject(new Error(`Windows node attach failed with code ${code}\n${stderr}`)); - return; - } - try { - resolve(JSON.parse(stdout)); - } catch (error) { - reject( - new Error(`Windows node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) - ); - } - }); - }); -} - -function runWindowsNode(addr, grant) { - return new Promise((resolve, reject) => { - const child = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-node", - "--bin", - "clusterflux-node", - "--", - "--coordinator", - `${addr.host}:${addr.port}`, - "--tenant", - "tenant", - "--project-id", - "project", - "--node", - forgejoWindowsNode, - "--public-key", - forgejoWindowsIdentity.publicKey, - "--enrollment-grant", - grant, - "--process", - "vp-forgejo-windows", - "--task", - "windows-command-dev", - "--command", - "cmd", - "--arg", - "/C", - "--arg", - "echo clusterflux-windows-runner", - "--artifact", - "/vfs/artifacts/windows-runner-output.txt" - ], - { - cwd: repo, - env: { - ...process.env, - CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, - }, - } - ); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk.toString(); - }); - child.stderr.on("data", (chunk) => { - stderr += chunk.toString(); - }); - child.on("exit", (code) => { - if (code !== 0) { - reject(new Error(`Windows node smoke failed with code ${code}\n${stderr}`)); - return; - } - try { - resolve(JSON.parse(stdout.trim().split(/\r?\n/).at(-1))); - } catch (error) { - reject(new Error(`node output was not JSON: ${stdout}\n${error.stack || error.message}`)); - } - }); - }); -} - -async function assertDebuggerShowsWindowsThread() { - const client = new DapClient(); - try { - const initialize = client.send("initialize", { - adapterID: "clusterflux", - linesStartAt1: true, - columnsStartAt1: true - }); - await client.response(initialize, "initialize"); - - const launch = client.send("launch", { - entry: "build", - project: path.join(repo, "tests/fixtures/runtime-conformance"), - runtimeBackend: "simulated" - }); - await client.response(launch, "launch"); - await client.waitFor( - (message) => message.type === "event" && message.event === "initialized" - ); - - const configurationDone = client.send("configurationDone"); - await client.response(configurationDone, "configurationDone"); - - const threadsRequest = client.send("threads"); - const threads = (await client.response(threadsRequest, "threads")).body.threads; - assert( - threads.some((thread) => thread.name.includes("compile windows")), - "debugger must represent the Windows task as a virtual thread" - ); - - await client.close(); - } catch (error) { - client.child.kill("SIGKILL"); - throw error; - } -} - -(async () => { - if (process.platform !== "win32") { - throw new Error( - `Windows runner validation requires win32; current platform is ${process.platform}` - ); - } - - const coordinator = cp.spawn( - "cargo", - [ - "run", - "-q", - "-p", - "clusterflux-coordinator", - "--bin", - "clusterflux-coordinator", - "--", - "--listen", - "127.0.0.1:0", - "--allow-local-trusted-loopback" - ], - { cwd: repo } - ); - let coordinatorStderr = ""; - coordinator.stderr.on("data", (chunk) => { - coordinatorStderr += chunk.toString(); - }); - - try { - const ready = await waitForJsonLine(coordinator); - const [host, portText] = ready.listen.split(":"); - const addr = { host, port: Number(portText) }; - assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); - - const attachGrant = await send(addr, { - type: "create_node_enrollment_grant", - tenant: "tenant", - project: "project", - actor_user: "operator", - ttl_seconds: 900 - }); - assert.strictEqual(attachGrant.type, "node_enrollment_grant_created"); - assert.strictEqual(attachGrant.scope, "node:attach"); - - const attach = await runWindowsAttach(addr, attachGrant.grant); - assert.strictEqual(attach.plan.node, forgejoWindowsNode); - assert.strictEqual(attach.boundary.cli_contacted_coordinator, true); - assert.strictEqual(attach.boundary.used_enrollment_exchange, true); - assert.strictEqual(attach.coordinator_response.type, "node_enrollment_exchanged"); - assert.strictEqual(attach.coordinator_response.credential.scope, "node:attach"); - assert(attach.plan.capabilities.capabilities.includes("WindowsCommandDev")); - - const runtimeGrant = await send(addr, { - type: "create_node_enrollment_grant", - tenant: "tenant", - project: "project", - actor_user: "operator", - ttl_seconds: 900 - }); - assert.strictEqual(runtimeGrant.type, "node_enrollment_grant_created"); - - const report = await runWindowsNode(addr, runtimeGrant.grant); - assert.strictEqual(report.node_status, "completed"); - assert.strictEqual(report.virtual_thread, "windows-command-dev"); - assert.strictEqual(report.status_code, 0); - assert.strictEqual(report.large_bytes_uploaded, false); - assert.strictEqual( - report.staged_artifact.path, - "/vfs/artifacts/windows-runner-output.txt" - ); - assert.strictEqual(report.registration_response.type, "node_enrollment_exchanged"); - assert.strictEqual(report.registration_response.credential.scope, "node:attach"); - assert.strictEqual(report.coordinator_response.type, "task_recorded"); - - const recorded = await send(addr, signedNodeRequest(forgejoWindowsNode, forgejoWindowsIdentity, "report_node_capabilities", { - type: "report_node_capabilities", - tenant: "tenant", - project: "project", - node: forgejoWindowsNode, - capabilities: { - os: "Windows", - arch: "x86_64", - capabilities: ["Command", "WindowsCommandDev", "VfsArtifacts"], - environment_backends: ["WindowsCommandDev"], - source_providers: ["filesystem"] - }, - cached_environment_digests: [windowsEnvironmentDigest], - source_snapshots: [sourceSnapshot], - artifact_locations: ["windows-runner-output.txt"], - direct_connectivity: true, - online: true - })); - assert.strictEqual(recorded.type, "node_capabilities_recorded"); - - const placement = await send(addr, { - type: "schedule_task", - tenant: "tenant", - project: "project", - environment: { - os: "Windows", - arch: null, - capabilities: ["WindowsCommandDev"] - }, - environment_digest: windowsEnvironmentDigest, - required_capabilities: ["Command"], - source_snapshot: sourceSnapshot, - required_artifacts: ["windows-runner-output.txt"], - prefer_node: null - }); - assert.strictEqual(placement.type, "task_placement"); - assert.strictEqual(placement.placement.node, "forgejo-windows-node"); - - const link = await send(addr, { - type: "create_artifact_download_link", - tenant: "tenant", - project: "project", - actor_user: "user", - artifact: "windows-runner-output.txt", - max_bytes: 1024 - }); - assert.strictEqual(link.type, "artifact_download_link"); - assert.deepStrictEqual(link.link.source, { - RetainedNode: "forgejo-windows-node" - }); - } catch (error) { - if (coordinatorStderr) { - error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; - } - throw error; - } finally { - coordinator.kill("SIGTERM"); - } - - await assertDebuggerShowsWindowsThread(); - - const outDir = path.join(repo, "target", "acceptance"); - fs.mkdirSync(outDir, { recursive: true }); - fs.writeFileSync( - path.join(outDir, "windows-runner.json"), - `${JSON.stringify( - { - kind: "clusterflux_windows_runner_validation", - platform: process.platform, - runner: process.env.FORGEJO_RUNNER_NAME || process.env.RUNNER_NAME || null, - validated: true - }, - null, - 2 - )}\n` - ); - - console.log("Windows runner smoke passed"); -})().catch((error) => { - console.error(error.stack || error.message); - process.exit(1); -}); diff --git a/tests/fixtures/runtime-conformance/Cargo.toml b/tests/fixtures/runtime-conformance/Cargo.toml deleted file mode 100644 index 8558f01..0000000 --- a/tests/fixtures/runtime-conformance/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "runtime-conformance" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -[lib] -crate-type = ["rlib", "cdylib"] - -[dependencies] -clusterflux = { package = "clusterflux-sdk", path = "../../../crates/clusterflux-sdk" } -futures-executor.workspace = true -serde.workspace = true -serde_json.workspace = true diff --git a/tests/fixtures/runtime-conformance/README.md b/tests/fixtures/runtime-conformance/README.md deleted file mode 100644 index 15eed25..0000000 --- a/tests/fixtures/runtime-conformance/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Runtime conformance fixture - -This is test-only coverage for raw task ABI, cancellation, long joins, placement, -debug probes, and failure injection. It is intentionally not a public SDK -example. Start with examples/hello-build. diff --git a/tests/fixtures/runtime-conformance/envs/linux/Containerfile b/tests/fixtures/runtime-conformance/envs/linux/Containerfile deleted file mode 100644 index ba87ba3..0000000 --- a/tests/fixtures/runtime-conformance/envs/linux/Containerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM docker.io/library/alpine:3.20 -RUN apk add --no-cache build-base tar zstd -WORKDIR /workspace diff --git a/tests/fixtures/runtime-conformance/envs/windows/Dockerfile b/tests/fixtures/runtime-conformance/envs/windows/Dockerfile deleted file mode 100644 index 2978ba6..0000000 --- a/tests/fixtures/runtime-conformance/envs/windows/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -# User-attached Windows development execution contract. -# This is not a managed untrusted Windows sandbox. diff --git a/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c b/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c deleted file mode 100644 index 79b67fd..0000000 --- a/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(void) { - puts("hello from a real Clusterflux build"); - return 0; -} diff --git a/tests/fixtures/runtime-conformance/src/lib.rs b/tests/fixtures/runtime-conformance/src/lib.rs deleted file mode 100644 index 91cf21f..0000000 --- a/tests/fixtures/runtime-conformance/src/lib.rs +++ /dev/null @@ -1,501 +0,0 @@ -use clusterflux::{Artifact, EnvRef, SourceSnapshot}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] -pub struct BuildReport { - pub linux_thread: u64, - pub linux_parallel_thread: u64, - pub package_thread: u64, - pub linux_artifact: Artifact, - pub package_artifact: Artifact, - pub source: SourceSnapshot, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] -pub struct PackageInput { - pub release_name: String, - pub source: SourceSnapshot, - pub executable: Option, - pub inputs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] -pub struct IdentityProbeInput { - pub source: SourceSnapshot, - pub label: String, - pub delay_seconds: u64, -} - -pub fn linux_env() -> EnvRef { - clusterflux::env!("linux") -} - -pub fn windows_env() -> EnvRef { - clusterflux::env!("windows") -} - -#[clusterflux::task(capabilities = "source_filesystem")] -pub async fn prepare_source() -> SourceSnapshot { - #[cfg(target_arch = "wasm32")] - { - return clusterflux::source::snapshot() - .await - .expect("the source task should snapshot the node checkout it was placed on"); - } - #[cfg(not(target_arch = "wasm32"))] - SourceSnapshot { - digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - .to_owned(), - } -} - -#[clusterflux::task(capabilities = "command")] -pub async fn compile_linux(source: SourceSnapshot) -> Artifact { - let _ = &source; - #[cfg(target_arch = "wasm32")] - { - let executable = clusterflux::fs::output_path("hello-clusterflux") - .expect("the executable output path should be task-local"); - let output = clusterflux::command::Command::new("cc") - .args([ - "-Os", - "-static", - "-s", - "fixture/hello-clusterflux.c", - "-o", - executable.as_str(), - ]) - .current_dir("/workspace") - .env("SOURCE_DATE_EPOCH", "0") - .timeout(std::time::Duration::from_secs(180)) - .network_disabled() - .output() - .await - .expect("the declared Linux environment should provide a real C compiler"); - assert_eq!(output.status_code, Some(0)); - return clusterflux::fs::flush(&executable) - .await - .expect("the command-created executable should flush to node artifact storage"); - } - #[cfg(not(target_arch = "wasm32"))] - Artifact { - id: "hello-clusterflux-native-test".to_owned(), - digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - .to_owned(), - size_bytes: 0, - } -} - -#[clusterflux::task] -#[unsafe(no_mangle)] -pub extern "C" fn task_add_one(input: i32) -> i32 { - input + 1 -} - -#[clusterflux::task] -#[unsafe(no_mangle)] -pub extern "C" fn task_trap(_input: i32) -> i32 { - #[cfg(target_arch = "wasm32")] - core::arch::wasm32::unreachable(); - #[cfg(not(target_arch = "wasm32"))] - panic!("intentional task trap") -} - -#[clusterflux::task(capabilities = "command")] -pub async fn package_release(input: PackageInput) -> Artifact { - #[cfg(target_arch = "wasm32")] - { - assert_eq!(input.release_name, "release.tar"); - let executable = input - .executable - .as_ref() - .expect("the package input should contain the compiler artifact"); - assert!(input.inputs.iter().any(|artifact| artifact == executable)); - let executable = clusterflux::fs::materialize(&executable, "package/hello-clusterflux") - .await - .expect("the retaining node should materialize the compiler artifact locally"); - let release = clusterflux::fs::output_path("release.tar") - .expect("the release output path should be task-local"); - let output = clusterflux::command::Command::new("tar") - .args([ - "--sort=name", - "--mtime=@0", - "--owner=0", - "--group=0", - "--numeric-owner", - "--mode=0755", - "-cf", - release.as_str(), - "-C", - "/clusterflux/output/package", - "hello-clusterflux", - ]) - .current_dir("/workspace") - .env("SOURCE_DATE_EPOCH", "0") - .timeout(std::time::Duration::from_secs(180)) - .network_disabled() - .output() - .await - .expect("the declared Linux environment should package the materialized executable"); - assert_eq!(output.status_code, Some(0)); - assert_eq!( - executable.as_str(), - "/clusterflux/output/package/hello-clusterflux" - ); - return clusterflux::fs::flush(&release) - .await - .expect("the command-created release archive should flush as the final artifact"); - } - #[cfg(not(target_arch = "wasm32"))] - { - let _ = input; - Artifact { - id: "release-native-test".to_owned(), - digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - .to_owned(), - size_bytes: 0, - } - } -} - -#[clusterflux::task(capabilities = "command")] -pub async fn abort_probe(source: SourceSnapshot) -> i32 { - let _ = &source; - #[cfg(target_arch = "wasm32")] - { - let output = clusterflux::command::Command::new("sh") - .args(["-c", "sleep 90"]) - .timeout(std::time::Duration::from_secs(120)) - .network_disabled() - .output() - .await - .expect("abort probe command should either finish or be stopped by process abort"); - return output.status_code.unwrap_or(-1); - } - #[cfg(not(target_arch = "wasm32"))] - 0 -} - -#[clusterflux::task(capabilities = "command")] -pub async fn long_join_probe(source: SourceSnapshot) -> i32 { - let _ = &source; - #[cfg(target_arch = "wasm32")] - { - let output = clusterflux::command::Command::new("sh") - .args(["-c", "sleep 125"]) - .timeout(std::time::Duration::from_secs(180)) - .network_disabled() - .output() - .await - .expect("the controlled long-running command should complete normally"); - return output.status_code.unwrap_or(-1); - } - #[cfg(not(target_arch = "wasm32"))] - 0 -} - -#[clusterflux::task(capabilities = "command")] -pub async fn identity_probe(input: IdentityProbeInput) -> String { - let _ = &input.source; - #[cfg(target_arch = "wasm32")] - { - let script = format!( - "sleep {}; printf '%s' '{}'", - input.delay_seconds, input.label - ); - let output = clusterflux::command::Command::new("sh") - .args(["-c", script.as_str()]) - .timeout(std::time::Duration::from_secs(60)) - .network_disabled() - .output() - .await - .expect("identity probe command should complete"); - assert_eq!(output.status_code, Some(0)); - return input.label; - } - #[cfg(not(target_arch = "wasm32"))] - input.label -} - -#[clusterflux::task] -pub fn cooperative_cancellation_probe() -> i32 { - #[cfg(target_arch = "wasm32")] - loop { - if clusterflux::process::cancellation_requested() - .expect("task control should remain available while the Wasm task is running") - { - // Cancellation is a request observed and handled by task code. Returning - // normally is intentionally distinct from the runtime's forced abort path. - return 17; - } - } - #[cfg(not(target_arch = "wasm32"))] - 17 -} - -#[clusterflux::task] -pub fn debug_child_probe() -> i32 { - #[cfg(target_arch = "wasm32")] - loop { - if clusterflux::process::cancellation_requested() - .expect("child debug participant should retain task control") - { - return 23; - } - } - #[cfg(not(target_arch = "wasm32"))] - 23 -} - -#[clusterflux::task] -pub async fn debug_parent_probe() -> i32 { - #[cfg(target_arch = "wasm32")] - { - let child = clusterflux::spawn::task(debug_child_probe) - .name("debug child participant") - .start() - .await - .expect("parent debug participant should spawn its child"); - return child - .join() - .await - .expect("parent debug participant should join its child"); - } - #[cfg(not(target_arch = "wasm32"))] - 23 -} - -#[clusterflux::main] -pub async fn build_main() -> BuildReport { - run_build_workflow().await -} - -#[clusterflux::main(name = "fail")] -pub async fn fail_main() -> Result { - #[cfg(target_arch = "wasm32")] - { - let child = clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) - .task_id("task_trap") - .name("intentional failing child") - .start() - .await - .map_err(|error| format!("failing entrypoint could not launch its child: {error}"))?; - return child - .join() - .await - .map_err(|error| format!("intentional child failure: {error}")); - } - #[cfg(not(target_arch = "wasm32"))] - Err("intentional child failure".to_owned()) -} - -#[clusterflux::main(name = "restart")] -pub async fn restart_main() -> i32 { - #[cfg(target_arch = "wasm32")] - { - return clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) - .task_id("task_trap") - .name("edited restart probe") - .failure_policy(clusterflux::spawn::TaskFailurePolicy::AwaitOperator) - .start() - .await - .expect("restart probe should launch its child") - .join() - .await - .expect("restart probe child should complete"); - } - #[cfg(not(target_arch = "wasm32"))] - task_add_one(41) -} - -#[clusterflux::main(name = "park-wake")] -pub async fn park_wake_main() -> i32 { - #[cfg(target_arch = "wasm32")] - { - let mut value = 0_i32; - for _ in 0..16 { - let next = clusterflux::spawn::task_with_arg(value, |input| task_add_one(input)) - .task_id("task_add_one") - .name("park/wake fuel refill probe") - .start() - .await - .expect("park/wake probe should launch") - .join() - .await - .expect("park/wake probe should complete"); - value = i32::try_from(next).expect("park/wake result should remain in i32 range"); - } - return value; - } - #[cfg(not(target_arch = "wasm32"))] - 16 -} - -#[clusterflux::main(name = "long-join")] -pub async fn long_join_main() -> i32 { - #[cfg(target_arch = "wasm32")] - { - let source = clusterflux::spawn::async_task(prepare_source) - .name("prepare source for controlled long-running task") - .start() - .await - .expect("long join source preparation should launch") - .join() - .await - .expect("long join source preparation should complete"); - return clusterflux::spawn::async_task_with_arg(source, long_join_probe) - .name("controlled task longer than two minutes") - .env(linux_env()) - .start() - .await - .expect("long join probe should launch") - .join() - .await - .expect("long join probe should remain joinable beyond two minutes"); - } - #[cfg(not(target_arch = "wasm32"))] - 0 -} - -#[clusterflux::main(name = "identity")] -pub async fn identity_main() -> Vec { - #[cfg(target_arch = "wasm32")] - { - let source = clusterflux::spawn::async_task(prepare_source) - .name("prepare source for identity probes") - .start() - .await - .expect("identity source preparation should launch") - .join() - .await - .expect("identity source preparation should complete"); - let slow = clusterflux::spawn::async_task_with_arg( - IdentityProbeInput { - source: source.clone(), - label: "slow-first".to_owned(), - delay_seconds: 30, - }, - identity_probe, - ) - .name("identity probe slow") - .env(linux_env()) - .start() - .await - .expect("slow identity probe should launch"); - let fast = clusterflux::spawn::async_task_with_arg( - IdentityProbeInput { - source, - label: "fast-second".to_owned(), - delay_seconds: 20, - }, - identity_probe, - ) - .name("identity probe fast") - .env(linux_env()) - .start() - .await - .expect("fast identity probe should launch"); - let fast_result = fast - .join() - .await - .expect("fast identity probe should complete first"); - let slow_result = slow - .join() - .await - .expect("slow identity probe should remain independently joinable"); - return vec![fast_result, slow_result]; - } - #[cfg(not(target_arch = "wasm32"))] - vec!["fast-second".to_owned(), "slow-first".to_owned()] -} - -pub async fn run_build_workflow() -> BuildReport { - let source = clusterflux::spawn::async_task(prepare_source) - .name("prepare source snapshot") - .start() - .await - .unwrap() - .join() - .await - .unwrap(); - let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) - .name("compile linux") - .env(linux_env()) - .start() - .await - .unwrap(); - let linux_parallel = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) - .name("compile linux in parallel") - .env(linux_env()) - .start() - .await - .unwrap(); - let linux_thread = linux.virtual_thread_id(); - let linux_parallel_thread = linux_parallel.virtual_thread_id(); - let linux_artifact = linux.join().await.unwrap(); - let linux_parallel_artifact = linux_parallel.join().await.unwrap(); - assert_eq!(linux_artifact.digest, linux_parallel_artifact.digest); - let package = clusterflux::spawn::async_task_with_arg( - PackageInput { - release_name: "release.tar".to_owned(), - source: source.clone(), - executable: Some(linux_artifact.clone()), - inputs: vec![linux_artifact.clone()], - }, - package_release, - ) - .name("package artifacts") - .env(linux_env()) - .start() - .await - .unwrap(); - - let package_thread = package.virtual_thread_id(); - let package_artifact = package.join().await.unwrap(); - - BuildReport { - linux_thread, - linux_parallel_thread, - package_thread, - linux_artifact, - package_artifact, - source, - } -} - -#[cfg(test)] -mod tests { - use futures_executor::block_on; - - use super::*; - - #[test] - fn flagship_workflow_is_rust_source_with_spawned_virtual_tasks() { - let main_report = block_on(build_main()); - assert_eq!( - main_report.linux_artifact.id, - "hello-clusterflux-native-test" - ); - assert_eq!(task_add_one(41), 42); - assert_eq!(block_on(restart_main()), 42); - assert_eq!(block_on(park_wake_main()), 16); - assert_eq!(block_on(long_join_main()), 0); - assert_eq!( - block_on(identity_main()), - vec!["fast-second".to_owned(), "slow-first".to_owned()] - ); - assert_eq!(linux_env().name, "linux"); - assert_eq!(windows_env().name, "windows"); - - let report = block_on(run_build_workflow()); - - assert_ne!(report.linux_thread, report.package_thread); - assert_ne!(report.linux_thread, report.linux_parallel_thread); - assert_eq!(report.linux_artifact.id, "hello-clusterflux-native-test"); - assert_eq!(report.package_artifact.id, "release-native-test"); - assert_eq!( - report.source.digest, - "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ); - } -}