diff --git a/.gitignore b/.gitignore index 28c74c7..4bdddd1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /target/ -/web/target/ /.clusterflux/ **/.clusterflux/ /vscode-extension/node_modules/ +/private/*/Cargo.lock +!/private/hosted-policy/Cargo.lock +/private/*/target/ +/scripts/containers-home/ diff --git a/CLUSTERFLUX_PUBLIC_TREE.json b/CLUSTERFLUX_PUBLIC_TREE.json new file mode 100644 index 0000000..d6b5837 --- /dev/null +++ b/CLUSTERFLUX_PUBLIC_TREE.json @@ -0,0 +1,22 @@ +{ + "kind": "clusterflux-filtered-public-tree", + "source_commit": "fcace1d13a4b4100c5fe95267d06de68ee6675af", + "release_name": "release-fcace1d13a4b", + "filtered_out": [ + "private/**", + "internal/**", + "experiments/**", + ".git", + "target", + "git-ignored source paths", + "root/*.md except README.md", + "**/.clusterflux/**", + ".forgejo/**" + ], + "public_export": { + "host_neutral": true, + "include_forgejo_workflows": false + }, + "forgejo_host": "git.michelpaulissen.com", + "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com" +} diff --git a/Cargo.lock b/Cargo.lock index 26517cc..c4ac325 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,20 +290,6 @@ dependencies = [ "wasmparser 0.245.1", ] -[[package]] -name = "clusterflux-client" -version = "0.1.0" -dependencies = [ - "base64", - "clusterflux-control", - "clusterflux-coordinator", - "clusterflux-core", - "serde", - "serde_json", - "thiserror 1.0.69", - "tokio", -] - [[package]] name = "clusterflux-control" version = "0.1.0" @@ -1726,6 +1712,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "runtime-conformance" +version = "0.1.0" +dependencies = [ + "clusterflux-sdk", + "futures-executor", + "serde", + "serde_json", +] + [[package]] name = "rustc-hash" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index b4ff4d4..3c10aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,7 @@ [workspace] resolver = "2" -exclude = ["web"] members = [ "crates/clusterflux-cli", - "crates/clusterflux-client", "crates/clusterflux-control", "crates/clusterflux-coordinator", "crates/clusterflux-core", @@ -14,12 +12,13 @@ members = [ "crates/clusterflux-wasm-runtime", "examples/hello-build", "examples/recovery-build", + "tests/fixtures/runtime-conformance", ] [workspace.package] edition = "2021" license = "Apache-2.0 OR MIT" -repository = "https://clusterflux.lesstuff.com" +repository = "https://git.michelpaulissen.com/michel/clusterflux-public" [workspace.dependencies] anyhow = "1.0" diff --git a/README.md b/README.md index e2c11ac..de9d56c 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,9 @@ # Clusterflux -Clusterflux runs a Rust-defined workflow as one distributed virtual process. The -async main runs serverless, provisioning nodes to run tasks through rootless -containers. Tasks on nodes exchange data simply and efficiently. - -The user experience is built to be as much as possible like building a regular -program. The processes are debuggable as normal through a debugger adapter. - -The primary use case is to consolidate build processes into a single streamlined -developer experience. An example program can be seen below: - -~~~rust -use clusterflux::prelude::*; - -#[clusterflux::task(capabilities = "command")] -pub async fn compile(source: SourceSnapshot) -> Result { - let executable = fs::output("hello-clusterflux")?; - Command::new("cc") - .args([ - "-Os", - "-static", - "-s", - "fixture/hello-clusterflux.c", - "-o", - executable.as_str(), - ]) - .cwd(source.mount()?) - .env("SOURCE_DATE_EPOCH", "0") - .network_disabled() - .run() - .await?; - fs::publish(&executable).await -} - -#[clusterflux::main] -pub async fn build() -> Result { - let source = source::current_project().snapshot().await?; - let compile = clusterflux::spawn!(compile(source)) - .on(clusterflux::env!("linux")) - .await?; - compile.join().await -} -~~~ - -After setup, this build pipeline could be deployed as easily as launching it -through your IDE. This repository includes a VS Code extension to make -development as straightforward as possible. A full collection of CLI tools is -included for advanced usage. - -Clusterflux is explicitly local-first. It is trivial to provision existing -hardware as resources. Bulk data will typically not leave the local network, -allowing maximum throughput. The same capability, however, also makes it -possible to leverage cloud resources easily. +Clusterflux runs a Rust-defined workflow as one distributed virtual process. A +coordinator hosts the async main, while attached nodes execute Wasm tasks, +rootless containers, and native commands. Tasks exchange canonical values and +portable typed handles instead of sharing host memory. Start with [Getting started](docs/getting-started.md). It takes you through authentication, project setup, node enrollment, a run, debugging, task restart, @@ -97,9 +49,6 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ -On NixOS or another system with Nix, the equivalent package is available with -`nix profile install .#clusterflux-tools`. - Rootless Podman is required on Linux nodes that build or run a declared Containerfile environment. Install VS Code when you want the graphical debug workflow. diff --git a/SECURITY.md b/SECURITY.md index 2c1cd10..a050c6d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained. ## Report a vulnerability -Email security@clusterflux.lesstuff.com with a concise description, affected version +Email security@michelpaulissen.com with a concise description, affected version or source revision, reproduction steps, and impact. Do not open a public issue for an unpatched vulnerability or include credentials, session tokens, private keys, provider tokens, customer data, or operator secrets in a public report. diff --git a/crates/clusterflux-cli/src/admin.rs b/crates/clusterflux-cli/src/admin.rs index ce17d88..83f19a6 100644 --- a/crates/clusterflux-cli/src/admin.rs +++ b/crates/clusterflux-cli/src/admin.rs @@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { .unwrap_or(json!(false)), "response": response, "safe_default": "read_only", - "external_website_required": false, + "private_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { "command": "admin status", "mode": "self_hosted_local", "safe_default": "read_only", - "external_website_required": false, + "private_website_required": false, })) } @@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "project": project.clone(), "user": user, "coordinator": scope.coordinator, - "external_website_required": false, + "private_website_required": false, "self_hosted_cli_only": true, "project_config_written": project_init .get("project_config_written") @@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> { "step": "start_self_hosted_coordinator", "command": "clusterflux-coordinator --listen 127.0.0.1:0", - "external_website_required": false, + "private_website_required": false, }, { "step": "create_or_link_project", "command": "clusterflux project init --yes", "completed": true, - "external_website_required": false, + "private_website_required": false, }, { "step": "create_node_enrollment_grant", @@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "attach_worker_node", @@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "run_process", @@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "inspect_status_logs_artifacts", @@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux artifact list", "clusterflux quota status", ], - "external_website_required": false, + "private_website_required": false, }, { "step": "revoke_access", "command": "clusterflux admin revoke-node --node --yes", - "external_website_required": false, + "private_website_required": false, } ], })) @@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "actor_tenant": actor_tenant, "actor_user": actor_user, "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"), - "external_website_required": false, + "private_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "status": "requires_coordinator", "requires_confirmation": !args.yes, "tenant": tenant, - "external_website_required": false, + "private_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/artifact.rs b/crates/clusterflux-cli/src/artifact.rs index a2c0620..a0440be 100644 --- a/crates/clusterflux-cli/src/artifact.rs +++ b/crates/clusterflux-cli/src/artifact.rs @@ -12,7 +12,6 @@ use crate::client::{ }; use crate::config::StoredCliSession; use crate::errors::cli_error_summary_for_category; -use crate::process::hydrate_process_scope; use crate::process_events::{ artifact_download_grant_disclosures, artifact_download_session_summary, artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries, @@ -28,10 +27,9 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result { } pub(crate) fn artifact_list_report_with_session( - mut args: ArtifactListArgs, + args: ArtifactListArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, @@ -55,10 +53,9 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( @@ -146,10 +143,9 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result } pub(crate) fn artifact_export_report_with_session( - mut args: ArtifactExportArgs, + args: ArtifactExportArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( diff --git a/crates/clusterflux-cli/src/auth.rs b/crates/clusterflux-cli/src/auth.rs index d820cbc..608c9ee 100644 --- a/crates/clusterflux-cli/src/auth.rs +++ b/crates/clusterflux-cli/src/auth.rs @@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result Value { "account_status": "unknown", "suspension_known": false, "account_state_known": false, - "sensitive_moderation_details_exposed": false, + "private_moderation_details_exposed": false, "signup_failure_details_exposed": false, "next_actions": ["clusterflux login --browser"], }) diff --git a/crates/clusterflux-cli/src/config.rs b/crates/clusterflux-cli/src/config.rs index ab744ca..db22a3f 100644 --- a/crates/clusterflux-cli/src/config.rs +++ b/crates/clusterflux-cli/src/config.rs @@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize}; use crate::CliScopeArgs; -pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com"; +pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = + "https://clusterflux.michelpaulissen.com"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct ProjectConfig { diff --git a/crates/clusterflux-cli/src/debug.rs b/crates/clusterflux-cli/src/debug.rs index 356ce8b..d39c780 100644 --- a/crates/clusterflux-cli/src/debug.rs +++ b/crates/clusterflux-cli/src/debug.rs @@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result { "command": "dap", "adapter": dap_binary_path()?.display().to_string(), "args": args.args, - "external_website_required": false, + "private_website_required": false, })) } @@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session( .cloned() .unwrap_or_else(|| json!(0)), "debug_reads_quota_limited": true, - "external_website_required": false, + "private_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session( "dap": dap, "authorized": "unknown_without_coordinator", "debug_reads_quota_limited": "unknown_without_coordinator", - "external_website_required": false, + "private_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/errors.rs b/crates/clusterflux-cli/src/errors.rs index 282a91d..0dce2be 100644 --- a/crates/clusterflux-cli/src/errors.rs +++ b/crates/clusterflux-cli/src/errors.rs @@ -38,10 +38,7 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s ); object.insert("community_tier_language".to_owned(), json!(true)); object.insert("community_tier_label".to_owned(), json!("community tier")); - object.insert( - "sensitive_abuse_heuristics_exposed".to_owned(), - json!(false), - ); + object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false)); } } summary diff --git a/crates/clusterflux-cli/src/logs.rs b/crates/clusterflux-cli/src/logs.rs index dcad5f7..a12815a 100644 --- a/crates/clusterflux-cli/src/logs.rs +++ b/crates/clusterflux-cli/src/logs.rs @@ -3,7 +3,6 @@ use serde_json::{json, Value}; use crate::client::list_task_events_if_available_with_session; use crate::config::StoredCliSession; -use crate::process::hydrate_process_scope; use crate::process_events::log_entries; use crate::LogsArgs; @@ -13,10 +12,9 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result { } pub(crate) fn logs_report_with_session( - mut args: LogsArgs, + args: LogsArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, diff --git a/crates/clusterflux-cli/src/node.rs b/crates/clusterflux-cli/src/node.rs index 3f04516..917310d 100644 --- a/crates/clusterflux-cli/src/node.rs +++ b/crates/clusterflux-cli/src/node.rs @@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result Result Result }; let heartbeat_request = json!({ "type": "node_heartbeat", - "tenant": &tenant, - "project": &project, "node": &plan.node, }); let heartbeat_signature = sign_node_request( diff --git a/crates/clusterflux-cli/src/output.rs b/crates/clusterflux-cli/src/output.rs index 296f142..d5c2b57 100644 --- a/crates/clusterflux-cli/src/output.rs +++ b/crates/clusterflux-cli/src/output.rs @@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String { } push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason"); if let Some(exposed) = account - .get("sensitive_moderation_details_exposed") + .get("private_moderation_details_exposed") .and_then(Value::as_bool) { - lines.push(format!("sensitive moderation details exposed: {exposed}")); + lines.push(format!("private moderation details exposed: {exposed}")); } } if let Some(coordinator_selection) = value.get("coordinator") { @@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String { )); } if let Some(flag) = value - .get("external_website_required") + .get("private_website_required") .and_then(Value::as_bool) { - lines.push(format!("external website required: {flag}")); + lines.push(format!("private website required: {flag}")); } if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) { let actions = next_actions diff --git a/crates/clusterflux-cli/src/process.rs b/crates/clusterflux-cli/src/process.rs index 5e6adde..e7696b4 100644 --- a/crates/clusterflux-cli/src/process.rs +++ b/crates/clusterflux-cli/src/process.rs @@ -15,10 +15,7 @@ use crate::{ ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs, }; -pub(crate) fn hydrate_process_scope( - scope: &mut CliScopeArgs, - stored_session: Option<&StoredCliSession>, -) { +fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) { if scope.coordinator.is_none() { scope.coordinator = stored_session .filter(|session| session.session_secret.is_some()) diff --git a/crates/clusterflux-cli/src/process_events.rs b/crates/clusterflux-cli/src/process_events.rs index 50adc60..fa577c7 100644 --- a/crates/clusterflux-cli/src/process_events.rs +++ b/crates/clusterflux-cli/src/process_events.rs @@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value { "cross_tenant_reuse_allowed": false, "unauthorized_project_reuse_allowed": false, "default_durable_store_assumed": false, - "external_website_required": false, + "private_website_required": false, }]) } @@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option< "current_usage": current_usage, "limits": quota_limits_value(), "next_blocked_action": next_blocked_action, - "sensitive_abuse_heuristics_exposed": false, + "private_abuse_heuristics_exposed": false, }) } diff --git a/crates/clusterflux-cli/src/project.rs b/crates/clusterflux-cli/src/project.rs index 1573880..47140db 100644 --- a/crates/clusterflux-cli/src/project.rs +++ b/crates/clusterflux-cli/src/project.rs @@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result Ok(json!({ "command": "project init", "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, - "external_website_required": false, + "private_website_required": false, "project_config_written": true, "project_config_write_after_coordinator_acceptance": coordinator.is_some(), "coordinator_create_before_local_write": coordinator.is_some(), @@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "config_format": "clusterflux_project_config_v1", "links_current_directory": true, "writes_current_directory_only": true, - "external_website_required": false, + "private_website_required": false, }, "safe_defaults": { "tenant": config.tenant.clone(), @@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "default_project_id_used": args.new_project == "project", "default_project_name_used": args.name == "Clusterflux Project", "browser_interaction_required": false, - "external_website_required": false, + "private_website_required": false, }, "project_config": config, "config_file": project_config_file(&cwd), @@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "user": user, "projects": projects, "project_count": project_count, - "external_website_required": false, + "private_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "source": "local_project_config", "projects": projects, "project_count": project_count, - "external_website_required": false, + "private_website_required": false, })) } @@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, "selected_project": selected_project, "project_config_written": true, - "external_website_required": false, + "private_website_required": false, "project_config": config, "coordinator_response": coordinator_response, })) diff --git a/crates/clusterflux-cli/src/quota.rs b/crates/clusterflux-cli/src/quota.rs index 9c30880..a37977f 100644 --- a/crates/clusterflux-cli/src/quota.rs +++ b/crates/clusterflux-cli/src/quota.rs @@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "user": effective_scope.user, "coordinator": coordinator, "project_config": config, - "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined", + "policy_surface": "generic public quota categories; hosted tuning remains private policy", "limits": limits, "window_seconds": window_seconds, "current_usage": current_usage, @@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "next_blocked_action": quota_next_blocked_action(¤t_usage), "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" }, "quota_tier": quota_tier, - "sensitive_abuse_heuristics_exposed": false, + "private_abuse_heuristics_exposed": false, "quota_response": quota_status, })) } diff --git a/crates/clusterflux-cli/src/run.rs b/crates/clusterflux-cli/src/run.rs index 7dfc187..b12d5a5 100644 --- a/crates/clusterflux-cli/src/run.rs +++ b/crates/clusterflux-cli/src/run.rs @@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu "safe_failure": true, "message": message, "next_actions": next_actions, - "external_website_required": false, + "private_website_required": false, "machine_error": machine_error, }) } @@ -329,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result { "task_launch": launch_task_response, "coordinator_response": response, "coordinator_session_requests": session.requests(), - "external_website_required": false, + "private_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/tests.rs b/crates/clusterflux-cli/src/tests.rs index 57b4afe..c122a86 100644 --- a/crates/clusterflux-cli/src/tests.rs +++ b/crates/clusterflux-cli/src/tests.rs @@ -124,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() { assert_eq!(summary["resource_category"], "api_calls"); assert_eq!(summary["community_tier_language"], true); assert_eq!(summary["community_tier_label"], "community tier"); - assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false); + assert_eq!(summary["private_abuse_heuristics_exposed"], false); let rendered = human_report(&json!({ "command": "run", "machine_error": summary, @@ -207,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() { assert_eq!(report["status"], "authentication_required"); assert_eq!(report["non_interactive"], true); assert_eq!(report["browser_opened"], false); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["machine_error"]["category"], "authentication"); assert_eq!(report["machine_error"]["stable_exit_code"], 20); assert_eq!(report["machine_error"]["browser_opened"], false); @@ -827,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() { assert_eq!(rejected["machine_error"]["resource_category"], "api_calls"); assert_eq!(rejected["machine_error"]["community_tier_language"], true); assert_eq!( - rejected["machine_error"]["sensitive_abuse_heuristics_exposed"], + rejected["machine_error"]["private_abuse_heuristics_exposed"], false ); assert!(rejected["machine_error"]["next_actions"] @@ -1665,11 +1665,12 @@ fn node_attach_refuses_a_symlink_credential_target() { fn hosted_coordinator_remains_a_real_https_control_endpoint() { assert_eq!( control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") + .unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert!(control_endpoint_identity("http://operator.example.test").is_err()); assert_eq!( @@ -1787,7 +1788,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { assert!(!line.contains(r#""actor_user":"user-session""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, + br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1850,7 +1851,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { "active" ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); } @@ -1932,7 +1933,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() { } #[test] -fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() { +fn auth_status_queries_coordinator_account_state_without_private_moderation_details() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap().to_string(); let server = std::thread::spawn(move || { @@ -1946,7 +1947,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de assert!(line.contains(r#""actor_user":"user-live""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#, + br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1986,7 +1987,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de "account or tenant is suspended by hosted policy" ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); assert_eq!( @@ -2004,13 +2005,13 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de let serialized = serde_json::to_string(&report).unwrap(); assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); - assert!(!serialized.contains("sensitive moderation note")); + assert!(!serialized.contains("private moderation note")); let rendered = human_report(&report); assert!(rendered.contains("account status: suspended")); assert!(rendered.contains("account suspended: true")); - assert!(rendered.contains("sensitive moderation details exposed: false")); - assert!(!rendered.contains("sensitive moderation note")); + assert!(rendered.contains("private moderation details exposed: false")); + assert!(!rendered.contains("private moderation note")); } #[test] @@ -2061,11 +2062,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { "manual_review": manual_review, "sanitized_reason": reason, "next_actions": ["contact the hosted operator"], - "sensitive_moderation_details_exposed": false, + "private_moderation_details_exposed": false, "signup_failure_details_exposed": false, "abuse_score": 99, - "moderation_notes": "sensitive moderation note", - "signup_policy_trace": "sensitive signup trace", + "moderation_notes": "private moderation note", + "signup_policy_trace": "private signup trace", }) ) .unwrap(); @@ -2104,7 +2105,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { true ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); assert_eq!( @@ -2115,11 +2116,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); assert!(!serialized.contains("signup_policy_trace")); - assert!(!serialized.contains("sensitive moderation note")); + assert!(!serialized.contains("private moderation note")); let rendered = human_report(&report); assert!(rendered.contains(&format!("account status: {status}"))); assert!(rendered.contains(rendered_marker)); - assert!(!rendered.contains("sensitive moderation note")); + assert!(!rendered.contains("private moderation note")); } server.join().unwrap(); } @@ -2234,11 +2235,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { assert_eq!(report["command"], "admin bootstrap"); assert_eq!(report["mode"], "self_hosted_local"); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["self_hosted_cli_only"], true); assert_eq!(report["project_config_written"], true); assert_eq!(report["project_init"]["command"], "project init"); - assert_eq!(report["project_init"]["external_website_required"], false); + assert_eq!(report["project_init"]["private_website_required"], false); assert_eq!( report["project_init"]["project_config"]["project"], "self-hosted" @@ -2264,7 +2265,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { } assert!(steps.iter().all(|step| { !step - .get("external_website_required") + .get("private_website_required") .and_then(Value::as_bool) .unwrap_or(false) })); @@ -2892,13 +2893,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() { assert_eq!(status["command"], "admin status"); assert_eq!(status["safe_default"], "read_only"); - assert_eq!(status["external_website_required"], false); + assert_eq!(status["private_website_required"], false); assert_eq!(status["suspended"], false); assert_eq!(suspended["command"], "admin suspend-tenant"); assert_eq!(suspended["tenant"], "tenant"); assert_eq!(suspended["actor_tenant"], "admin-tenant"); assert_eq!(suspended["suspended"], true); - assert_eq!(suspended["external_website_required"], false); + assert_eq!(suspended["private_website_required"], false); } #[test] @@ -2967,7 +2968,7 @@ fn debug_attach_reports_public_authorization() { assert_eq!(report["charged_debug_read_bytes"], 1024); assert_eq!(report["used_debug_read_bytes"], 1024); assert_eq!(report["debug_reads_quota_limited"], true); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); } #[test] @@ -3004,14 +3005,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { "restart_task", br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(), ), - ( - "list_task_events", - br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(), - ), - ( - "list_task_events", - br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(), - ), ( "create_artifact_download_link", br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(), @@ -3116,26 +3109,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); - let logs = logs_report_with_session( - LogsArgs { - scope: scope.clone(), - process: Some("vp".to_owned()), - task: Some("task-a".to_owned()), - }, - Some(&session), - ) - .unwrap(); - let artifacts = artifact_list_report_with_session( - ArtifactListArgs { - scope: scope.clone(), - process: Some("vp".to_owned()), - }, - Some(&session), - ) - .unwrap(); - let download = artifact_download_report_with_session( + artifact_download_report_with_session( ArtifactDownloadArgs { - scope, + scope: coordinator_scope.clone(), artifact: "app.txt".to_owned(), to: None, max_bytes: 2048, @@ -3143,9 +3119,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); - assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n"); - assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt"); - assert_eq!(download["coordinator"], session.coordinator); debug_attach_report_with_dap_and_session( DebugAttachArgs { scope: coordinator_scope, @@ -3338,7 +3311,7 @@ fn project_init_uses_public_create_before_writing_local_config() { created["safe_defaults"]["browser_interaction_required"], false ); - assert_eq!(created["external_website_required"], false); + assert_eq!(created["private_website_required"], false); assert_eq!( read_project_config(temp_success.path()) .unwrap() @@ -3438,7 +3411,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(list["source"], "public_coordinator_api"); assert_eq!(list["project_count"], 1); assert_eq!(list["projects"][0]["id"], "project-a"); - assert_eq!(list["external_website_required"], false); + assert_eq!(list["private_website_required"], false); assert_eq!(list["coordinator_session_requests"], 1); let selected = project_select_report( @@ -3453,7 +3426,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(selected["source"], "public_coordinator_api"); assert_eq!(selected["selected_project"]["id"], "project-a"); assert_eq!(selected["project_config_written"], true); - assert_eq!(selected["external_website_required"], false); + assert_eq!(selected["private_website_required"], false); assert_eq!( read_project_config(temp.path()).unwrap().unwrap().project, "project-a" @@ -4482,7 +4455,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { let temp = tempfile::tempdir().unwrap(); let project = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../..") - .join("examples/hello-build"); + .join("tests/fixtures/runtime-conformance"); let output = temp.path().join("bundle"); let report = build_report( @@ -4500,8 +4473,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { assert_eq!(report["command"], "build"); assert_eq!(report["content_addressed"], true); assert_eq!(report["contains_full_repository_upload"], false); - assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2); - assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1); + assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 12); + assert_eq!(report["bundle_artifact"]["entrypoint_count"], 6); assert!(output.join("module.wasm").is_file()); assert!(output.join("manifest.json").is_file()); assert!(output.join("task-descriptors.json").is_file()); @@ -4512,9 +4485,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { .unwrap() .iter() .any(|descriptor| { - descriptor["name"] == "compile" - && descriptor["argument_schema"] == "source : SourceSnapshot" - && descriptor["result_schema"] == "Result < Artifact >" + descriptor["name"] == "task_add_one" + && descriptor["argument_schema"] == "input : i32" + && descriptor["result_schema"] == "i32" && descriptor["restart_compatibility_hash"] .as_str() .unwrap() @@ -4675,7 +4648,7 @@ fn node_enroll_reports_short_lived_public_api_grant() { assert_eq!(report["command"], "node enroll"); assert_eq!(report["status"], "created"); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["tenant"], "tenant"); assert_eq!(report["project"], "project"); assert_eq!(report["user"], "user"); @@ -4818,7 +4791,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() { ) .unwrap(); assert_eq!(enroll["status"], "requires_coordinator"); - assert_eq!(enroll["external_website_required"], false); + assert_eq!(enroll["private_website_required"], false); assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null); assert_eq!(enroll["requested_ttl_seconds"], 60); diff --git a/crates/clusterflux-client/Cargo.toml b/crates/clusterflux-client/Cargo.toml deleted file mode 100644 index fa2b5cc..0000000 --- a/crates/clusterflux-client/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "clusterflux-client" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -base64.workspace = true -clusterflux-control = { path = "../clusterflux-control" } -clusterflux-core = { path = "../clusterflux-core" } -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tokio = { workspace = true, features = ["sync", "time"] } - -[dev-dependencies] -clusterflux-coordinator = { path = "../clusterflux-coordinator" } diff --git a/crates/clusterflux-client/README.md b/crates/clusterflux-client/README.md deleted file mode 100644 index 89951c1..0000000 --- a/crates/clusterflux-client/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# clusterflux-client - -`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web -backend. It talks to the same versioned control and login endpoints as the CLI. -Callers do not construct JSON envelopes or place session secrets into request -objects. - -The crate intentionally contains no coordinator implementation, persistence -types, scheduler policy, or website-specific business rules. A web backend only -needs this crate to authenticate and work with account status, projects, Agent -keys, node enrollment and liveness, current/recent processes, task attempts, -recent logs, artifacts, quota state, process control, task recovery, and Debug -Epoch state. It does not expose workflow launch or whole-process replay. - -```rust,no_run -use clusterflux_client::{ClusterfluxClient, SessionCredential}; - -# async fn example() -> Result<(), clusterflux_client::ClientError> { -let credential = SessionCredential::from_secret( - std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"), -); -let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")? - .with_session_credential(&credential); - -let account = client.account_status().await?; -let nodes = client.list_nodes().await?; -let processes = client.list_processes(None, 20).await?; -# let _ = (account, nodes, processes); -# Ok(()) -# } -``` - -## Login boundary - -`begin_browser_login` starts the hosted Authentik flow. The identity callback -returns only to the fixed configured website URL with a short-lived handoff. -The website backend calls `exchange_browser_login_handoff` once and stores the -returned `SessionCredential` only in encrypted server-side session storage. -The credential has redacted `Debug` output and deliberately does not implement -Serde serialization. Logout calls the existing session-revocation operation. - -The browser must not receive or persist the Clusterflux session credential, -provider tokens, authorization code, PKCE verifier, or CLI polling secret. - -## Errors, bounds, and transport - -API failures are returned as `ClientError::Api(ApiError)`. Decisions should use -the stable `code`, `category`, `retryable`, and `request_id` fields, while -`message` is for people. The client rejects an error whose request ID does not -match the originating request. - -Paginated list methods require an explicit bounded page size. Process pages are -limited to 100 entries; node, artifact, and recent-log pages to 200. -`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes -its cursor. Recent logs are ephemeral and can report truncation or sequence -loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates -offsets and decodes bounded chunks without materializing the complete artifact. - -`ControlTransport` has bounded connect and I/O timeouts and performs blocking -network work outside the async executor. Dropping a request future cancels the -caller’s wait; any already-started blocking I/O remains bounded by its timeout. -`MockTransport` supplies deterministic responses and records exact envelopes -for application tests. - -`CLIENT_API_VERSION` is the one supported control protocol version. The -checked-in `web_operations.json` contract fixture records every website -operation and its stable error shape, and the crate’s contract suite checks the -fixture. diff --git a/crates/clusterflux-client/src/lib.rs b/crates/clusterflux-client/src/lib.rs deleted file mode 100644 index 93c6feb..0000000 --- a/crates/clusterflux-client/src/lib.rs +++ /dev/null @@ -1,924 +0,0 @@ -mod protocol; -mod transport; -mod types; - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; - -use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; -use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; -use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION}; -use protocol::{AuthenticatedRequest, LoginRequest, WireResponse}; -use serde_json::json; -use thiserror::Error; -use transport::{ClientTransport, TransportRequest}; - -pub use clusterflux_core::{ - AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind, - Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId, - ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId, - UserId, VfsPath, -}; -pub use transport::{ - ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse, -}; -pub use types::*; - -pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION; - -#[derive(Debug, Error)] -pub enum ClientError { - #[error("Clusterflux API error: {0}")] - Api(ApiError), - #[error(transparent)] - Transport(#[from] ClientTransportError), - #[error("Clusterflux client protocol error: {0}")] - Protocol(String), -} - -#[derive(Clone)] -pub struct ClusterfluxClient { - transport: Arc, - session_secret: Arc>>, - next_request: Arc, -} - -impl ClusterfluxClient { - pub fn connect(endpoint: impl Into) -> Result { - Ok(Self::with_transport(ControlTransport::new(endpoint)?)) - } - - pub fn with_transport(transport: impl ClientTransport) -> Self { - Self { - transport: Arc::new(transport), - session_secret: Arc::new(Mutex::new(None)), - next_request: Arc::new(AtomicU64::new(1)), - } - } - - pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self { - self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone()))); - self - } - - pub fn is_session_configured(&self) -> bool { - self.session_secret - .lock() - .map(|secret| secret.is_some()) - .unwrap_or(false) - } - - pub async fn begin_browser_login(&self) -> Result { - match self - .send_login(LoginRequest::BeginWebBrowserLogin {}) - .await? - { - WireResponse::WebBrowserLoginStarted { - transaction_id, - authorization_url, - expires_at_epoch_seconds, - } => Ok(BrowserLoginStart { - transaction_id, - authorization_url, - expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("web_browser_login_started")), - } - } - - pub async fn exchange_browser_login_handoff( - &self, - transaction_id: impl Into, - handoff_code: impl Into, - ) -> Result { - match self - .send_login(LoginRequest::ExchangeWebLoginHandoff { - transaction_id: transaction_id.into(), - handoff_code: handoff_code.into(), - }) - .await? - { - WireResponse::WebBrowserSession { session } => Ok(BrowserSession { - tenant: session.tenant, - project: session.project, - user: session.user, - credential: SessionCredential(session.session_secret), - expires_at_epoch_seconds: session.expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("web_browser_session")), - } - } - - pub async fn cancel_browser_login( - &self, - transaction_id: impl Into, - ) -> Result<(), ClientError> { - match self - .send_login(LoginRequest::CancelWebBrowserLogin { - transaction_id: transaction_id.into(), - }) - .await? - { - WireResponse::WebBrowserLoginCancelled {} => Ok(()), - _ => Err(unexpected_response("web_browser_login_cancelled")), - } - } - - pub async fn account_status(&self) -> Result { - match self - .send_authenticated(AuthenticatedRequest::AuthStatus) - .await? - { - WireResponse::AuthStatus { - tenant, - project, - actor, - authenticated, - account_status, - suspended, - disabled, - deleted, - manual_review, - sanitized_reason, - next_actions, - } => Ok(AccountStatus { - tenant, - project, - actor, - authenticated, - account_status, - suspended, - disabled, - deleted, - manual_review, - sanitized_reason, - next_actions, - }), - _ => Err(unexpected_response("auth_status")), - } - } - - pub async fn logout(&self) -> Result<(), ClientError> { - match self - .send_authenticated(AuthenticatedRequest::RevokeCliSession) - .await? - { - WireResponse::CliSessionRevoked {} => { - *self.session_secret.lock().map_err(|_| { - ClientError::Protocol("session credential lock was poisoned".to_owned()) - })? = None; - Ok(()) - } - _ => Err(unexpected_response("cli_session_revoked")), - } - } - - pub async fn list_projects(&self) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListProjects) - .await? - { - WireResponse::Projects { projects } => Ok(projects), - _ => Err(unexpected_response("projects")), - } - } - - pub async fn create_project( - &self, - project: ProjectId, - name: impl Into, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateProject { - project, - name: name.into(), - }) - .await? - { - WireResponse::ProjectCreated { project } => Ok(project), - _ => Err(unexpected_response("project_created")), - } - } - - pub async fn select_project(&self, project: ProjectId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::SelectProject { project }) - .await? - { - WireResponse::ProjectSelected { project } => Ok(project), - _ => Err(unexpected_response("project_selected")), - } - } - - pub async fn register_agent_public_key( - &self, - agent: AgentId, - public_key: impl Into, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey { - agent, - public_key: public_key.into(), - }) - .await - } - - pub async fn list_agent_public_keys(&self) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys) - .await? - { - WireResponse::AgentPublicKeys { records } => Ok(records), - _ => Err(unexpected_response("agent_public_keys")), - } - } - - pub async fn rotate_agent_public_key( - &self, - agent: AgentId, - public_key: impl Into, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey { - agent, - public_key: public_key.into(), - }) - .await - } - - pub async fn revoke_agent_public_key( - &self, - agent: AgentId, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent }) - .await - } - - async fn agent_key_mutation( - &self, - request: AuthenticatedRequest, - ) -> Result { - match self.send_authenticated(request).await? { - WireResponse::AgentPublicKey { record } => Ok(record), - _ => Err(unexpected_response("agent_public_key")), - } - } - - pub async fn create_node_enrollment_grant( - &self, - ttl_seconds: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds }) - .await? - { - WireResponse::NodeEnrollmentGrantCreated { - tenant, - project, - grant, - scope, - expires_at_epoch_seconds, - } => Ok(NodeEnrollmentGrant { - tenant, - project, - grant, - scope, - expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("node_enrollment_grant_created")), - } - } - - pub async fn list_nodes(&self) -> Result, ClientError> { - Ok(self.list_nodes_page(None, 200).await?.nodes) - } - - pub async fn list_nodes_page( - &self, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit }) - .await? - { - WireResponse::NodeSummaries { nodes, next_cursor } => { - Ok(NodePage { nodes, next_cursor }) - } - _ => Err(unexpected_response("node_summaries")), - } - } - - pub async fn revoke_node(&self, node: NodeId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node }) - .await? - { - WireResponse::NodeCredentialRevoked { - node, - tenant, - project, - actor, - descriptor_removed, - queued_assignments_removed, - } => Ok(NodeRevocation { - node, - tenant, - project, - actor, - descriptor_removed, - queued_assignments_removed, - }), - _ => Err(unexpected_response("node_credential_revoked")), - } - } - - pub async fn list_processes( - &self, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit }) - .await? - { - WireResponse::ProcessSummaries { - processes, - next_cursor, - } => Ok(ProcessPage { - processes, - next_cursor, - }), - _ => Err(unexpected_response("process_summaries")), - } - } - - pub async fn cancel_process( - &self, - process: ProcessId, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CancelProcess { process }) - .await? - { - WireResponse::ProcessCancellationRequested { - process, - cancelled_tasks, - affected_nodes, - } => Ok(ProcessCancellation { - process, - affected_tasks: cancelled_tasks, - affected_nodes, - aborted: false, - }), - _ => Err(unexpected_response("process_cancellation_requested")), - } - } - - pub async fn abort_process( - &self, - process: ProcessId, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::AbortProcess { - process, - launch_attempt: None, - }) - .await? - { - WireResponse::ProcessAborted { - process, - aborted_tasks, - affected_nodes, - } => Ok(ProcessCancellation { - process, - affected_tasks: aborted_tasks, - affected_nodes, - aborted: true, - }), - _ => Err(unexpected_response("process_aborted")), - } - } - - pub async fn quota_status(&self) -> Result { - match self - .send_authenticated(AuthenticatedRequest::QuotaStatus) - .await? - { - WireResponse::QuotaStatus { - tenant, - project, - actor, - policy_label, - limits, - window_seconds, - usage, - window_started_epoch_seconds, - } => Ok(QuotaStatus { - tenant, - project, - actor, - policy_label, - limits, - window_seconds, - usage, - window_started_epoch_seconds, - }), - _ => Err(unexpected_response("quota_status")), - } - } - - pub async fn list_task_events( - &self, - process: Option, - ) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListTaskEvents { process }) - .await? - { - WireResponse::TaskEvents { events } => Ok(events), - _ => Err(unexpected_response("task_events")), - } - } - - pub async fn list_task_snapshots( - &self, - process: ProcessId, - ) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process }) - .await? - { - WireResponse::TaskSnapshots { snapshots } => Ok(snapshots), - _ => Err(unexpected_response("task_snapshots")), - } - } - - pub async fn list_recent_logs( - &self, - process: ProcessId, - task: Option, - after_sequence: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListRecentLogs { - process, - task, - after_sequence, - limit, - }) - .await? - { - WireResponse::RecentLogs { - entries, - next_sequence, - history_truncated, - } => Ok(RecentLogPage { - entries, - next_sequence, - history_truncated, - }), - _ => Err(unexpected_response("recent_logs")), - } - } - - pub async fn restart_task( - &self, - process: ProcessId, - task: TaskInstanceId, - replacement_bundle: Option, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::RestartTask { - process, - task, - replacement_bundle, - }) - .await? - { - WireResponse::TaskRestart { - process, - task, - restarted_task_instance, - restarted_attempt_id, - actor, - accepted, - clean_boundary_available, - active_task, - completed_event_observed, - requires_whole_process_restart, - message, - audit_event, - } => Ok(TaskRestart { - process, - task, - restarted_task_instance, - restarted_attempt_id, - actor, - accepted, - clean_boundary_available, - active_task, - completed_event_observed, - requires_whole_process_restart, - message, - audit_event, - }), - _ => Err(unexpected_response("task_restart")), - } - } - - pub async fn resolve_task_failure( - &self, - process: ProcessId, - task: TaskInstanceId, - resolution: TaskFailureResolution, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ResolveTaskFailure { - process, - task, - resolution, - }) - .await? - { - WireResponse::TaskFailureResolved { - process, - task, - attempt_id, - resolution, - } => Ok(TaskFailureResolutionResult { - process, - task, - attempt_id, - resolution, - }), - _ => Err(unexpected_response("task_failure_resolved")), - } - } - - pub async fn debug_attach(&self, process: ProcessId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::DebugAttach { process }) - .await? - { - WireResponse::DebugAttach { - process, - actor, - authorization, - audit_event, - } => Ok(DebugAttach { - process, - actor, - authorization, - audit_event, - }), - _ => Err(unexpected_response("debug_attach")), - } - } - - pub async fn create_debug_epoch( - &self, - process: ProcessId, - stopped_task: TaskInstanceId, - reason: impl Into, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateDebugEpoch { - process, - stopped_task, - reason: reason.into(), - }) - .await? - { - WireResponse::DebugEpoch { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - } => Ok(DebugEpochControl { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch")), - } - } - - pub async fn resume_debug_epoch( - &self, - process: ProcessId, - epoch: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch }) - .await? - { - WireResponse::DebugEpoch { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - } => Ok(DebugEpochControl { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch")), - } - } - - pub async fn inspect_debug_epoch( - &self, - process: ProcessId, - epoch: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch }) - .await? - { - WireResponse::DebugEpochStatus { - process, - actor, - epoch, - command, - expected_tasks, - acknowledgements, - fully_frozen, - partially_frozen, - fully_resumed, - failed, - failure_messages, - audit_event, - } => Ok(DebugEpochStatus { - process, - actor, - epoch, - command, - expected_tasks, - acknowledgements, - fully_frozen, - partially_frozen, - fully_resumed, - failed, - failure_messages, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch_status")), - } - } - - pub async fn list_artifacts( - &self, - process: Option, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListArtifacts { - process, - cursor, - limit, - }) - .await? - { - WireResponse::Artifacts { - artifacts, - next_cursor, - } => Ok(ArtifactPage { - artifacts, - next_cursor, - }), - _ => Err(unexpected_response("artifacts")), - } - } - - pub async fn get_artifact(&self, artifact: ArtifactId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::GetArtifact { artifact }) - .await? - { - WireResponse::Artifact { artifact } => Ok(artifact), - _ => Err(unexpected_response("artifact")), - } - } - - pub async fn begin_artifact_download( - &self, - artifact: ArtifactId, - max_bytes: u64, - ttl_seconds: u64, - chunk_bytes: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink { - artifact: artifact.clone(), - max_bytes, - ttl_seconds, - }) - .await? - { - WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload { - client: self.clone(), - artifact, - max_bytes, - chunk_bytes, - token_digest: link.scoped_token_digest.clone(), - link, - expected_offset: 0, - finished: false, - }), - _ => Err(unexpected_response("artifact_download_link")), - } - } - - async fn send_authenticated( - &self, - request: AuthenticatedRequest, - ) -> Result { - let session_secret = self - .session_secret - .lock() - .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))? - .clone() - .ok_or_else(|| { - ClientError::Api(ApiError::from_message( - "client", - "no authenticated Clusterflux session is configured", - )) - })?; - self.send( - CONTROL_API_PATH, - json!({ - "type": "authenticated", - "session_secret": session_secret, - "request": request, - }), - ) - .await - } - - async fn send_login(&self, request: LoginRequest) -> Result { - let payload = serde_json::to_value(request) - .map_err(|error| ClientError::Protocol(error.to_string()))?; - self.send(LOGIN_API_PATH, payload).await - } - - async fn send( - &self, - api_path: &str, - payload: serde_json::Value, - ) -> Result { - let request_number = self.next_request.fetch_add(1, Ordering::Relaxed); - let request_id = format!("client-{request_number}"); - let envelope = coordinator_wire_request(&request_id, payload); - let body = serde_json::to_vec(&envelope) - .map_err(|error| ClientError::Protocol(error.to_string()))?; - let response = self - .transport - .send(TransportRequest { - api_path: api_path.to_owned(), - body, - }) - .await?; - let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| { - ClientError::Protocol(format!("decode typed API response: {error}")) - })?; - match response { - WireResponse::Error { - code, - category, - message, - retryable, - request_id: response_request_id, - } => { - if response_request_id != request_id { - return Err(ClientError::Protocol(format!( - "error response request_id {response_request_id} does not match {request_id}" - ))); - } - Err(ClientError::Api(ApiError::new( - code, - category, - message, - retryable, - response_request_id, - ))) - } - response => Ok(response), - } - } -} - -pub struct ArtifactDownload { - client: ClusterfluxClient, - artifact: ArtifactId, - max_bytes: u64, - chunk_bytes: u64, - token_digest: Digest, - link: clusterflux_core::DownloadLink, - expected_offset: u64, - finished: bool, -} - -impl ArtifactDownload { - pub fn link(&self) -> &clusterflux_core::DownloadLink { - &self.link - } - - pub async fn next_chunk(&mut self) -> Result { - if self.finished { - return Ok(ArtifactDownloadPoll::Chunk { - offset: self.expected_offset, - bytes: Vec::new(), - eof: true, - }); - } - match self - .client - .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream { - artifact: self.artifact.clone(), - max_bytes: self.max_bytes, - token_digest: self.token_digest.clone(), - chunk_bytes: self.chunk_bytes, - }) - .await? - { - WireResponse::ArtifactDownloadStream { - content_bytes_available, - content_offset, - content_eof, - content_base64, - .. - } => { - if !content_bytes_available { - return Ok(ArtifactDownloadPoll::Pending); - } - let offset = content_offset.ok_or_else(|| { - ClientError::Protocol( - "artifact response contained bytes without an offset".to_owned(), - ) - })?; - if offset != self.expected_offset { - return Err(ClientError::Protocol(format!( - "artifact chunk offset {offset} does not match expected {}", - self.expected_offset - ))); - } - let bytes = BASE64_STANDARD - .decode(content_base64.unwrap_or_default()) - .map_err(|error| { - ClientError::Protocol(format!( - "artifact response contains invalid base64: {error}" - )) - })?; - self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64); - self.finished = content_eof; - Ok(ArtifactDownloadPoll::Chunk { - offset, - bytes, - eof: content_eof, - }) - } - _ => Err(unexpected_response("artifact_download_stream")), - } - } - - pub async fn cancel(mut self) -> Result<(), ClientError> { - if self.finished { - return Ok(()); - } - match self - .client - .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink { - artifact: self.artifact.clone(), - token_digest: self.token_digest.clone(), - }) - .await? - { - WireResponse::ArtifactDownloadLinkRevoked {} => { - self.finished = true; - Ok(()) - } - _ => Err(unexpected_response("artifact_download_link_revoked")), - } - } -} - -fn unexpected_response(expected: &str) -> ClientError { - ClientError::Protocol(format!( - "expected typed response {expected}, received another response variant" - )) -} diff --git a/crates/clusterflux-client/src/protocol.rs b/crates/clusterflux-client/src/protocol.rs deleted file mode 100644 index a71ecff..0000000 --- a/crates/clusterflux-client/src/protocol.rs +++ /dev/null @@ -1,403 +0,0 @@ -use std::collections::BTreeMap; - -use clusterflux_core::{ - AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink, - LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId, -}; -use serde::{Deserialize, Serialize}; - -use crate::types::*; - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum AuthenticatedRequest { - AuthStatus, - RevokeCliSession, - CreateProject { - project: ProjectId, - name: String, - }, - SelectProject { - project: ProjectId, - }, - ListProjects, - RegisterAgentPublicKey { - agent: AgentId, - public_key: String, - }, - ListAgentPublicKeys, - RotateAgentPublicKey { - agent: AgentId, - public_key: String, - }, - RevokeAgentPublicKey { - agent: AgentId, - }, - CreateNodeEnrollmentGrant { - ttl_seconds: u64, - }, - ListNodeSummaries { - cursor: Option, - limit: u32, - }, - RevokeNodeCredential { - node: NodeId, - }, - ListProcessSummaries { - cursor: Option, - limit: u32, - }, - CancelProcess { - process: ProcessId, - }, - AbortProcess { - process: ProcessId, - launch_attempt: Option, - }, - QuotaStatus, - ListTaskEvents { - process: Option, - }, - ListTaskSnapshots { - process: ProcessId, - }, - ListRecentLogs { - process: ProcessId, - task: Option, - after_sequence: Option, - limit: u32, - }, - RestartTask { - process: ProcessId, - task: TaskInstanceId, - replacement_bundle: Option, - }, - ResolveTaskFailure { - process: ProcessId, - task: TaskInstanceId, - resolution: TaskFailureResolution, - }, - DebugAttach { - process: ProcessId, - }, - CreateDebugEpoch { - process: ProcessId, - stopped_task: TaskInstanceId, - reason: String, - }, - ResumeDebugEpoch { - process: ProcessId, - epoch: u64, - }, - InspectDebugEpoch { - process: ProcessId, - epoch: u64, - }, - ListArtifacts { - process: Option, - cursor: Option, - limit: u32, - }, - GetArtifact { - artifact: ArtifactId, - }, - CreateArtifactDownloadLink { - artifact: ArtifactId, - max_bytes: u64, - ttl_seconds: u64, - }, - OpenArtifactDownloadStream { - artifact: ArtifactId, - max_bytes: u64, - token_digest: Digest, - chunk_bytes: u64, - }, - RevokeArtifactDownloadLink { - artifact: ArtifactId, - token_digest: Digest, - }, -} - -#[derive(Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum LoginRequest { - BeginWebBrowserLogin {}, - CancelWebBrowserLogin { - transaction_id: String, - }, - ExchangeWebLoginHandoff { - transaction_id: String, - handoff_code: String, - }, -} - -#[derive(Clone, Deserialize)] -pub(crate) struct WireBrowserSession { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub session_secret: String, - pub expires_at_epoch_seconds: u64, -} - -#[allow(clippy::large_enum_variant)] -#[derive(Clone, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum WireResponse { - AuthStatus { - tenant: TenantId, - project: ProjectId, - actor: UserId, - authenticated: bool, - account_status: String, - suspended: bool, - disabled: bool, - deleted: bool, - manual_review: bool, - sanitized_reason: Option, - next_actions: Vec, - }, - CliSessionRevoked {}, - ProjectCreated { - project: Project, - }, - ProjectSelected { - project: Project, - }, - Projects { - projects: Vec, - }, - AgentPublicKey { - record: AgentPublicKey, - }, - AgentPublicKeys { - records: Vec, - }, - NodeEnrollmentGrantCreated { - tenant: TenantId, - project: ProjectId, - grant: String, - scope: String, - expires_at_epoch_seconds: u64, - }, - NodeSummaries { - nodes: Vec, - next_cursor: Option, - }, - NodeCredentialRevoked { - node: NodeId, - tenant: TenantId, - project: ProjectId, - actor: UserId, - descriptor_removed: bool, - queued_assignments_removed: usize, - }, - ProcessSummaries { - processes: Vec, - next_cursor: Option, - }, - ProcessCancellationRequested { - process: ProcessId, - cancelled_tasks: Vec, - affected_nodes: Vec, - }, - ProcessAborted { - process: ProcessId, - aborted_tasks: Vec, - affected_nodes: Vec, - }, - QuotaStatus { - tenant: TenantId, - project: ProjectId, - actor: UserId, - policy_label: Option, - limits: ResourceLimits, - window_seconds: BTreeMap, - usage: BTreeMap, - window_started_epoch_seconds: BTreeMap, - }, - TaskEvents { - events: Vec, - }, - TaskSnapshots { - snapshots: Vec, - }, - RecentLogs { - entries: Vec, - next_sequence: Option, - history_truncated: bool, - }, - TaskRestart { - process: ProcessId, - task: TaskInstanceId, - restarted_task_instance: Option, - restarted_attempt_id: Option, - actor: UserId, - accepted: bool, - clean_boundary_available: bool, - active_task: bool, - completed_event_observed: bool, - requires_whole_process_restart: bool, - message: String, - audit_event: DebugAuditEvent, - }, - TaskFailureResolved { - process: ProcessId, - task: TaskInstanceId, - attempt_id: String, - resolution: TaskFailureResolution, - }, - DebugAttach { - process: ProcessId, - actor: UserId, - authorization: Authorization, - audit_event: DebugAuditEvent, - }, - DebugEpoch { - process: ProcessId, - actor: UserId, - epoch: u64, - command: String, - affected_tasks: Vec, - all_stop_requested: bool, - audit_event: DebugAuditEvent, - }, - DebugEpochStatus { - process: ProcessId, - actor: UserId, - epoch: u64, - command: String, - expected_tasks: Vec, - acknowledgements: Vec, - fully_frozen: bool, - partially_frozen: bool, - fully_resumed: bool, - failed: bool, - failure_messages: Vec, - audit_event: DebugAuditEvent, - }, - Artifacts { - artifacts: Vec, - next_cursor: Option, - }, - Artifact { - artifact: ArtifactSummary, - }, - ArtifactDownloadLink { - link: DownloadLink, - }, - ArtifactDownloadLinkRevoked {}, - ArtifactDownloadStream { - #[serde(rename = "link")] - _link: DownloadLink, - content_bytes_available: bool, - content_offset: Option, - content_eof: bool, - content_base64: Option, - }, - WebBrowserLoginStarted { - transaction_id: String, - authorization_url: String, - expires_at_epoch_seconds: u64, - }, - WebBrowserLoginCancelled {}, - WebBrowserSession { - session: WireBrowserSession, - }, - Error { - code: ApiErrorCode, - category: ApiErrorCategory, - message: String, - retryable: bool, - request_id: String, - }, -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeSet; - - use serde::Deserialize; - use serde_json::Value; - - use super::*; - - #[derive(Deserialize)] - struct OperationFixture { - operation: String, - boundary: String, - request: Value, - response: Value, - } - - #[test] - fn website_operation_contract_fixtures_cover_every_typed_request() { - let fixtures: Vec = - serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap(); - let expected = BTreeSet::from([ - "abort_process", - "auth_status", - "begin_web_browser_login", - "cancel_web_browser_login", - "cancel_process", - "create_artifact_download_link", - "create_debug_epoch", - "create_node_enrollment_grant", - "create_project", - "debug_attach", - "exchange_web_login_handoff", - "get_artifact", - "inspect_debug_epoch", - "list_agent_public_keys", - "list_artifacts", - "list_node_summaries", - "list_process_summaries", - "list_projects", - "list_recent_logs", - "list_task_events", - "list_task_snapshots", - "open_artifact_download_stream", - "quota_status", - "register_agent_public_key", - "resolve_task_failure", - "restart_task", - "resume_debug_epoch", - "revoke_agent_public_key", - "revoke_artifact_download_link", - "revoke_cli_session", - "revoke_node_credential", - "rotate_agent_public_key", - "select_project", - ]) - .into_iter() - .map(str::to_owned) - .collect(); - let mut observed = BTreeSet::new(); - - for fixture in fixtures { - assert_eq!(fixture.request["type"], fixture.operation); - let round_trip = match fixture.boundary.as_str() { - "control" => { - let typed: AuthenticatedRequest = - serde_json::from_value(fixture.request.clone()).unwrap(); - serde_json::to_value(typed).unwrap() - } - "login" => { - let typed: LoginRequest = - serde_json::from_value(fixture.request.clone()).unwrap(); - serde_json::to_value(typed).unwrap() - } - boundary => panic!("unknown fixture boundary {boundary}"), - }; - assert_eq!(round_trip, fixture.request); - let response: WireResponse = serde_json::from_value(fixture.response).unwrap(); - assert!( - !matches!(response, WireResponse::Error { .. }), - "{} must carry its operation-specific success response fixture", - fixture.operation - ); - assert!(observed.insert(fixture.operation)); - } - assert_eq!(observed, expected); - } -} diff --git a/crates/clusterflux-client/src/transport.rs b/crates/clusterflux-client/src/transport.rs deleted file mode 100644 index b480c0f..0000000 --- a/crates/clusterflux-client/src/transport.rs +++ /dev/null @@ -1,250 +0,0 @@ -use std::collections::{BTreeMap, VecDeque}; -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use clusterflux_control::{endpoint_identity, ControlSession}; -use thiserror::Error; - -pub type TransportFuture = - Pin> + Send + 'static>>; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransportRequest { - pub api_path: String, - pub body: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransportResponse { - pub body: Vec, -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum ClientTransportError { - #[error("client transport failed: {0}")] - Failed(String), - #[error("client transport task failed: {0}")] - Task(String), -} - -pub trait ClientTransport: Send + Sync + 'static { - fn send(&self, request: TransportRequest) -> TransportFuture; -} - -type ControlSessionPool = Arc>>>; - -pub struct ControlTransport { - endpoint: String, - connect_timeout: Duration, - io_timeout: Duration, - sessions: Arc>>, - next_session: Arc, -} - -// A small pool allows independent HTMX reads to progress concurrently while -// keeping connection and blocking-worker use strictly bounded. -const SESSIONS_PER_API_PATH: usize = 4; - -impl ControlTransport { - pub fn new(endpoint: impl Into) -> Result { - Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30)) - } - - pub fn with_timeouts( - endpoint: impl Into, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result { - let endpoint = endpoint.into(); - endpoint_identity(&endpoint) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?; - Ok(Self { - endpoint, - connect_timeout, - io_timeout, - sessions: Arc::new(Mutex::new(BTreeMap::new())), - next_session: Arc::new(AtomicU64::new(0)), - }) - } -} - -impl ClientTransport for ControlTransport { - fn send(&self, request: TransportRequest) -> TransportFuture { - let endpoint = self.endpoint.clone(); - let connect_timeout = self.connect_timeout; - let io_timeout = self.io_timeout; - let sessions = Arc::clone(&self.sessions); - let next_session = Arc::clone(&self.next_session); - Box::pin(async move { - let pool = { - let mut sessions = sessions.lock().map_err(|_| { - ClientTransportError::Failed( - "client transport pool lock was poisoned".to_owned(), - ) - })?; - Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| { - Arc::new( - (0..SESSIONS_PER_API_PATH) - .map(|_| Mutex::new(None)) - .collect(), - ) - })) - }; - let slot_index = - next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH; - tokio::task::spawn_blocking(move || { - let value = serde_json::from_slice(&request.body) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?; - let mut selected = None; - for offset in 0..SESSIONS_PER_API_PATH { - let index = (slot_index + offset) % SESSIONS_PER_API_PATH; - match pool[index].try_lock() { - Ok(session) if session.is_some() => { - selected = Some(session); - break; - } - Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {} - Err(std::sync::TryLockError::Poisoned(_)) => { - return Err(ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - )); - } - } - } - if selected.is_none() { - for offset in 0..SESSIONS_PER_API_PATH { - let index = (slot_index + offset) % SESSIONS_PER_API_PATH; - match pool[index].try_lock() { - Ok(session) => { - selected = Some(session); - break; - } - Err(std::sync::TryLockError::WouldBlock) => {} - Err(std::sync::TryLockError::Poisoned(_)) => { - return Err(ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - )); - } - } - } - } - let mut session = match selected { - Some(session) => session, - None => pool[slot_index].lock().map_err(|_| { - ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - ) - })?, - }; - if session.is_none() { - *session = Some( - ControlSession::connect_to_api_path_with_timeouts( - &endpoint, - &request.api_path, - connect_timeout, - io_timeout, - ) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?, - ); - } - let response = session - .as_mut() - .expect("session was initialized for the requested API path") - .request(&value); - match response { - Ok(response) => serde_json::to_vec(&response) - .map(|body| TransportResponse { body }) - .map_err(|error| ClientTransportError::Failed(error.to_string())), - Err(error) => { - *session = None; - Err(ClientTransportError::Failed(error.to_string())) - } - } - }) - .await - .map_err(|error| ClientTransportError::Task(error.to_string()))? - }) - } -} - -#[derive(Clone, Default)] -pub struct MockTransport { - state: Arc>, -} - -#[derive(Default)] -struct MockTransportState { - responses: VecDeque, ClientTransportError>>, - requests: Vec, -} - -impl MockTransport { - pub fn from_json_responses(responses: impl IntoIterator>) -> Self { - let responses = responses - .into_iter() - .map(|response| Ok(response.into().into_bytes())) - .collect(); - Self { - state: Arc::new(Mutex::new(MockTransportState { - responses, - requests: Vec::new(), - })), - } - } - - pub fn push_json_response(&self, response: impl Into) { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .responses - .push_back(Ok(response.into().into_bytes())); - } - - pub fn push_error(&self, message: impl Into) { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .responses - .push_back(Err(ClientTransportError::Failed(message.into()))); - } - - pub fn request_bodies(&self) -> Vec { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .requests - .iter() - .map(|request| String::from_utf8_lossy(&request.body).into_owned()) - .collect() - } - - pub fn requests(&self) -> Vec { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .requests - .clone() - } -} - -impl ClientTransport for MockTransport { - fn send(&self, request: TransportRequest) -> TransportFuture { - let state = Arc::clone(&self.state); - Box::pin(async move { - let mut state = state.lock().map_err(|_| { - ClientTransportError::Failed("mock transport lock was poisoned".to_owned()) - })?; - state.requests.push(request); - state - .responses - .pop_front() - .ok_or_else(|| { - ClientTransportError::Failed("mock transport has no queued response".to_owned()) - })? - .map(|body| TransportResponse { body }) - }) - } -} diff --git a/crates/clusterflux-client/src/types.rs b/crates/clusterflux-client/src/types.rs deleted file mode 100644 index eb17e6e..0000000 --- a/crates/clusterflux-client/src/types.rs +++ /dev/null @@ -1,468 +0,0 @@ -use std::collections::BTreeMap; - -use clusterflux_core::{ - AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement, - ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy, - TaskInstanceId, TenantId, UserId, VfsPath, -}; -use serde::{Deserialize, Serialize}; -use std::fmt; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AccountStatus { - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub authenticated: bool, - pub account_status: String, - pub suspended: bool, - pub disabled: bool, - pub deleted: bool, - pub manual_review: bool, - pub sanitized_reason: Option, - pub next_actions: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Project { - pub id: ProjectId, - pub tenant: TenantId, - pub name: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AgentPublicKey { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub agent: AgentId, - pub public_key: String, - pub public_key_fingerprint: Digest, - pub version: u64, - pub revoked: bool, - pub scopes: Vec, - pub human_account_creation_privilege: bool, - pub browser_interaction_required_each_run: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeEnrollmentGrant { - pub tenant: TenantId, - pub project: ProjectId, - pub grant: String, - pub scope: String, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeSummary { - pub id: NodeId, - pub display_name: String, - pub online: bool, - pub stale: bool, - pub last_seen_epoch_seconds: Option, - pub capabilities: NodeCapabilities, - pub direct_connectivity: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodePage { - pub nodes: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeRevocation { - pub node: NodeId, - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub descriptor_removed: bool, - pub queued_assignments_removed: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessLifecycleState { - Active, - RecentTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessActivityState { - Running, - WaitingForNode, - WaitingForTask, - AwaitingAction, - DebugEpochPartial, - Cancelling, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessFinalResult { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochSummary { - pub epoch: u64, - pub command: String, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessSummary { - pub process: ProcessId, - pub lifecycle: ProcessLifecycleState, - pub activity: ProcessActivityState, - pub main_wait_state: Option, - pub started_at_epoch_seconds: u64, - pub ended_at_epoch_seconds: Option, - pub final_result: Option, - pub connected_nodes: Vec, - pub current_debug_epoch: Option, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessPage { - pub processes: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskTerminalState { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskExecutor { - CoordinatorMain, - Node, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskCompletionEvent { - pub tenant: TenantId, - pub project: ProjectId, - pub process: ProcessId, - pub node: NodeId, - pub executor: TaskExecutor, - pub task_definition: TaskDefinitionId, - pub task: TaskInstanceId, - pub attempt_id: Option, - pub placement: Option, - pub terminal_state: TaskTerminalState, - pub status_code: Option, - pub stdout_bytes: u64, - pub stderr_bytes: u64, - pub stdout_tail: String, - pub stderr_tail: String, - pub stdout_truncated: bool, - pub stderr_truncated: bool, - pub artifact_path: Option, - pub artifact_digest: Option, - pub artifact_size_bytes: Option, - pub result: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskAttemptState { - Queued, - Running, - FailedAwaitingAction, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskAttemptSnapshot { - pub process: ProcessId, - pub task: TaskInstanceId, - pub attempt_id: String, - pub attempt_number: u32, - pub task_definition: TaskDefinitionId, - pub display_name: String, - pub state: TaskAttemptState, - pub current: bool, - pub node: Option, - pub environment_id: Option, - pub environment_digest: Option, - pub argument_summary: Vec, - pub handle_summary: Vec, - pub command_state: Option, - pub vfs_checkpoint: String, - pub probe_symbol: Option, - pub source_path: Option, - pub source_line: Option, - pub restart_compatible: bool, - pub failure_policy: TaskFailurePolicy, - pub artifact_path: Option, - pub artifact_digest: Option, - pub artifact_size_bytes: Option, - pub status_code: Option, - pub error: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskLogStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogEntry { - pub sequence: u64, - pub process: ProcessId, - pub task: TaskInstanceId, - pub stream: TaskLogStream, - pub text: String, - pub server_timestamp_epoch_seconds: u64, - pub truncated: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogPage { - pub entries: Vec, - pub next_sequence: Option, - pub history_truncated: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactAvailability { - Available, - NodeOffline, - Unavailable, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactRetentionState { - NodeRetained, - ExplicitStorage, - Lost, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactSummary { - pub id: ArtifactId, - pub display_path: String, - pub display_name: String, - pub process: ProcessId, - pub producer_task: TaskInstanceId, - pub safe_node: Option, - pub digest: Digest, - pub size_bytes: u64, - pub availability: ArtifactAvailability, - pub downloadable_now: bool, - pub retention_state: ArtifactRetentionState, - pub explicit_storage: bool, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactPage { - pub artifacts: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct QuotaStatus { - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub policy_label: Option, - pub limits: ResourceLimits, - pub window_seconds: BTreeMap, - pub usage: BTreeMap, - pub window_started_epoch_seconds: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskCancellationTarget { - pub process: ProcessId, - pub task: TaskInstanceId, - pub node: NodeId, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessCancellation { - pub process: ProcessId, - pub affected_tasks: Vec, - pub affected_nodes: Vec, - pub aborted: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugAuditEvent { - pub tenant: TenantId, - pub project: ProjectId, - pub process: ProcessId, - pub task: Option, - pub actor: UserId, - pub operation: String, - pub allowed: bool, - pub reason: String, - pub charged_debug_read_bytes: u64, - pub used_debug_read_bytes: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugAttach { - pub process: ProcessId, - pub actor: UserId, - pub authorization: Authorization, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochControl { - pub process: ProcessId, - pub actor: UserId, - pub epoch: u64, - pub command: String, - pub affected_tasks: Vec, - pub all_stop_requested: bool, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DebugAcknowledgementState { - Frozen, - Running, - Failed, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugParticipantAcknowledgement { - pub node: NodeId, - pub task_definition: TaskDefinitionId, - pub task: TaskInstanceId, - pub epoch: u64, - pub state: DebugAcknowledgementState, - pub stack_frames: Vec, - pub local_values: Vec<(String, String)>, - pub task_args: Vec<(String, String)>, - pub handles: Vec<(String, String)>, - pub command_status: Option, - pub recent_output: Vec, - pub message: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochStatus { - pub process: ProcessId, - pub actor: UserId, - pub epoch: u64, - pub command: String, - pub expected_tasks: Vec, - pub acknowledgements: Vec, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, - pub failure_messages: Vec, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskRestart { - pub process: ProcessId, - pub task: TaskInstanceId, - pub restarted_task_instance: Option, - pub restarted_attempt_id: Option, - pub actor: UserId, - pub accepted: bool, - pub clean_boundary_available: bool, - pub active_task: bool, - pub completed_event_observed: bool, - pub requires_whole_process_restart: bool, - pub message: String, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskFailureResolutionResult { - pub process: ProcessId, - pub task: TaskInstanceId, - pub attempt_id: String, - pub resolution: TaskFailureResolution, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskFailureResolution { - AcceptFailure, - Cancel, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskReplacementBundle { - pub bundle_digest: Digest, - pub wasm_module_base64: String, - pub source_snapshot: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct BrowserLoginStart { - pub transaction_id: String, - pub authorization_url: String, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, PartialEq, Eq)] -pub struct SessionCredential(pub(crate) String); - -impl SessionCredential { - pub fn from_secret(secret: impl Into) -> Self { - Self(secret.into()) - } - - pub fn expose_secret(&self) -> &str { - &self.0 - } -} - -impl fmt::Debug for SessionCredential { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("SessionCredential([REDACTED])") - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct BrowserSession { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub credential: SessionCredential, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ArtifactDownloadPoll { - Pending, - Chunk { - offset: u64, - bytes: Vec, - eof: bool, - }, -} diff --git a/crates/clusterflux-client/tests/client_contract.rs b/crates/clusterflux-client/tests/client_contract.rs deleted file mode 100644 index 8851a54..0000000 --- a/crates/clusterflux-client/tests/client_contract.rs +++ /dev/null @@ -1,254 +0,0 @@ -use std::net::TcpListener; -use std::time::Duration; - -use clusterflux_client::{ - ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError, - ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId, - UserId, CLIENT_API_VERSION, -}; -use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; -use clusterflux_coordinator::service::CoordinatorService; -use serde_json::{json, Value}; - -#[tokio::test] -async fn mock_transport_exercises_typed_envelope_and_session_plumbing() { - let transport = MockTransport::from_json_responses([json!({ - "type": "projects", - "projects": [{ - "id": "project-one", - "tenant": "tenant-one", - "name": "Project one" - }], - "actor": "user-one" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport.clone()) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let projects = client.list_projects().await.unwrap(); - assert_eq!(projects.len(), 1); - assert_eq!(projects[0].id, ProjectId::from("project-one")); - - let requests = transport.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].api_path, CONTROL_API_PATH); - let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION); - assert_eq!(envelope["request_id"], "client-1"); - assert_eq!(envelope["operation"], "authenticated"); - assert_eq!(envelope["payload"]["type"], "authenticated"); - assert_eq!(envelope["payload"]["request"]["type"], "list_projects"); - assert_eq!(envelope["payload"]["session_secret"], "test-session-secret"); -} - -#[tokio::test] -async fn structured_errors_retain_machine_fields_and_originating_request_id() { - let transport = MockTransport::from_json_responses([json!({ - "type": "error", - "code": "account_suspended", - "category": "authorization", - "message": "account access is suspended", - "retryable": false, - "request_id": "client-1" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let ClientError::Api(error) = client.account_status().await.unwrap_err() else { - panic!("expected typed API error"); - }; - assert_eq!(error.code, ApiErrorCode::AccountSuspended); - assert_eq!(error.category, ApiErrorCategory::Authorization); - assert_eq!(error.request_id, "client-1"); - assert!(!error.retryable); -} - -#[tokio::test] -async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() { - let transport = MockTransport::from_json_responses([ - json!({ "type": "web_browser_login_cancelled" }).to_string(), - ]); - let client = ClusterfluxClient::with_transport(transport.clone()); - - client - .cancel_browser_login("login-transaction") - .await - .unwrap(); - - let requests = transport.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].api_path, LOGIN_API_PATH); - let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert_eq!(envelope["operation"], "cancel_web_browser_login"); - assert_eq!( - envelope["payload"], - json!({ - "type": "cancel_web_browser_login", - "transaction_id": "login-transaction" - }) - ); -} - -#[tokio::test] -async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() { - let transport = MockTransport::from_json_responses([json!({ - "type": "error", - "code": "validation_error", - "category": "validation", - "message": "bad request", - "retryable": false, - "request_id": "another-request" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else { - panic!("expected protocol error"); - }; - assert!(message.contains("does not match client-1")); -} - -#[test] -fn session_credentials_are_redacted_from_debug_output() { - let credential = SessionCredential::from_secret("must-not-appear"); - assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])"); -} - -#[tokio::test] -async fn artifact_download_is_a_typed_bounded_chunk_stream() { - let link = json!({ - "artifact": "artifact-one", - "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "artifact_size_bytes": 5, - "source": { "RetainedNode": "node-one" }, - "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", - "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "expires_at_epoch_seconds": 1000, - "tenant": "tenant-one", - "project": "project-one", - "process": "process-one", - "actor": { "User": "user-one" }, - "max_bytes": 1024, - "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - }); - let transport = MockTransport::from_json_responses([ - json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(), - json!({ - "type": "artifact_download_stream", - "link": link.clone(), - "streamed_bytes": 0, - "charged_download_bytes": 0, - "content_bytes_available": false, - "content_eof": false - }) - .to_string(), - json!({ - "type": "artifact_download_stream", - "link": link, - "streamed_bytes": 5, - "charged_download_bytes": 5, - "content_bytes_available": true, - "content_offset": 0, - "content_eof": true, - "content_base64": "aGVsbG8=" - }) - .to_string(), - ]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - let mut download = client - .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64) - .await - .unwrap(); - assert_eq!( - download.next_chunk().await.unwrap(), - ArtifactDownloadPoll::Pending - ); - assert_eq!( - download.next_chunk().await.unwrap(), - ArtifactDownloadPoll::Chunk { - offset: 0, - bytes: b"hello".to_vec(), - eof: true, - } - ); -} - -#[tokio::test] -async fn typed_client_runs_against_the_real_strict_control_endpoint() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant-one"), - ProjectId::from("project-one"), - UserId::from("user-one"), - "real-endpoint-session", - None, - ) - .unwrap(); - let (stream, _) = listener.accept().unwrap(); - service.handle_stream(stream).unwrap(); - }); - - let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}")) - .unwrap() - .with_session_credential(&SessionCredential::from_secret("real-endpoint-session")); - let projects = client.list_projects().await.unwrap(); - assert_eq!(projects.len(), 1); - assert_eq!(projects[0].id, ProjectId::from("project-one")); - let status = client.account_status().await.unwrap(); - assert!(status.authenticated); - assert_eq!(status.actor, UserId::from("user-one")); - - drop(client); - server.join().unwrap(); -} - -#[tokio::test] -async fn concurrent_requests_expand_the_bounded_pool_without_serializing() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let handlers = (0..2) - .map(|_| { - let (stream, _) = listener.accept().unwrap(); - std::thread::spawn(move || { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant-one"), - ProjectId::from("project-one"), - UserId::from("user-one"), - "concurrent-session", - None, - ) - .unwrap(); - service.handle_stream(stream).unwrap(); - }) - }) - .collect::>(); - for handler in handlers { - handler.join().unwrap(); - } - }); - - let transport = ControlTransport::with_timeouts( - format!("clusterflux+tcp://{address}"), - Duration::from_secs(2), - Duration::from_secs(5), - ) - .unwrap(); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("concurrent-session")); - let (first, second) = tokio::join!(client.account_status(), client.account_status()); - assert!(first.unwrap().authenticated); - assert!(second.unwrap().authenticated); - - drop(client); - server.join().unwrap(); -} diff --git a/crates/clusterflux-client/tests/fixtures/web_operations.json b/crates/clusterflux-client/tests/fixtures/web_operations.json deleted file mode 100644 index 45b7b23..0000000 --- a/crates/clusterflux-client/tests/fixtures/web_operations.json +++ /dev/null @@ -1,200 +0,0 @@ -[ - { - "operation": "begin_web_browser_login", - "boundary": "login", - "request": { "type": "begin_web_browser_login" }, - "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 } - }, - { - "operation": "cancel_web_browser_login", - "boundary": "login", - "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" }, - "response": { "type": "web_browser_login_cancelled" } - }, - { - "operation": "exchange_web_login_handoff", - "boundary": "login", - "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" }, - "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } } - }, - { - "operation": "auth_status", - "boundary": "control", - "request": { "type": "auth_status" }, - "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] } - }, - { - "operation": "revoke_cli_session", - "boundary": "control", - "request": { "type": "revoke_cli_session" }, - "response": { "type": "cli_session_revoked" } - }, - { - "operation": "create_project", - "boundary": "control", - "request": { "type": "create_project", "project": "project-new", "name": "New project" }, - "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } } - }, - { - "operation": "select_project", - "boundary": "control", - "request": { "type": "select_project", "project": "project-selected" }, - "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } } - }, - { - "operation": "list_projects", - "boundary": "control", - "request": { "type": "list_projects" }, - "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] } - }, - { - "operation": "register_agent_public_key", - "boundary": "control", - "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "list_agent_public_keys", - "boundary": "control", - "request": { "type": "list_agent_public_keys" }, - "response": { "type": "agent_public_keys", "records": [] } - }, - { - "operation": "rotate_agent_public_key", - "boundary": "control", - "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "revoke_agent_public_key", - "boundary": "control", - "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "create_node_enrollment_grant", - "boundary": "control", - "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 }, - "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 } - }, - { - "operation": "list_node_summaries", - "boundary": "control", - "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 }, - "response": { "type": "node_summaries", "nodes": [], "next_cursor": null } - }, - { - "operation": "revoke_node_credential", - "boundary": "control", - "request": { "type": "revoke_node_credential", "node": "node-one" }, - "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 } - }, - { - "operation": "list_process_summaries", - "boundary": "control", - "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 }, - "response": { "type": "process_summaries", "processes": [], "next_cursor": null } - }, - { - "operation": "cancel_process", - "boundary": "control", - "request": { "type": "cancel_process", "process": "process-one" }, - "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] } - }, - { - "operation": "abort_process", - "boundary": "control", - "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null }, - "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] } - }, - { - "operation": "quota_status", - "boundary": "control", - "request": { "type": "quota_status" }, - "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } } - }, - { - "operation": "list_task_events", - "boundary": "control", - "request": { "type": "list_task_events", "process": "process-one" }, - "response": { "type": "task_events", "events": [] } - }, - { - "operation": "list_task_snapshots", - "boundary": "control", - "request": { "type": "list_task_snapshots", "process": "process-one" }, - "response": { "type": "task_snapshots", "snapshots": [] } - }, - { - "operation": "list_recent_logs", - "boundary": "control", - "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 }, - "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false } - }, - { - "operation": "restart_task", - "boundary": "control", - "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null }, - "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "resolve_task_failure", - "boundary": "control", - "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" }, - "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" } - }, - { - "operation": "debug_attach", - "boundary": "control", - "request": { "type": "debug_attach", "process": "process-one" }, - "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "create_debug_epoch", - "boundary": "control", - "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" }, - "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "resume_debug_epoch", - "boundary": "control", - "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 }, - "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "inspect_debug_epoch", - "boundary": "control", - "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 }, - "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "list_artifacts", - "boundary": "control", - "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 }, - "response": { "type": "artifacts", "artifacts": [], "next_cursor": null } - }, - { - "operation": "get_artifact", - "boundary": "control", - "request": { "type": "get_artifact", "artifact": "artifact-one" }, - "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } } - }, - { - "operation": "create_artifact_download_link", - "boundary": "control", - "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 }, - "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } } - }, - { - "operation": "open_artifact_download_stream", - "boundary": "control", - "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 }, - "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" } - }, - { - "operation": "revoke_artifact_download_link", - "boundary": "control", - "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, - "response": { "type": "artifact_download_link_revoked" } - } -] diff --git a/crates/clusterflux-control/src/lib.rs b/crates/clusterflux-control/src/lib.rs index 39a150d..43be04b 100644 --- a/crates/clusterflux-control/src/lib.rs +++ b/crates/clusterflux-control/src/lib.rs @@ -61,34 +61,12 @@ impl ControlSession { endpoint: &str, api_path: &str, ) -> Result { - Self::connect_to_api_path_with_timeouts( - endpoint, - api_path, - Duration::from_secs(10), - Duration::from_secs(30), - ) - } - - pub fn connect_to_api_path_with_timeouts( - endpoint: &str, - api_path: &str, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result { - let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?; + let mut session = Self::connect(endpoint)?; #[cfg(not(target_arch = "wasm32"))] - { - let mut session = session; - if let ControlTransport::Https { url, .. } = &mut session.transport { - *url = endpoint_api_url(endpoint, api_path)?; - } - Ok(session) - } - #[cfg(target_arch = "wasm32")] - { - let _ = api_path; - Ok(session) + if let ControlTransport::Https { url, .. } = &mut session.transport { + *url = endpoint_api_url(endpoint, api_path)?; } + Ok(session) } pub fn connect_with_timeouts( diff --git a/crates/clusterflux-coordinator/src/durable.rs b/crates/clusterflux-coordinator/src/durable.rs index 0cc8a66..cd3e3e5 100644 --- a/crates/clusterflux-coordinator/src/durable.rs +++ b/crates/clusterflux-coordinator/src/durable.rs @@ -33,39 +33,6 @@ pub struct NodeIdentityRecord { pub enrollment_scope: String, } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub struct NodeScopeKey { - pub tenant: TenantId, - pub project: ProjectId, - pub node: NodeId, -} - -impl NodeScopeKey { - pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self { - Self { - tenant, - project, - node, - } - } - - pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self { - Self::new(tenant.clone(), project.clone(), node.clone()) - } - - pub fn credential_subject(&self) -> String { - format!( - "node:{}:{}:{}:{}:{}:{}", - self.tenant.as_str().len(), - self.tenant, - self.project.as_str().len(), - self.project, - self.node.as_str().len(), - self.node - ) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CredentialRecord { pub subject: String, @@ -140,7 +107,7 @@ pub struct DurableState { pub tenants: BTreeMap, pub users: BTreeMap, pub projects: BTreeMap, - pub node_identities: BTreeMap, + pub node_identities: BTreeMap, pub credentials: BTreeMap, pub cli_sessions: BTreeMap, pub source_provider_configs: diff --git a/crates/clusterflux-coordinator/src/lib.rs b/crates/clusterflux-coordinator/src/lib.rs index a110834..c0252cd 100644 --- a/crates/clusterflux-coordinator/src/lib.rs +++ b/crates/clusterflux-coordinator/src/lib.rs @@ -13,7 +13,7 @@ pub mod service; mod sessions; pub use durable::{ AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState, - DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey, + DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; @@ -128,9 +128,8 @@ impl Coordinator { public_key: impl Into, enrollment_scope: impl Into, ) { - let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone()); self.durable.node_identities.insert( - key, + node.clone(), NodeIdentityRecord { id: node, tenant, @@ -182,13 +181,10 @@ impl Coordinator { public_key, credential.scope.clone(), ); - let subject = - NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node) - .credential_subject(); self.durable.credentials.insert( - subject.clone(), + format!("node:{node}"), CredentialRecord { - subject, + subject: format!("node:{node}"), tenant: credential.tenant.clone(), project: Some(credential.project.clone()), kind: credential.credential_kind.clone(), @@ -360,12 +356,15 @@ impl Coordinator { project: &ProjectId, process: &ProcessId, ) -> Result<(), CoordinatorError> { - if !self + let identity = self .durable .node_identities - .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) - { - return Err(CoordinatorError::UnknownNode); + .get(node) + .ok_or(CoordinatorError::UnknownNode)?; + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node identity is outside the requested tenant/project scope".to_owned(), + )); } if !self .active_processes @@ -380,18 +379,14 @@ impl Coordinator { pub fn reconnect_node( &mut self, - tenant: &TenantId, - project: &ProjectId, node: &NodeId, process: Option<(&ProcessId, u64)>, ) -> Result<(), CoordinatorError> { - if !self + let identity = self .durable .node_identities - .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) - { - return Err(CoordinatorError::UnknownNode); - } + .get(node) + .ok_or(CoordinatorError::UnknownNode)?; if let Some((process_id, stale_epoch)) = process { if stale_epoch != self.coordinator_epoch { @@ -400,7 +395,11 @@ impl Coordinator { current_epoch: self.coordinator_epoch, }); } - let key = (tenant.clone(), project.clone(), process_id.clone()); + let key = ( + identity.tenant.clone(), + identity.project.clone(), + process_id.clone(), + ); if let Some(active) = self.active_processes.get_mut(&key) { active.connected_nodes.insert(node.clone()); } @@ -417,27 +416,23 @@ impl Coordinator { let identity = self .durable .node_identities - .get(&NodeScopeKey::from_refs( - &context.tenant, - &context.project, - node, - )) + .get(node) .ok_or(CoordinatorError::UnknownNode)? .clone(); + if identity.tenant != context.tenant || identity.project != context.project { + return Err(CoordinatorError::Unauthorized( + "node credential is outside the signed-in tenant/project scope".to_owned(), + )); + } if !matches!(context.actor, Actor::User(_)) { return Err(CoordinatorError::Unauthorized( "node credential revocation requires a user identity".to_owned(), )); } - let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node); - self.durable.node_identities.remove(&key); - self.durable.credentials.remove(&key.credential_subject()); - for active in self - .active_processes - .values_mut() - .filter(|active| active.tenant == context.tenant && active.project == context.project) - { - active.connected_nodes.remove(&key.node); + self.durable.node_identities.remove(node); + self.durable.credentials.remove(&format!("node:{node}")); + for active in self.active_processes.values_mut() { + active.connected_nodes.remove(node); } Ok(identity) } @@ -585,31 +580,8 @@ impl Coordinator { .count() } - pub fn tenant_count(&self) -> usize { - self.durable.tenants.len() - } - - pub fn user_count(&self) -> usize { - self.durable.users.len() - } - - pub fn project_count(&self) -> usize { - self.durable.projects.len() - } - - pub fn node_identity_count(&self) -> usize { - self.durable.node_identities.len() - } - - pub fn node_identity( - &self, - tenant: &TenantId, - project: &ProjectId, - id: &NodeId, - ) -> Option<&NodeIdentityRecord> { - self.durable - .node_identities - .get(&NodeScopeKey::from_refs(tenant, project, id)) + pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> { + self.durable.node_identities.get(id) } pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize { @@ -711,24 +683,12 @@ mod tests { .contains_key(&TenantId::from("tenant"))); assert!(restarted.durable.users.contains_key(&UserId::from("user"))); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); - let node_subject = NodeScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - NodeId::from("node"), - ) - .credential_subject(); + assert!(restarted.node_identity(&NodeId::from("node")).is_some()); assert_eq!( restarted .durable .credentials - .get(&node_subject) + .get("node:node") .map(|credential| &credential.kind), Some(&CredentialKind::NodeCredential) ); @@ -752,12 +712,7 @@ mod tests { ); assert_eq!(rerun.coordinator_epoch, 2); restarted - .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - Some((&process, 2)), - ) + .reconnect_node(&NodeId::from("node"), Some((&process, 2))) .unwrap(); assert!(restarted .active_process( @@ -770,68 +725,6 @@ mod tests { .contains(&NodeId::from("node"))); } - #[test] - fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() { - let store = InMemoryDurableStore::default(); - let mut coordinator = Coordinator::boot(&store, 1); - let node = NodeId::from("shared-node"); - let scopes = [ - (TenantId::from("tenant-a"), ProjectId::from("project-a")), - (TenantId::from("tenant-b"), ProjectId::from("project-b")), - (TenantId::from("tenant-a"), ProjectId::from("project-c")), - ]; - for (index, (tenant, project)) in scopes.iter().enumerate() { - let mut grant = coordinator.create_node_enrollment_grant( - tenant.clone(), - project.clone(), - format!("grant-{index}"), - "node:attach", - 100, - ); - coordinator - .exchange_node_enrollment_grant( - &mut grant, - node.clone(), - &format!("public-key-{index}"), - "node:attach", - 99, - ) - .unwrap(); - } - - let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node); - let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node); - let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node); - assert_ne!(scope_a.credential_subject(), scope_b.credential_subject()); - assert_ne!(scope_a.credential_subject(), scope_c.credential_subject()); - assert_eq!( - coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node), - coordinator.durable.node_identities.get(&scope_a) - ); - assert_eq!( - coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node), - coordinator.durable.node_identities.get(&scope_b) - ); - assert_eq!( - coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node), - coordinator.durable.node_identities.get(&scope_c) - ); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_a.credential_subject())); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_b.credential_subject())); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_c.credential_subject())); - assert_eq!(coordinator.durable.node_identities.len(), 3); - assert_eq!(coordinator.durable.credentials.len(), 3); - } - #[test] fn identical_process_ids_are_isolated_by_tenant_and_project() { let store = InMemoryDurableStore::default(); @@ -880,18 +773,11 @@ mod tests { let mut restarted = Coordinator::boot(&store, 2); restarted - .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - None, - ) + .reconnect_node(&NodeId::from("node"), None) .unwrap(); let error = restarted .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -923,13 +809,7 @@ mod tests { .unwrap(); assert_eq!(credential.credential_kind, CredentialKind::NodeCredential); - assert!(coordinator - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); + assert!(coordinator.node_identity(&NodeId::from("node")).is_some()); } #[test] @@ -943,16 +823,10 @@ mod tests { "public-key", "node:attach", ); - let node_subject = NodeScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - NodeId::from("node"), - ) - .credential_subject(); coordinator.durable.credentials.insert( - node_subject.clone(), + "node:node".to_owned(), CredentialRecord { - subject: node_subject.clone(), + subject: "node:node".to_owned(), tenant: TenantId::from("tenant"), project: Some(ProjectId::from("project")), kind: CredentialKind::NodeCredential, @@ -966,8 +840,6 @@ mod tests { ); coordinator .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -983,7 +855,7 @@ mod tests { &NodeId::from("node"), ) .unwrap_err(); - assert!(matches!(foreign, CoordinatorError::UnknownNode)); + assert!(matches!(foreign, CoordinatorError::Unauthorized(_))); let revoked = coordinator .revoke_node_credential( @@ -996,14 +868,8 @@ mod tests { ) .unwrap(); assert_eq!(revoked.id, NodeId::from("node")); - assert!(coordinator - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_none()); - assert!(!coordinator.durable.credentials.contains_key(&node_subject)); + assert!(coordinator.node_identity(&NodeId::from("node")).is_none()); + assert!(!coordinator.durable.credentials.contains_key("node:node")); assert!(!coordinator .active_process( &TenantId::from("tenant"), @@ -1062,7 +928,7 @@ mod tests { } #[test] - fn account_policy_state_summarizes_sensitive_admin_records_safely() { + fn account_policy_state_summarizes_private_admin_records_safely() { let tenant = TenantId::from("tenant"); let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1); @@ -1218,7 +1084,7 @@ mod tests { ) .unwrap_err(); - assert!(matches!(error, CoordinatorError::UnknownNode)); + assert!(matches!(error, CoordinatorError::Unauthorized(_))); } #[test] diff --git a/crates/clusterflux-coordinator/src/main.rs b/crates/clusterflux-coordinator/src/main.rs index 8344f5f..8c9ce7d 100644 --- a/crates/clusterflux-coordinator/src/main.rs +++ b/crates/clusterflux-coordinator/src/main.rs @@ -5,40 +5,17 @@ use clusterflux_core::{ProjectId, TenantId, UserId}; use serde_json::json; fn main() -> Result<(), Box> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux coordinator.\n\n\ - Usage: clusterflux-coordinator [OPTIONS]\n\n\ - Options:\n \ - --listen
[default: 127.0.0.1:0]\n \ - --allow-local-trusted-loopback\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - _ => {} - } - let mut listen = "127.0.0.1:0".to_owned(); let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK") .ok() .as_deref() == Some("1"); - let mut args = raw_args.into_iter(); + let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { if arg == "--listen" { listen = args.next().ok_or("--listen requires an address")?; } else if arg == "--allow-local-trusted-loopback" { allow_local_trusted = true; - } else { - return Err(format!("unknown argument: {arg}").into()); } } diff --git a/crates/clusterflux-coordinator/src/postgres_store.rs b/crates/clusterflux-coordinator/src/postgres_store.rs index 18d5206..4ddfd02 100644 --- a/crates/clusterflux-coordinator/src/postgres_store.rs +++ b/crates/clusterflux-coordinator/src/postgres_store.rs @@ -4,8 +4,8 @@ use thiserror::Error; use crate::{ AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord, - DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord, - ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, + DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, + ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; #[derive(Clone, Debug, PartialEq, Eq)] @@ -154,16 +154,9 @@ impl FallibleDurableStore for PostgresDurableStore { state.projects.insert(record.id.clone(), record); } for record in self.query_records::( - "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id", + "SELECT record FROM clusterflux_node_identities ORDER BY node_id", )? { - state.node_identities.insert( - NodeScopeKey::new( - record.tenant.clone(), - record.project.clone(), - record.id.clone(), - ), - record, - ); + state.node_identities.insert(record.id.clone(), record); } for record in self.query_records::( "SELECT record FROM clusterflux_credentials ORDER BY subject", @@ -383,59 +376,12 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects ( ); CREATE TABLE IF NOT EXISTS clusterflux_node_identities ( + node_id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE, - node_id TEXT NOT NULL, - record JSONB NOT NULL, - PRIMARY KEY (tenant_id, project_id, node_id) + record JSONB NOT NULL ); -DO $clusterflux_node_scope_migration$ -DECLARE - current_primary_key TEXT; - current_primary_key_columns TEXT[]; -BEGIN - IF EXISTS ( - SELECT 1 - FROM clusterflux_node_identities - WHERE record->>'id' IS DISTINCT FROM node_id - OR record->>'tenant' IS DISTINCT FROM tenant_id - OR record->>'project' IS DISTINCT FROM project_id - ) THEN - RAISE EXCEPTION - 'clusterflux node identity migration refused an internally inconsistent legacy row'; - END IF; - - SELECT - constraint_record.conname, - array_agg(attribute.attname ORDER BY key_column.ordinality) - INTO current_primary_key, current_primary_key_columns - FROM pg_constraint AS constraint_record - CROSS JOIN LATERAL unnest(constraint_record.conkey) - WITH ORDINALITY AS key_column(attribute_number, ordinality) - JOIN pg_attribute AS attribute - ON attribute.attrelid = constraint_record.conrelid - AND attribute.attnum = key_column.attribute_number - WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass - AND constraint_record.contype = 'p' - GROUP BY constraint_record.conname; - - IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN - EXECUTE format( - 'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I', - current_primary_key - ); - ALTER TABLE clusterflux_node_identities - ADD PRIMARY KEY (tenant_id, project_id, node_id); - ELSIF current_primary_key_columns - IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[] - THEN - RAISE EXCEPTION - 'clusterflux node identity migration found an unexpected primary key shape'; - END IF; -END -$clusterflux_node_scope_migration$; - CREATE TABLE IF NOT EXISTS clusterflux_credentials ( subject TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -443,74 +389,6 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials ( record JSONB NOT NULL ); -DO $clusterflux_node_credential_migration$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM clusterflux_credentials - WHERE record->>'subject' IS DISTINCT FROM subject - OR record->>'tenant' IS DISTINCT FROM tenant_id - OR record->>'project' IS DISTINCT FROM project_id - ) THEN - RAISE EXCEPTION - 'clusterflux node credential migration refused an internally inconsistent legacy row'; - END IF; - - UPDATE clusterflux_credentials AS credential - SET subject = format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - ), - record = jsonb_set( - credential.record, - '{subject}', - to_jsonb(format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - )), - false - ) - FROM clusterflux_node_identities AS identity - WHERE credential.subject = 'node:' || identity.node_id - AND credential.tenant_id = identity.tenant_id - AND credential.project_id = identity.project_id; - - IF EXISTS ( - SELECT 1 - FROM clusterflux_credentials AS credential - WHERE credential.subject LIKE 'node:%' - AND NOT EXISTS ( - SELECT 1 - FROM clusterflux_node_identities AS identity - WHERE credential.tenant_id = identity.tenant_id - AND credential.project_id = identity.project_id - AND credential.subject = format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - ) - ) - ) THEN - RAISE EXCEPTION - 'clusterflux node credential migration found an unscoped or orphaned node subject'; - END IF; -END -$clusterflux_node_credential_migration$; - CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions ( session_digest TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -651,13 +529,7 @@ mod tests { let restarted = Coordinator::try_boot(&mut store, 2).unwrap(); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); + assert!(restarted.node_identity(&NodeId::from("node")).is_some()); assert_eq!(restarted.active_process_count(), 0); } diff --git a/crates/clusterflux-coordinator/src/service.rs b/crates/clusterflux-coordinator/src/service.rs index bab6c13..abef88d 100644 --- a/crates/clusterflux-coordinator/src/service.rs +++ b/crates/clusterflux-coordinator/src/service.rs @@ -7,14 +7,13 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::time::{SystemTime, UNIX_EPOCH}; use clusterflux_core::{ - Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry, - CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport, - NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit, - TenantId, TransportError, UserId, + Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError, + NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId, + RateLimit, TenantId, TransportError, UserId, }; use thiserror::Error; -use crate::{Coordinator, CoordinatorError, NodeScopeKey}; +use crate::{Coordinator, CoordinatorError}; mod admin; mod artifacts; @@ -35,7 +34,6 @@ mod quota; mod relay; mod routing; mod signed_nodes; -mod summaries; mod tcp; mod wire_protocol; use authorization::authorize_authenticated_user_operation; @@ -45,14 +43,12 @@ use keys::{ ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey, }; pub use protocol::{ - ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment, - AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, - DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement, - NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, - RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment, - TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent, - TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState, - VirtualProcessStatus, WorkflowActor, + ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest, + CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent, + DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, + TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, + TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle, + TaskTerminalState, VirtualProcessStatus, WorkflowActor, }; pub use quota::CoordinatorQuotaConfiguration; pub use relay::{ @@ -73,15 +69,9 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128; const MAX_TASK_EVENTS_TOTAL: usize = 8_192; const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192; const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096; -const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000; +const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096; const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256; const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024; -const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256; -const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024; -const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024; -const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024; -const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32; -const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192; const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30; fn bounded_ttl(requested: u64, maximum: u64) -> u64 { requested.clamp(1, maximum) @@ -121,24 +111,6 @@ pub struct CoordinatorAdmission { pub max_artifact_download_ttl_seconds: u64, } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CoordinatorOperationalMetrics { - pub tenants: usize, - pub users: usize, - pub projects: usize, - pub enrolled_nodes: usize, - pub reported_nodes: usize, - pub live_nodes: usize, - pub active_processes: usize, - pub active_coordinator_mains: usize, - pub max_active_coordinator_mains: usize, - pub active_tasks: usize, - pub queued_tasks: usize, - pub artifacts: usize, - pub retained_download_links: usize, - pub relay: ArtifactRelayUsage, -} - impl Default for CoordinatorAdmission { fn default() -> Self { Self { @@ -179,139 +151,16 @@ pub enum CoordinatorServiceError { Durable(String), } -impl CoordinatorServiceError { - pub fn api_error(&self, request_id: impl Into) -> ApiError { - let request_id = request_id.into(); - let message = self.to_string(); - let (code, category, retryable) = match self { - Self::Io(_) => ( - ApiErrorCode::TemporaryCapacity, - ApiErrorCategory::Availability, - true, - ), - Self::Json(_) - | Self::CapabilityReport(_) - | Self::InvalidArtifactPath(_) - | Self::InvalidTaskLogTail(_) => ( - ApiErrorCode::ValidationError, - ApiErrorCategory::Validation, - false, - ), - Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => { - return ApiError::from_message(request_id, message); - } - Self::Coordinator(CoordinatorError::UnknownNode) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Coordinator(CoordinatorError::Enrollment(_)) => ( - ApiErrorCode::Unauthenticated, - ApiErrorCategory::Authentication, - false, - ), - Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => { - (ApiErrorCode::Conflict, ApiErrorCategory::State, true) - } - Self::Download(DownloadError::NotFound) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Download(DownloadError::Unavailable) - | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => ( - ApiErrorCode::ArtifactUnavailable, - ApiErrorCategory::Availability, - true, - ), - Self::Download(DownloadError::LimitExceeded { .. }) => ( - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCategory::Resource, - false, - ), - Self::Download(DownloadError::Unauthorized(_)) - | Self::Download(DownloadError::InvalidToken) - | Self::Download(DownloadError::Expired) - | Self::Download(DownloadError::Revoked) => ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ), - Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ), - Self::Scheduler(_) => ( - ApiErrorCode::NoCapableNode, - ApiErrorCategory::Availability, - true, - ), - Self::Transport(_) => ( - ApiErrorCode::NodeOffline, - ApiErrorCategory::Availability, - true, - ), - Self::Panel(PanelError::RateLimited) => ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ), - Self::Panel(PanelError::UnknownWidget(_)) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Panel(_) => ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ), - Self::Durable(_) => ( - ApiErrorCode::InternalError, - ApiErrorCategory::Internal, - true, - ), - }; - ApiError::new(code, category, message, retryable, request_id) - } -} - pub struct CoordinatorService { coordinator: Coordinator, store: RuntimeDurableStore, - node_descriptors: BTreeMap, - node_last_seen_epoch_seconds: BTreeMap, + node_descriptors: BTreeMap, + node_last_seen_epoch_seconds: BTreeMap, node_stale_after_seconds: u64, debug_freeze_timeout: std::time::Duration, enrollment_grants: BTreeMap, task_events: VecDeque, process_scope_history: VecDeque, - process_summaries: BTreeMap, - process_summary_order: VecDeque, - next_process_summary_order: u64, - task_terminal_states: BTreeMap, - recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque>, - recent_log_dropped_through: BTreeMap, - recent_log_accounted_bytes: BTreeMap< - ( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - ), - u64, - >, - recent_log_truncated_streams: BTreeSet<( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - )>, - recent_log_quota_truncated_streams: BTreeSet<( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - )>, - next_recent_log_sequence: u64, debug_audit_events: VecDeque, debug_epochs: BTreeMap, debug_epoch_runtime: BTreeMap, @@ -331,7 +180,7 @@ pub struct CoordinatorService { process_cancellations: BTreeSet, process_aborts: BTreeSet, agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>, - node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>, + node_replay_nonces: BTreeMap<(NodeId, String), u64>, panel_snapshots: BTreeMap, stopped_panels: BTreeSet, panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>, @@ -366,29 +215,6 @@ impl CoordinatorService { self.artifact_relay.usage() } - pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics { - CoordinatorOperationalMetrics { - tenants: self.coordinator.tenant_count(), - users: self.coordinator.user_count(), - projects: self.coordinator.project_count(), - enrolled_nodes: self.coordinator.node_identity_count(), - reported_nodes: self.node_descriptors.len(), - live_nodes: self - .node_descriptors - .keys() - .filter(|scope| self.node_is_live(scope)) - .count(), - active_processes: self.coordinator.active_process_count(), - active_coordinator_mains: self.main_runtime.active_main_count(), - max_active_coordinator_mains: self.main_runtime.max_active_mains(), - active_tasks: self.active_tasks.len(), - queued_tasks: self.pending_task_launches.len(), - artifacts: self.artifact_registry.artifact_count(), - retained_download_links: self.artifact_registry.retained_download_link_count(), - relay: self.artifact_relay.usage(), - } - } - fn commit_artifact_relay( &mut self, candidate: relay::ArtifactRelayLedger, @@ -458,10 +284,15 @@ impl CoordinatorService { { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node process-control request is outside its enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } return Ok(()); } self.coordinator @@ -578,16 +409,6 @@ impl CoordinatorService { enrollment_grants: BTreeMap::new(), task_events: VecDeque::new(), process_scope_history: VecDeque::new(), - process_summaries: BTreeMap::new(), - process_summary_order: VecDeque::new(), - next_process_summary_order: 1, - task_terminal_states: BTreeMap::new(), - recent_logs: BTreeMap::new(), - recent_log_dropped_through: BTreeMap::new(), - recent_log_accounted_bytes: BTreeMap::new(), - recent_log_truncated_streams: BTreeSet::new(), - recent_log_quota_truncated_streams: BTreeSet::new(), - next_recent_log_sequence: 1, debug_audit_events: VecDeque::new(), debug_epochs: BTreeMap::new(), debug_epoch_runtime: BTreeMap::new(), diff --git a/crates/clusterflux-coordinator/src/service/artifacts.rs b/crates/clusterflux-coordinator/src/service/artifacts.rs index 1a94c7e..c6698f9 100644 --- a/crates/clusterflux-coordinator/src/service/artifacts.rs +++ b/crates/clusterflux-coordinator/src/service/artifacts.rs @@ -8,7 +8,7 @@ use clusterflux_core::{ }; use sha2::{Digest as _, Sha256}; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::relay::RelayFinishReason; use super::{ @@ -52,11 +52,7 @@ impl CoordinatorService { let action = self .artifact_registry .download_action(&context, &artifact, &policy)?; - self.ensure_download_source_connectivity( - &context.tenant, - &context.project, - &action.source, - )?; + self.ensure_download_source_connectivity(&action.source)?; let downloadable_size = self .artifact_registry .downloadable_size(&context, &artifact, &policy)?; @@ -167,11 +163,7 @@ impl CoordinatorService { }, &mut validation_meter, )?; - self.ensure_download_source_connectivity( - &stream.link.tenant, - &stream.link.project, - &stream.link.source, - )?; + self.ensure_download_source_connectivity(&stream.link.source)?; self.expire_artifact_reverse_transfers(now_epoch_seconds)?; if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() { @@ -310,7 +302,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&context.tenant, &context.project, &artifact) + .metadata(&artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let transfer_id = generate_opaque_token("artifact_transfer") .map_err(CoordinatorServiceError::Protocol)?; @@ -418,7 +410,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&context.tenant, &context.project, &artifact) + .metadata(&artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?; let destination = @@ -690,10 +682,15 @@ impl CoordinatorService { ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "artifact reverse transfer node is outside its enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } Ok(()) } @@ -728,12 +725,15 @@ impl CoordinatorService { ) -> Result { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); - let node_scope = NodeScopeKey::from_refs(tenant, project, node); - let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "artifact export node is outside the tenant/project scope".to_owned(), + ) + .into()); + } + let descriptor = self.node_descriptors.get(node).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} has not reported export connectivity" )) @@ -744,7 +744,7 @@ impl CoordinatorService { ) .into()); } - if !self.node_is_live(&node_scope) { + if !self.node_is_live(node) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} is offline for artifact export" @@ -769,20 +769,17 @@ impl CoordinatorService { fn ensure_download_source_connectivity( &self, - tenant: &TenantId, - project: &ProjectId, source: &StorageLocation, ) -> Result<(), clusterflux_core::DownloadError> { let StorageLocation::RetainedNode(node) = source else { return Ok(()); }; - let node_scope = NodeScopeKey::from_refs(tenant, project, node); - let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { + let _descriptor = self.node_descriptors.get(node).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} has not reported online status for artifact download" )) })?; - if !self.node_is_live(&node_scope) { + if !self.node_is_live(node) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} is offline for artifact download" diff --git a/crates/clusterflux-coordinator/src/service/authorization.rs b/crates/clusterflux-coordinator/src/service/authorization.rs index f4d60fd..92c1e6d 100644 --- a/crates/clusterflux-coordinator/src/service/authorization.rs +++ b/crates/clusterflux-coordinator/src/service/authorization.rs @@ -17,7 +17,6 @@ pub(super) enum PublicUserOperation { RevokeAgentPublicKey, CreateNodeEnrollmentGrant, ListNodeDescriptors, - ListNodeSummaries, RevokeNodeCredential, StartProcess, ScheduleTask, @@ -25,7 +24,6 @@ pub(super) enum PublicUserOperation { CancelProcess, AbortProcess, ListProcesses, - ListProcessSummaries, QuotaStatus, RestartTask, ResolveTaskFailure, @@ -37,10 +35,7 @@ pub(super) enum PublicUserOperation { InspectDebugEpoch, ListTaskEvents, ListTaskSnapshots, - ListRecentLogs, JoinTask, - ListArtifacts, - GetArtifact, CreateArtifactDownloadLink, OpenArtifactDownloadStream, RevokeArtifactDownloadLink, @@ -61,7 +56,6 @@ impl PublicUserOperation { Self::RevokeAgentPublicKey => "revoke_agent_public_key", Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant", Self::ListNodeDescriptors => "list_node_descriptors", - Self::ListNodeSummaries => "list_node_summaries", Self::RevokeNodeCredential => "revoke_node_credential", Self::StartProcess => "start_process", Self::ScheduleTask => "schedule_task", @@ -69,7 +63,6 @@ impl PublicUserOperation { Self::CancelProcess => "cancel_process", Self::AbortProcess => "abort_process", Self::ListProcesses => "list_processes", - Self::ListProcessSummaries => "list_process_summaries", Self::QuotaStatus => "quota_status", Self::RestartTask => "restart_task", Self::ResolveTaskFailure => "resolve_task_failure", @@ -81,10 +74,7 @@ impl PublicUserOperation { Self::InspectDebugEpoch => "inspect_debug_epoch", Self::ListTaskEvents => "list_task_events", Self::ListTaskSnapshots => "list_task_snapshots", - Self::ListRecentLogs => "list_recent_logs", Self::JoinTask => "join_task", - Self::ListArtifacts => "list_artifacts", - Self::GetArtifact => "get_artifact", Self::CreateArtifactDownloadLink => "create_artifact_download_link", Self::OpenArtifactDownloadStream => "open_artifact_download_stream", Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link", @@ -115,7 +105,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { Self::CreateNodeEnrollmentGrant } AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors, - AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries, AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => { Self::RevokeNodeCredential } @@ -125,9 +114,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess, AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess, AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses, - AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => { - Self::ListProcessSummaries - } AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus, AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask, AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure, @@ -143,10 +129,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch, AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents, AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots, - AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs, AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask, - AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts, - AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact, AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => { Self::CreateArtifactDownloadLink } diff --git a/crates/clusterflux-coordinator/src/service/debug.rs b/crates/clusterflux-coordinator/src/service/debug.rs index d5af181..efa1406 100644 --- a/crates/clusterflux-coordinator/src/service/debug.rs +++ b/crates/clusterflux-coordinator/src/service/debug.rs @@ -35,7 +35,6 @@ pub(super) struct DebugEpochRuntime { #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugBreakpointPlan { pub(super) actor: UserId, - pub(super) revision: u64, pub(super) probe_symbols: BTreeSet, pub(super) hit_epoch: Option, pub(super) hit_task: Option, @@ -86,7 +85,6 @@ impl CoordinatorService { project: String, actor_user: String, process: String, - revision: u64, probe_symbols: Vec, ) -> Result { let probe_symbols = validate_probe_symbols(probe_symbols)?; @@ -117,28 +115,10 @@ impl CoordinatorService { )) .into()); } - let key = process_control_key(&tenant, &project, &process); - if let Some(current) = self.debug_breakpoints.get(&key) { - if current.actor == actor && revision < current.revision { - return Ok(CoordinatorResponse::DebugBreakpoints { - process, - actor, - revision: current.revision, - probe_symbols: current.probe_symbols.iter().cloned().collect(), - hit_epoch: current.hit_epoch, - hit_task: current.hit_task.clone(), - hit_probe_symbol: current.hit_probe_symbol.clone(), - charged_debug_read_bytes: audit_event.charged_debug_read_bytes, - used_debug_read_bytes: audit_event.used_debug_read_bytes, - audit_event, - }); - } - } self.debug_breakpoints.insert( - key, + process_control_key(&tenant, &project, &process), DebugBreakpointPlan { actor: actor.clone(), - revision, probe_symbols: probe_symbols.iter().cloned().collect(), hit_epoch: None, hit_task: None, @@ -148,7 +128,6 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, - revision, probe_symbols, hit_epoch: None, hit_task: None, @@ -211,7 +190,6 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, - revision: plan.revision, probe_symbols: plan.probe_symbols.into_iter().collect(), hit_epoch: plan.hit_epoch, hit_task: plan.hit_task, diff --git a/crates/clusterflux-coordinator/src/service/debug_requests.rs b/crates/clusterflux-coordinator/src/service/debug_requests.rs index 343f177..4aae6c0 100644 --- a/crates/clusterflux-coordinator/src/service/debug_requests.rs +++ b/crates/clusterflux-coordinator/src/service/debug_requests.rs @@ -33,14 +33,12 @@ impl CoordinatorService { project, actor_user, process, - revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant, project, actor_user, process, - revision, probe_symbols, ), CoordinatorRequest::InspectDebugBreakpoints { @@ -98,14 +96,12 @@ impl CoordinatorService { ), AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, - revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant.as_str().to_owned(), project.as_str().to_owned(), actor.as_str().to_owned(), process, - revision, probe_symbols, ), AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self @@ -226,25 +222,24 @@ impl CoordinatorService { } else if !completed_event_observed { "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned() } else if let Some(checkpoint) = checkpoint { - let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold( - true, - |available, (path, object)| { - let artifact = super::keys::artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?; - Ok::<_, CoordinatorServiceError>( - available - && self - .artifact_registry - .metadata(&tenant, &project, &artifact) - .is_some_and(|metadata| { - metadata.digest == object.digest - && metadata.size == object.size - && !metadata.retaining_nodes.is_empty() - }), - ) - }, - )?; + let vfs_available = + checkpoint + .checkpoint + .vfs_manifest + .objects + .iter() + .all(|(path, object)| { + let artifact = super::keys::artifact_id_from_path(path); + self.artifact_registry + .metadata(&artifact) + .is_some_and(|metadata| { + metadata.tenant == tenant + && metadata.project == project + && metadata.digest == object.digest + && metadata.size == object.size + && !metadata.retaining_nodes.is_empty() + }) + }); if !vfs_available { "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned() } else { @@ -480,7 +475,6 @@ impl CoordinatorService { } self.record_task_completion_event(event.clone()); self.notify_coordinator_main_waiters(&event); - self.maybe_retire_terminal_process(&tenant, &project, &process)?; Ok(CoordinatorResponse::TaskFailureResolved { process, task, diff --git a/crates/clusterflux-coordinator/src/service/keys.rs b/crates/clusterflux-coordinator/src/service/keys.rs index 2e14ba8..8cc82c5 100644 --- a/crates/clusterflux-coordinator/src/service/keys.rs +++ b/crates/clusterflux-coordinator/src/service/keys.rs @@ -1,7 +1,6 @@ use clusterflux_core::{ ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath, }; -use thiserror::Error; pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId); pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId); @@ -64,47 +63,11 @@ pub(super) fn enrollment_grant_key( (tenant.clone(), project.clone(), grant.to_owned()) } -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub(super) enum ArtifactPathError { - #[error("path must start with the exact /vfs/artifacts/ prefix")] - WrongPrefix, - #[error("path must name an artifact after /vfs/artifacts/")] - EmptyArtifact, - #[error("mapped artifact identifier is invalid: {0}")] - InvalidArtifactId(#[from] clusterflux_core::IdParseError), -} - -pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result { +pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId { let value = path .as_str() .strip_prefix("/vfs/artifacts/") - .ok_or(ArtifactPathError::WrongPrefix)?; - if value.is_empty() { - return Err(ArtifactPathError::EmptyArtifact); - } - ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn artifact_id_conversion_is_fallible_and_structural() { - assert_eq!( - artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(), - ArtifactId::from("build:output") - ); - for path in [ - "/vfs/other/output", - "/vfs/artifacts", - "/vfs/artifacts/bad artifact!", - ] { - let path = VfsPath::new(path).unwrap(); - assert!(artifact_id_from_path(&path).is_err(), "{path:?}"); - } - - let mapped = format!("/vfs/artifacts/{}", "x".repeat(256)); - assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err()); - } + .unwrap_or(path.as_str()) + .replace('/', ":"); + ArtifactId::new(value) } diff --git a/crates/clusterflux-coordinator/src/service/logs.rs b/crates/clusterflux-coordinator/src/service/logs.rs index 1fb575b..fec1d2b 100644 --- a/crates/clusterflux-coordinator/src/service/logs.rs +++ b/crates/clusterflux-coordinator/src/service/logs.rs @@ -1,5 +1,5 @@ use clusterflux_core::{ - ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, + ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath, }; @@ -9,10 +9,7 @@ use super::keys::{process_control_key, task_control_key, task_restart_key}; use super::protocol::TaskAttemptState; use super::{ artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError, - RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState, - MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES, - MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT, - MAX_TASK_LOG_TAIL_BYTES, + TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES, }; impl CoordinatorService { @@ -39,44 +36,23 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; validate_task_log_tail("stdout_tail", &stdout_tail)?; validate_task_log_tail("stderr_tail", &stderr_tail)?; + let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - let stdout_retained = self.accept_final_log_stream( - &tenant, - &project, - &process, - &task, - TaskLogStream::Stdout, - stdout_bytes, - &stdout_tail, - stdout_truncated, - now_epoch_seconds, - )?; - let stderr_retained = self.accept_final_log_stream( - &tenant, - &project, - &process, - &task, - TaskLogStream::Stderr, - stderr_bytes, - &stderr_tail, - stderr_truncated, - now_epoch_seconds, - )?; + self.quota + .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; + self.quota + .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; Ok(CoordinatorResponse::TaskLogRecorded { process, task, stdout_bytes, stderr_bytes, - stdout_tail: if !stdout_retained { - "[log output truncated at project log quota]".to_owned() - } else if stdout_truncated { + stdout_tail: if stdout_truncated { format!("{stdout_tail}\n... truncated") } else { stdout_tail }, - stderr_tail: if !stderr_retained { - "[log output truncated at project log quota]".to_owned() - } else if stderr_truncated { + stderr_tail: if stderr_truncated { format!("{stderr_tail}\n... truncated") } else { stderr_tail @@ -85,158 +61,6 @@ impl CoordinatorService { }) } - #[allow(clippy::too_many_arguments)] - pub(super) fn handle_report_task_log_chunk( - &mut self, - tenant: String, - project: String, - process: String, - node: String, - task: String, - stream: TaskLogStream, - offset: u64, - source_bytes: u64, - text: String, - truncated: bool, - ) -> Result { - if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { - return Err(CoordinatorServiceError::InvalidTaskLogTail(format!( - "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}", - text.len() - ))); - } - if source_bytes == 0 && !text.is_empty() && !truncated { - return Err(CoordinatorServiceError::Protocol( - "live log chunk source_bytes must describe non-empty text".to_owned(), - )); - } - if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) { - return Err(CoordinatorServiceError::Protocol( - "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(), - )); - } - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let process = ProcessId::new(process); - let node = NodeId::new(node); - let task = TaskInstanceId::new(task); - self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; - let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream); - let expected = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let end = offset.checked_add(source_bytes).ok_or_else(|| { - CoordinatorServiceError::Protocol( - "live log chunk offset exceeds the supported range".to_owned(), - ) - })?; - let state_marker = source_bytes == 0 && truncated; - if end < expected || (end == expected && !state_marker) { - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence: None, - next_offset: expected, - }); - } - let now_epoch_seconds = self.current_epoch_seconds()?; - if self.recent_log_quota_truncated_streams.contains(&key) { - if end > expected { - self.recent_log_accounted_bytes.insert(key, end); - } - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence: None, - next_offset: end.max(expected), - }); - } - let newly_accounted = end.saturating_sub(expected); - if self - .quota - .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds) - .is_err() - { - if end > expected { - self.recent_log_accounted_bytes.insert(key.clone(), end); - } - let sequence = self.mark_log_quota_truncated( - &tenant, - &project, - &process, - &task, - &stream, - now_epoch_seconds, - ); - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence, - next_offset: end.max(expected), - }); - } - if offset > expected { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!("[log output lost: {} bytes]", offset - expected), - true, - now_epoch_seconds, - ); - } else if offset < expected { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[log output overlap omitted: {} new source bytes]", - end - expected - ), - true, - now_epoch_seconds, - ); - } - let marker_is_new = if truncated { - self.recent_log_truncated_streams.insert(key.clone()) - } else { - false - }; - let text = if state_marker && text.is_empty() { - "[log output truncated at source]".to_owned() - } else { - text - }; - let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new)) - .then(|| { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream, - text, - truncated, - now_epoch_seconds, - ) - }); - if end > expected { - self.recent_log_accounted_bytes.insert(key, end); - } - Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence, - next_offset: end, - }) - } - pub(super) fn handle_report_vfs_metadata( &mut self, tenant: String, @@ -261,9 +85,7 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?, + id: artifact_id_from_path(path), tenant, project, process: process.clone(), @@ -355,37 +177,20 @@ impl CoordinatorService { artifact_size_bytes, result, }; + let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - let stdout_retained = self.accept_final_log_stream( + self.quota.can_charge_log_bytes( &event.tenant, &event.project, - &event.process, - &event.task, - TaskLogStream::Stdout, - event.stdout_bytes, - &event.stdout_tail, - event.stdout_truncated, + reported_bytes, now_epoch_seconds, )?; - let stderr_retained = self.accept_final_log_stream( + self.quota.charge_log_bytes( &event.tenant, &event.project, - &event.process, - &event.task, - TaskLogStream::Stderr, - event.stderr_bytes, - &event.stderr_tail, - event.stderr_truncated, + reported_bytes, now_epoch_seconds, )?; - if !stdout_retained { - event.stdout_tail = "[log output truncated at project log quota]".to_owned(); - event.stdout_truncated = true; - } - if !stderr_retained { - event.stderr_tail = "[log output truncated at project log quota]".to_owned(); - event.stderr_truncated = true; - } let task_key = task_control_key( &event.tenant, &event.project, @@ -398,9 +203,7 @@ impl CoordinatorService { event.placement = self.task_placements.remove(&task_key); if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?, + id: artifact_id_from_path(path), tenant: event.tenant.clone(), project: event.project.clone(), process: event.process.clone(), @@ -414,22 +217,26 @@ impl CoordinatorService { self.task_aborts.remove(&task_key); self.debug_commands.remove(&task_key); self.active_tasks.remove(&task_key); - self.clear_recent_log_offsets_for_task( - &event.tenant, - &event.project, - &event.process, - &event.task, - ); - self.task_assignments.retain(|_, assignments| { - assignments.retain(|assignment| { - assignment.tenant != event.tenant - || assignment.project != event.project - || assignment.process != event.process - || assignment.node != event.node - || assignment.task != event.task - }); - !assignments.is_empty() - }); + let no_active_tasks = + !self + .active_tasks + .iter() + .any(|(task_tenant, task_project, task_process, _, _)| { + task_tenant == &event.tenant + && task_project == &event.project + && task_process == &event.process + }); + if no_active_tasks { + self.process_aborts.remove(&process_key); + if self.process_cancellations.remove(&process_key) + && !self.main_runtime.controls.contains_key(&process_key) + { + self.coordinator + .abort_process(&event.tenant, &event.project, &event.process)?; + self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process); + self.clear_operator_panel_state(&event.tenant, &event.project, &event.process); + } + } if process_was_aborted { let checkpoint_key = super::keys::task_restart_key( &event.tenant, @@ -446,20 +253,10 @@ impl CoordinatorService { if !awaiting_operator { self.notify_coordinator_main_waiters(&event); } - self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?; - let events_recorded = self - .task_events - .iter() - .filter(|recorded| { - recorded.tenant == event.tenant - && recorded.project == event.project - && recorded.process == event.process - }) - .count(); Ok(CoordinatorResponse::TaskRecorded { process: event.process, task: event.task, - events_recorded, + events_recorded: self.task_events.len(), }) } @@ -519,7 +316,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::TaskSnapshots { snapshots }) } - pub(super) fn authorize_task_event_process_scope( + fn authorize_task_event_process_scope( &self, tenant: &TenantId, project: &ProjectId, @@ -559,47 +356,6 @@ impl CoordinatorService { Ok(()) } - pub(super) fn handle_list_recent_logs( - &mut self, - tenant: String, - project: String, - actor_user: String, - process: String, - task: Option, - after_sequence: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let process = ProcessId::new(process); - let task = task.map(TaskInstanceId::new); - self.authorize_task_event_process_scope(&tenant, &project, &process)?; - let after_sequence = after_sequence.unwrap_or(0); - let retained = self.recent_logs.get(&(tenant.clone(), project.clone())); - let history_truncated = self - .recent_log_dropped_through - .get(&process_control_key(&tenant, &project, &process)) - .is_some_and(|dropped_through| *dropped_through > after_sequence); - let entries = retained - .into_iter() - .flatten() - .filter(|entry| { - entry.process == process - && entry.sequence > after_sequence - && task.as_ref().is_none_or(|task| &entry.task == task) - }) - .take(limit as usize) - .cloned() - .collect::>(); - let next_sequence = entries.last().map(|entry| entry.sequence); - Ok(CoordinatorResponse::RecentLogs { - entries, - next_sequence, - history_truncated, - }) - } - pub(super) fn handle_join_task( &mut self, tenant: String, @@ -718,22 +474,6 @@ impl CoordinatorService { pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) { event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated); event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated); - match event.executor { - super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state( - &event.tenant, - &event.project, - &event.process, - event.task_definition.clone(), - event.task.clone(), - event.terminal_state.clone(), - ), - super::TaskExecutor::Node => { - self.task_terminal_states.insert( - task_restart_key(&event.tenant, &event.project, &event.process, &event.task), - event.terminal_state.clone(), - ); - } - } let process_scope = ( event.tenant.clone(), event.project.clone(), @@ -771,321 +511,6 @@ impl CoordinatorService { self.task_events.push_back(event); } - #[allow(clippy::too_many_arguments)] - fn record_recent_log( - &mut self, - tenant: TenantId, - project: ProjectId, - process: ProcessId, - task: TaskInstanceId, - stream: TaskLogStream, - mut text: String, - mut truncated: bool, - server_timestamp_epoch_seconds: u64, - ) -> u64 { - if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { - let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES; - while !text.is_char_boundary(boundary) { - boundary -= 1; - } - text.truncate(boundary); - truncated = true; - } - let sequence = self.next_recent_log_sequence; - self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1); - let logs = self - .recent_logs - .entry((tenant.clone(), project.clone())) - .or_default(); - let mut dropped = Vec::new(); - while logs.iter().filter(|entry| entry.process == process).count() - >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS - { - let Some(index) = logs.iter().position(|entry| entry.process == process) else { - break; - }; - if let Some(entry) = logs.remove(index) { - dropped.push(entry); - } - } - while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT - || logs - .iter() - .map(|entry| entry.text.len()) - .sum::() - .saturating_add(text.len()) - > MAX_RECENT_LOG_BYTES_PER_PROJECT - { - match logs.pop_front() { - Some(entry) => dropped.push(entry), - None => break, - } - } - logs.push_back(RecentLogEntry { - sequence, - process, - task, - stream, - text, - server_timestamp_epoch_seconds, - truncated, - }); - for entry in dropped { - let key = process_control_key(&tenant, &project, &entry.process); - self.recent_log_dropped_through - .entry(key) - .and_modify(|dropped_through| { - *dropped_through = (*dropped_through).max(entry.sequence); - }) - .or_insert(entry.sequence); - } - sequence - } - - #[allow(clippy::too_many_arguments)] - fn accept_final_log_stream( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: TaskLogStream, - total_source_bytes: u64, - final_tail: &str, - source_truncated: bool, - now_epoch_seconds: u64, - ) -> Result { - let key = recent_log_offset_key(tenant, project, process, task, &stream); - let accounted = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| { - let stream_name = match stream { - TaskLogStream::Stdout => "stdout", - TaskLogStream::Stderr => "stderr", - }; - CoordinatorServiceError::Protocol(format!( - "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted" - )) - })?; - if self.recent_log_quota_truncated_streams.contains(&key) { - self.recent_log_accounted_bytes - .insert(key, total_source_bytes); - return Ok(false); - } - if self - .quota - .charge_log_bytes(tenant, project, remaining, now_epoch_seconds) - .is_err() - { - self.recent_log_accounted_bytes - .insert(key, total_source_bytes); - self.mark_log_quota_truncated( - tenant, - project, - process, - task, - &stream, - now_epoch_seconds, - ); - return Ok(false); - } - self.reconcile_final_log_stream( - tenant, - project, - process, - task, - stream, - total_source_bytes, - final_tail, - source_truncated, - now_epoch_seconds, - ); - Ok(true) - } - - #[allow(clippy::too_many_arguments)] - fn mark_log_quota_truncated( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: &TaskLogStream, - now_epoch_seconds: u64, - ) -> Option { - let key = recent_log_offset_key(tenant, project, process, task, stream); - if !self.recent_log_quota_truncated_streams.insert(key.clone()) { - return None; - } - self.recent_log_truncated_streams.insert(key); - Some(self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - "[log output truncated at project log quota]".to_owned(), - true, - now_epoch_seconds, - )) - } - - #[allow(clippy::too_many_arguments)] - fn reconcile_final_log_stream( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: TaskLogStream, - total_source_bytes: u64, - final_tail: &str, - source_truncated: bool, - now_epoch_seconds: u64, - ) { - let key = recent_log_offset_key(tenant, project, process, task, &stream); - let accounted = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let mut visible_truncation = false; - if total_source_bytes > accounted { - let missing = total_source_bytes - accounted; - if final_tail.is_empty() { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!("[log output unavailable: {missing} source bytes]"), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } else if (final_tail.len() as u64) <= total_source_bytes { - let tail_source_start = total_source_bytes - final_tail.len() as u64; - if accounted < tail_source_start { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[log output lost before final tail: {} source bytes]", - tail_source_start - accounted - ), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } - let source_start = accounted.max(tail_source_start) - tail_source_start; - let mut byte_start = usize::try_from(source_start) - .unwrap_or(final_tail.len()) - .min(final_tail.len()); - while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) { - byte_start += 1; - } - let suffix = &final_tail[byte_start..]; - if !suffix.is_empty() { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - suffix.to_owned(), - source_truncated || visible_truncation, - now_epoch_seconds, - ); - visible_truncation |= source_truncated; - } - } else if accounted == 0 { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - final_tail.to_owned(), - source_truncated, - now_epoch_seconds, - ); - visible_truncation |= source_truncated; - } else { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[{missing} additional source bytes could not be merged without duplicating redacted output]" - ), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } - self.recent_log_accounted_bytes - .insert(key.clone(), total_source_bytes); - } - - let marker_is_new = (source_truncated || visible_truncation) - && self.recent_log_truncated_streams.insert(key); - if marker_is_new && !visible_truncation { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream, - "[log output truncated at source]".to_owned(), - true, - now_epoch_seconds, - ); - } - } - - pub(super) fn clear_recent_log_offsets_for_task( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - ) { - self.recent_log_accounted_bytes.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _), _| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - self.recent_log_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _)| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - self.recent_log_quota_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _)| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - } - fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool { let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task); let Some(attempt) = self @@ -1120,118 +545,6 @@ impl CoordinatorService { awaiting_operator } - pub(super) fn maybe_retire_terminal_process( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - ) -> Result { - let process_key = process_control_key(tenant, project, process); - if self.main_runtime.controls.contains_key(&process_key) { - return Ok(false); - } - - let has_runnable_remote_work = - self.active_tasks - .iter() - .any(|(task_tenant, task_project, task_process, _, _)| { - task_tenant == tenant && task_project == project && task_process == process - }) - || self.pending_task_launches.iter().any(|pending| { - &pending.tenant == tenant - && &pending.project == project - && &pending.process == process - }) - || self.task_assignments.values().any(|assignments| { - assignments.iter().any(|assignment| { - &assignment.tenant == tenant - && &assignment.project == project - && &assignment.process == process - }) - }) - || self.task_attempts.iter().any( - |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { - attempt_tenant == tenant - && attempt_project == project - && attempt_process == process - && attempts.iter().rev().any(|attempt| { - attempt.current - && matches!( - attempt.state, - TaskAttemptState::Queued - | TaskAttemptState::Running - | TaskAttemptState::FailedAwaitingAction - ) - }) - }, - ); - if has_runnable_remote_work { - return Ok(false); - } - - let main_completed = self - .process_summaries - .get(&process_key) - .and_then(|summary| summary.main_terminal_state.as_ref()) - .is_some_and(|state| matches!(state, TaskTerminalState::Completed)); - let cancellation_completed = self.process_cancellations.contains(&process_key); - if !main_completed && !cancellation_completed { - return Ok(false); - } - - self.process_aborts.remove(&process_key); - self.process_cancellations.remove(&process_key); - if self - .coordinator - .active_process(tenant, project, process) - .is_none() - { - return Ok(false); - } - let final_result = if cancellation_completed { - super::ProcessFinalResult::Cancelled - } else if self.task_terminal_states.iter().any( - |((task_tenant, task_project, task_process, _), terminal_state)| { - task_tenant == tenant - && task_project == project - && task_process == process - && terminal_state == &TaskTerminalState::Failed - }, - ) { - super::ProcessFinalResult::Failed - } else if self.task_terminal_states.iter().any( - |((task_tenant, task_project, task_process, _), terminal_state)| { - task_tenant == tenant - && task_project == project - && task_process == process - && terminal_state == &TaskTerminalState::Cancelled - }, - ) { - super::ProcessFinalResult::Cancelled - } else { - super::ProcessFinalResult::Completed - }; - self.record_process_terminal( - tenant, - project, - process, - final_result, - self.current_epoch_seconds()?, - ); - self.coordinator.abort_process(tenant, project, process)?; - self.clear_debug_state_for_process(tenant, project, process); - self.clear_operator_panel_state(tenant, project, process); - let (pinned, protected_processes) = - self.artifact_retention_guards_for_project(tenant, project); - self.artifact_registry.enforce_project_metadata_limit( - tenant, - project, - &pinned, - &protected_processes, - ); - Ok(true) - } - fn flush_artifact_metadata( &mut self, flush: ArtifactFlush, @@ -1239,59 +552,23 @@ impl CoordinatorService { let now_epoch_seconds = self.current_epoch_seconds()?; self.artifact_registry .expire_download_links(now_epoch_seconds); - let tenant = flush.tenant.clone(); - let project = flush.project.clone(); - let (pinned, protected_processes) = - self.artifact_retention_guards_for_project(&tenant, &project); + let pinned = self + .task_restart_checkpoints + .values() + .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter()) + .chain( + self.pending_task_launches + .iter() + .flat_map(|pending| pending.task_spec.required_artifacts.iter()), + ) + .cloned() + .collect::>(); self.artifact_registry - .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes) + .flush_metadata_bounded(flush, &pinned) .map(|_| ()) .map_err(CoordinatorServiceError::Protocol) } - fn artifact_retention_guards_for_project( - &self, - tenant: &TenantId, - project: &ProjectId, - ) -> ( - std::collections::BTreeSet, - std::collections::BTreeSet, - ) { - let mut pinned = std::collections::BTreeSet::new(); - for checkpoint in self.task_restart_checkpoints.values() { - if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project - { - continue; - } - for artifact in &checkpoint.assignment.task_spec.required_artifacts { - pinned.insert(ArtifactScopeKey::from_refs( - &checkpoint.assignment.tenant, - &checkpoint.assignment.project, - artifact, - )); - } - } - for pending in &self.pending_task_launches { - if &pending.tenant != tenant || &pending.project != project { - continue; - } - for artifact in &pending.task_spec.required_artifacts { - pinned.insert(ArtifactScopeKey::from_refs( - &pending.tenant, - &pending.project, - artifact, - )); - } - } - let protected_processes = self - .coordinator - .active_processes_for_project(tenant, project) - .into_iter() - .map(|process| process.id) - .collect(); - (pinned, protected_processes) - } - fn task_is_known_or_active( &self, tenant: &TenantId, @@ -1324,24 +601,15 @@ impl CoordinatorService { } } -fn recent_log_offset_key( - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: &TaskLogStream, -) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) { - ( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - match stream { - TaskLogStream::Stdout => "stdout", - TaskLogStream::Stderr => "stderr", - } - .to_owned(), - ) +fn checked_reported_log_bytes( + stdout_bytes: u64, + stderr_bytes: u64, +) -> Result { + stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| { + CoordinatorServiceError::Protocol( + "reported task log byte counts exceed the supported range".to_owned(), + ) + }) } fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> { @@ -1358,11 +626,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String { if value.len() <= MAX_TASK_LOG_TAIL_BYTES { return value; } - let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES; - while boundary < value.len() && !value.is_char_boundary(boundary) { - boundary += 1; + let mut boundary = MAX_TASK_LOG_TAIL_BYTES; + while !value.is_char_boundary(boundary) { + boundary -= 1; } - value.drain(..boundary); + value.truncate(boundary); *truncated = true; value } diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index 30b81a8..dd2feb2 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -30,14 +30,14 @@ use super::{ }; #[derive(Clone)] -pub(super) struct MainScope { - pub(super) tenant: TenantId, - pub(super) project: ProjectId, - pub(super) process: ProcessId, - pub(super) task_definition: TaskDefinitionId, - pub(super) task_instance: TaskInstanceId, - pub(super) epoch: u64, - pub(super) launch_id: u64, +struct MainScope { + tenant: TenantId, + project: ProjectId, + process: ProcessId, + task_definition: TaskDefinitionId, + task_instance: TaskInstanceId, + epoch: u64, + launch_id: u64, } enum MainCommand { @@ -109,14 +109,6 @@ impl Default for CoordinatorMainRuntime { } impl CoordinatorMainRuntime { - pub(super) fn active_main_count(&self) -> usize { - self.controls.len() - } - - pub(super) fn max_active_mains(&self) -> usize { - self.max_active_mains - } - pub(super) fn configure( &mut self, configuration: super::CoordinatorMainRuntimeConfiguration, @@ -737,13 +729,6 @@ impl CoordinatorService { &process, "coordinator main launch failed admission or validation", ); - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Failed, - self.liveness_now_epoch_seconds(), - ); let _ = self.coordinator.abort_process(&tenant, &project, &process); } result @@ -904,7 +889,7 @@ impl CoordinatorService { } } - pub(super) fn record_coordinator_main_completion( + fn record_coordinator_main_completion( &mut self, scope: MainScope, result: Result, @@ -944,7 +929,6 @@ impl CoordinatorService { ), Err(error) => (TaskTerminalState::Failed, None, error), }; - let main_completed = matches!(terminal_state, TaskTerminalState::Completed); let main_state = match terminal_state { TaskTerminalState::Completed => "completed", TaskTerminalState::Failed => "failed", @@ -986,12 +970,6 @@ impl CoordinatorService { } } let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process); - self.main_runtime.controls.remove(&process_key); - if main_completed { - let _ = - self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process); - return; - } for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() { if task_tenant == scope.tenant && task_project == scope.project @@ -1007,16 +985,10 @@ impl CoordinatorService { } } self.process_aborts.insert(process_key.clone()); - self.record_process_terminal( - &scope.tenant, - &scope.project, - &scope.process, - super::ProcessFinalResult::Failed, - self.liveness_now_epoch_seconds(), - ); let _ = self .coordinator .abort_process(&scope.tenant, &scope.project, &scope.process); + self.main_runtime.controls.remove(&process_key); self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process); self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process); } @@ -1185,120 +1157,6 @@ mod tests { assert!(!service.main_runtime.controls.contains_key(&process_key)); assert!(!service.debug_epochs.contains_key(&process_key)); assert!(!service.debug_epoch_runtime.contains_key(&process_key)); - assert!(!service.process_aborts.contains(&process_key)); - } - - #[test] - fn completed_main_keeps_process_and_debug_state_for_active_children() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("vp-current"); - let main_task = TaskInstanceId::from("ti:vp-current:main"); - let child_task = TaskInstanceId::from("ti:vp-current:child:1"); - let child_node = NodeId::from("worker"); - let scope = MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main_task.clone(), - epoch: 7, - launch_id: 1, - }; - service - .coordinator - .start_process(tenant.clone(), project.clone(), process.clone()); - let process_key = process_control_key(&tenant, &project, &process); - service.main_runtime.controls.insert( - process_key.clone(), - CoordinatorMainControl { - task_definition: scope.task_definition.clone(), - task_instance: main_task.clone(), - abort: Arc::new(AtomicBool::new(false)), - debug: Arc::new(WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: Arc::new(Mutex::new(HashMap::new())), - launch_id: 1, - }, - ); - let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task); - service.active_tasks.insert(child_key.clone()); - service.debug_epochs.insert(process_key.clone(), 2); - - service.record_coordinator_main_completion( - scope, - Ok(WasmTaskResult::completed( - main_task, - TaskBoundaryValue::SmallJson(serde_json::Value::Null), - )), - ); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(!service.main_runtime.controls.contains_key(&process_key)); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(!service.process_aborts.contains(&process_key)); - assert!(service.active_tasks.contains(&child_key)); - assert!(!service.task_aborts.contains(&child_key)); - } - - #[test] - fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("vp-failed-main"); - let main_task = TaskInstanceId::from("ti:vp-failed-main:main"); - let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1"); - let child_node = NodeId::from("worker"); - let scope = MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main_task.clone(), - epoch: 7, - launch_id: 1, - }; - service - .coordinator - .start_process(tenant.clone(), project.clone(), process.clone()); - let process_key = process_control_key(&tenant, &project, &process); - service.main_runtime.controls.insert( - process_key.clone(), - CoordinatorMainControl { - task_definition: scope.task_definition.clone(), - task_instance: main_task, - abort: Arc::new(AtomicBool::new(false)), - debug: Arc::new(WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: Arc::new(Mutex::new(HashMap::new())), - launch_id: 1, - }, - ); - let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task); - service.active_tasks.insert(child_key.clone()); - service.debug_epochs.insert(process_key.clone(), 2); - - service.record_coordinator_main_completion(scope, Err("main crashed".to_owned())); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(service.active_tasks.contains(&child_key)); - assert!(service.task_aborts.contains(&child_key)); assert!(service.process_aborts.contains(&process_key)); - assert!(!service.debug_epochs.contains_key(&process_key)); - assert!(service.task_events.iter().any(|event| { - event.process == process - && event.executor == TaskExecutor::CoordinatorMain - && event.terminal_state == TaskTerminalState::Failed - })); } } diff --git a/crates/clusterflux-coordinator/src/service/nodes.rs b/crates/clusterflux-coordinator/src/service/nodes.rs index cedae30..9db62c8 100644 --- a/crates/clusterflux-coordinator/src/service/nodes.rs +++ b/crates/clusterflux-coordinator/src/service/nodes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ SourceProviderKind, TenantId, UserId, }; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::{ bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService, @@ -27,9 +27,9 @@ impl CoordinatorService { unix_timestamp_seconds() } - pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool { + pub(super) fn node_is_live(&self, node: &NodeId) -> bool { self.node_last_seen_epoch_seconds - .get(scope) + .get(node) .is_some_and(|last_seen| { self.liveness_now_epoch_seconds().saturating_sub(*last_seen) <= self.node_stale_after_seconds @@ -41,11 +41,7 @@ impl CoordinatorService { .values() .cloned() .map(|mut descriptor| { - descriptor.online = self.node_is_live(&NodeScopeKey::from_refs( - &descriptor.tenant, - &descriptor.project, - &descriptor.id, - )); + descriptor.online = self.node_is_live(&descriptor.id); descriptor }) .collect() @@ -169,11 +165,7 @@ impl CoordinatorService { !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds }); self.coordinator.ensure_tenant_active(&tenant)?; - if self - .coordinator - .node_identity(&tenant, &project, &node) - .is_none() - { + if self.coordinator.node_identity(&node).is_none() { self.quota.ensure_node_admission( &tenant, self.coordinator.node_identity_count_for_tenant(&tenant), @@ -205,21 +197,12 @@ impl CoordinatorService { pub(super) fn handle_node_heartbeat( &mut self, - tenant: String, - project: String, node: String, node_signature: Option, payload_digest: &Digest, ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); let node = NodeId::new(node); - self.authenticate_node_request( - &NodeScopeKey::new(tenant, project, node.clone()), - node_signature, - "node_heartbeat", - payload_digest, - )?; + self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?; Ok(CoordinatorResponse::NodeHeartbeat { node, epoch: self.coordinator.coordinator_epoch(), @@ -242,13 +225,16 @@ impl CoordinatorService { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let node = NodeId::new(node); - let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node capability report is outside the enrolled tenant/project scope".to_owned(), + ) + .into()); + } capabilities.validate_public_report()?; for (kind, count) in [ ("cached environments", cached_environment_digests.len()), @@ -288,20 +274,16 @@ impl CoordinatorService { .into_iter() .map(ArtifactId::new) .collect::>(); - self.artifact_registry.reconcile_node_retention( - &tenant, - &project, - &node, - &artifact_locations, - ); + self.artifact_registry + .reconcile_node_retention(&node, &artifact_locations); - let online = self.node_is_live(&node_scope); + let online = self.node_is_live(&node); self.node_descriptors.insert( - node_scope, + node.clone(), NodeDescriptor { id: node.clone(), - tenant: tenant.clone(), - project: project.clone(), + tenant, + project, capabilities, cached_environments: cached_environment_digests.into_iter().collect(), dependency_caches: dependency_cache_digests.into_iter().collect(), @@ -311,14 +293,9 @@ impl CoordinatorService { online, }, ); - let node_descriptors = self - .node_descriptors - .values() - .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project) - .count(); Ok(CoordinatorResponse::NodeCapabilitiesRecorded { node, - node_descriptors, + node_descriptors: self.node_descriptors.len(), }) } @@ -356,13 +333,9 @@ impl CoordinatorService { actor: Actor::User(actor.clone()), }; self.coordinator.revoke_node_credential(&context, &node)?; - let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); - let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some(); - self.node_last_seen_epoch_seconds.remove(&node_scope); - self.node_replay_nonces - .retain(|(retained_scope, _), _| retained_scope != &node_scope); - self.artifact_registry - .garbage_collect_node(&tenant, &project, &node); + let descriptor_removed = self.node_descriptors.remove(&node).is_some(); + self.node_last_seen_epoch_seconds.remove(&node); + self.artifact_registry.garbage_collect_node(&node); let queued_assignments_removed = self .task_assignments .remove(&(tenant.clone(), project.clone(), node.clone())) @@ -396,14 +369,14 @@ impl CoordinatorService { pub(super) fn authenticate_node_request( &mut self, - scope: &NodeScopeKey, + node: &NodeId, node_signature: Option, request_kind: &str, payload_digest: &Digest, ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(&scope.tenant, &scope.project, &scope.node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; let signature = node_signature.ok_or_else(|| { CoordinatorError::Unauthorized( @@ -428,7 +401,7 @@ impl CoordinatorService { ) .into()); } - let replay_key = (scope.clone(), signature.nonce.clone()); + let replay_key = (node.clone(), signature.nonce.clone()); self.node_replay_nonces.retain(|_, accepted_at| { now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS) }); @@ -440,7 +413,7 @@ impl CoordinatorService { } verify_node_request_signature( &identity.public_key, - &scope.node, + node, request_kind, payload_digest, &signature, @@ -449,7 +422,7 @@ impl CoordinatorService { if self .node_replay_nonces .keys() - .filter(|(retained_scope, _)| retained_scope == scope) + .filter(|(retained_node, _)| retained_node == node) .count() >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY { @@ -463,8 +436,8 @@ impl CoordinatorService { .insert(replay_key, now_epoch_seconds); let seen_at = self.liveness_now_epoch_seconds(); self.node_last_seen_epoch_seconds - .insert(scope.clone(), seen_at); - if let Some(descriptor) = self.node_descriptors.get_mut(scope) { + .insert(node.clone(), seen_at); + if let Some(descriptor) = self.node_descriptors.get_mut(node) { descriptor.online = true; } Ok(()) diff --git a/crates/clusterflux-coordinator/src/service/panels.rs b/crates/clusterflux-coordinator/src/service/panels.rs index 9dd6a9a..4e66042 100644 --- a/crates/clusterflux-coordinator/src/service/panels.rs +++ b/crates/clusterflux-coordinator/src/service/panels.rs @@ -254,8 +254,7 @@ impl CoordinatorService { .rev() .find_map(|event| event.artifact_path.as_ref()) { - let artifact = artifact_id_from_path(path) - .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?; + let artifact = artifact_id_from_path(path); let context = clusterflux_core::AuthContext { tenant, project, diff --git a/crates/clusterflux-coordinator/src/service/process_launch.rs b/crates/clusterflux-coordinator/src/service/process_launch.rs index 81737c3..0a0a155 100644 --- a/crates/clusterflux-coordinator/src/service/process_launch.rs +++ b/crates/clusterflux-coordinator/src/service/process_launch.rs @@ -118,14 +118,14 @@ impl CoordinatorService { let mut objects = BTreeMap::new(); let mut missing_required_artifact = false; for artifact in &task_spec.required_artifacts { - let Some(metadata) = - self.artifact_registry - .metadata(&assignment.tenant, &assignment.project, artifact) - else { + let Some(metadata) = self.artifact_registry.metadata(artifact) else { missing_required_artifact = true; continue; }; - if metadata.retaining_nodes.is_empty() { + if metadata.tenant != assignment.tenant + || metadata.project != assignment.project + || metadata.retaining_nodes.is_empty() + { missing_required_artifact = true; continue; } @@ -483,14 +483,17 @@ impl CoordinatorService { .validate() .map_err(CoordinatorServiceError::Protocol)?; for artifact in &task_spec.required_artifacts { - let metadata = self - .artifact_registry - .metadata(&tenant, &project, artifact) - .ok_or_else(|| { - CoordinatorError::Unauthorized(format!( - "required artifact {artifact} is unavailable or has expired in this tenant/project scope" - )) - })?; + let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| { + CoordinatorError::Unauthorized(format!( + "required artifact {artifact} is unavailable or has expired" + )) + })?; + if metadata.tenant != tenant || metadata.project != project { + return Err(CoordinatorError::Unauthorized(format!( + "required artifact {artifact} is outside the task tenant/project scope" + )) + .into()); + } if metadata.retaining_nodes.is_empty() { return Err(CoordinatorError::Unauthorized(format!( "required artifact {artifact} has no retaining node" @@ -545,22 +548,13 @@ impl CoordinatorService { task_spec, wasm_module_base64, }); - let queued_tasks = self - .pending_task_launches - .iter() - .filter(|pending| { - pending.tenant == tenant - && pending.project == project - && pending.process == process - }) - .count(); return Ok(CoordinatorResponse::TaskQueued { process, task, actor, reason, charged_spawns, - queued_tasks, + queued_tasks: self.pending_task_launches.len(), }); } Err(err) => return Err(err.into()), @@ -671,9 +665,7 @@ impl CoordinatorService { )); }; self.task_attempts.remove(&removable); - self.task_terminal_states.remove(&removable); } - self.task_terminal_states.remove(&key); let attempts = self.task_attempts.entry(key).or_default(); for attempt in attempts.iter_mut() { attempt.current = false; diff --git a/crates/clusterflux-coordinator/src/service/processes.rs b/crates/clusterflux-coordinator/src/service/processes.rs index 646c607..7afaa44 100644 --- a/crates/clusterflux-coordinator/src/service/processes.rs +++ b/crates/clusterflux-coordinator/src/service/processes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId, }; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::keys::{process_control_key, task_control_key}; use super::{ @@ -47,10 +47,14 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "task assignment poll is outside the enrolled tenant/project scope".to_owned(), + ) + .into()); + } let assignment_key = (tenant.clone(), project.clone(), node.clone()); let assignment = self .task_assignments @@ -73,11 +77,7 @@ impl CoordinatorService { project: &ProjectId, node: &NodeId, ) -> Result, CoordinatorServiceError> { - let Some(descriptor) = self - .node_descriptors - .get(&NodeScopeKey::from_refs(tenant, project, node)) - .cloned() - else { + let Some(descriptor) = self.node_descriptors.get(node).cloned() else { return Ok(None); }; let mut remaining = VecDeque::new(); @@ -233,18 +233,20 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); - let descriptor = self - .node_descriptors - .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node)) - .ok_or_else(|| { - CoordinatorError::Unauthorized( - "source preparation completion requires a node capability report".to_owned(), - ) - })?; + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "source preparation completion is outside the enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } + let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| { + CoordinatorError::Unauthorized( + "source preparation completion requires a node capability report".to_owned(), + ) + })?; if !descriptor.source_snapshots.contains(&source_snapshot) && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND { @@ -382,10 +384,6 @@ impl CoordinatorService { || attempt_project != &project || attempt_process != &process }); - self.task_terminal_states - .retain(|(task_tenant, task_project, task_process, _), _| { - task_tenant != &tenant || task_project != &project || task_process != &process - }); self.restart_launches .retain(|(attempt_tenant, attempt_project, attempt_process, _)| { attempt_tenant != &tenant @@ -396,12 +394,11 @@ impl CoordinatorService { event.tenant != tenant || event.project != project || event.process != process }); let active = self.coordinator.start_process_for_launch_attempt( - tenant.clone(), - project.clone(), + tenant, + project, process.clone(), launch_attempt.map(clusterflux_core::LaunchAttemptId::new), ); - self.record_process_started(&tenant, &project, &process, now_epoch_seconds); Ok(CoordinatorResponse::ProcessStarted { process, launch_attempt: active @@ -416,18 +413,14 @@ impl CoordinatorService { pub(super) fn handle_reconnect_node( &mut self, - tenant: String, - project: String, node: String, process: String, epoch: u64, ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); let node = NodeId::new(node); let process = ProcessId::new(process); self.coordinator - .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?; + .reconnect_node(&node, Some((&process, epoch)))?; Ok(CoordinatorResponse::NodeReconnected { node, process }) } @@ -518,13 +511,6 @@ impl CoordinatorService { } let process_key = process_control_key(&tenant, &project, &process); if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) { - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Cancelled, - self.current_epoch_seconds()?, - ); self.coordinator .abort_process(&tenant, &project, &process)?; self.clear_operator_panel_state(&tenant, &project, &process); @@ -610,13 +596,6 @@ impl CoordinatorService { } } - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Cancelled, - self.current_epoch_seconds()?, - ); if let Some(launch_attempt) = launch_attempt.as_ref() { self.coordinator.abort_process_for_launch_attempt( &tenant, @@ -675,18 +654,10 @@ impl CoordinatorService { .map(|active| { let process_key = process_control_key(&active.tenant, &active.project, &active.id); let main = self.main_runtime.controls.get(&process_key); - let stored = self.process_summaries.get(&process_key); - let stored_main_state = stored - .and_then(|summary| summary.main_terminal_state.as_ref()) - .map(|state| match state { - super::TaskTerminalState::Completed => "completed", - super::TaskTerminalState::Failed => "failed", - super::TaskTerminalState::Cancelled => "cancelled", - }); let state = if self.process_cancellations.contains(&process_key) { "cancelling" } else { - "running" + main.map_or("running", |main| main.state.as_str()) }; let main_wait_state = main.and_then(|main| { if main.state != "running" { @@ -711,15 +682,9 @@ impl CoordinatorService { VirtualProcessStatus { process: active.id, state: state.to_owned(), - main_task_definition: main.map(|main| main.task_definition.clone()).or_else( - || stored.and_then(|summary| summary.main_task_definition.clone()), - ), - main_task_instance: main - .map(|main| main.task_instance.clone()) - .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())), - main_state: main - .map(|main| main.state.clone()) - .or_else(|| stored_main_state.map(str::to_owned)), + main_task_definition: main.map(|main| main.task_definition.clone()), + main_task_instance: main.map(|main| main.task_instance.clone()), + main_state: main.map(|main| main.state.clone()), main_wait_state, main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()), connected_nodes: active.connected_nodes.into_iter().collect(), diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index 61faf98..7b323a8 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -2,11 +2,10 @@ use std::collections::BTreeMap; use clusterflux_core::{ AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind, - DataPlaneObject, DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, - EnvironmentRequirements, LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, - NodeEndpoint, NodeId, NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, - ProjectId, ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryHandle, - TaskBoundaryValue, TaskDefinitionId, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, + DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements, + LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest, + PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation, + SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, UserId, VfsPath, }; use serde::{Deserialize, Serialize}; @@ -121,8 +120,6 @@ pub enum CoordinatorRequest { enrollment_grant: String, }, NodeHeartbeat { - tenant: String, - project: String, node: String, #[serde(default)] node_signature: Option, @@ -150,15 +147,6 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, - ListNodeSummaries { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, RevokeNodeCredential { tenant: String, project: String, @@ -280,8 +268,6 @@ pub enum CoordinatorRequest { restart: bool, }, ReconnectNode { - tenant: String, - project: String, node: String, process: String, epoch: u64, @@ -312,15 +298,6 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, - ListProcessSummaries { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, QuotaStatus { tenant: String, project: String, @@ -361,8 +338,6 @@ pub enum CoordinatorRequest { project: String, actor_user: String, process: String, - #[serde(default)] - revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { @@ -447,19 +422,6 @@ pub enum CoordinatorRequest { stderr_truncated: bool, backpressured: bool, }, - ReportTaskLogChunk { - tenant: String, - project: String, - process: String, - node: String, - task: String, - stream: TaskLogStream, - offset: u64, - source_bytes: u64, - text: String, - #[serde(default)] - truncated: bool, - }, ReportVfsMetadata { tenant: String, project: String, @@ -509,18 +471,6 @@ pub enum CoordinatorRequest { actor_user: String, process: String, }, - ListRecentLogs { - tenant: String, - project: String, - actor_user: String, - process: String, - #[serde(default)] - task: Option, - #[serde(default)] - after_sequence: Option, - #[serde(default = "default_log_page_limit")] - limit: u32, - }, JoinTask { tenant: String, project: String, @@ -553,23 +503,6 @@ pub enum CoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, - ListArtifacts { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - process: Option, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, - GetArtifact { - tenant: String, - project: String, - actor_user: String, - artifact: String, - }, OpenArtifactDownloadStream { tenant: String, project: String, @@ -598,10 +531,6 @@ pub enum CoordinatorRequest { } impl CoordinatorRequest { - pub fn validate_external_identifiers(&self) -> Result<(), String> { - validate_coordinator_request(self, "request") - } - pub fn operation(&self) -> Result { serde_json::to_value(self) .map_err(|err| format!("failed to encode coordinator request operation: {err}")) @@ -609,1225 +538,6 @@ impl CoordinatorRequest { } } -fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Result<(), String> { - match request { - CoordinatorRequest::Ping => Ok(()), - CoordinatorRequest::Authenticated { - session_secret, - request, - } => { - validate_external_token(session_secret, &format!("{path}.session_secret"), 512)?; - validate_authenticated_request(request, &format!("{path}.request")) - } - CoordinatorRequest::AuthStatus { - tenant, - project, - actor_user, - } - | CoordinatorRequest::CreateProject { - tenant, - project, - actor_user, - .. - } - | CoordinatorRequest::SelectProject { - tenant, - project, - actor_user, - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_project(project, &format!("{path}.project"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListProjects { tenant, actor_user } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::AdminStatus { - tenant, - actor_user, - admin_nonce, - .. - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) - } - CoordinatorRequest::SuspendTenant { - tenant, - actor_user, - target_tenant, - admin_nonce, - .. - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_tenant(target_tenant, &format!("{path}.target_tenant"))?; - validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) - } - CoordinatorRequest::RegisterAgentPublicKey { - tenant, - project, - user, - agent, - public_key, - } - | CoordinatorRequest::RotateAgentPublicKey { - tenant, - project, - user, - agent, - public_key, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user"))?; - validate_agent(agent, &format!("{path}.agent"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - CoordinatorRequest::ListAgentPublicKeys { - tenant, - project, - user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user")) - } - CoordinatorRequest::RevokeAgentPublicKey { - tenant, - project, - user, - agent, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user"))?; - validate_agent(agent, &format!("{path}.agent")) - } - CoordinatorRequest::AttachNode { - tenant, - project, - node, - public_key, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - CoordinatorRequest::CreateNodeEnrollmentGrant { - tenant, - project, - actor_user, - .. - } - | CoordinatorRequest::ListNodeDescriptors { - tenant, - project, - actor_user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListNodeSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::ExchangeNodeEnrollmentGrant { - tenant, - project, - node, - public_key, - enrollment_grant, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024)?; - validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512) - } - CoordinatorRequest::NodeHeartbeat { - tenant, - project, - node, - node_signature, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - if let Some(signature) = node_signature { - validate_node_signature(signature, &format!("{path}.node_signature"))?; - } - Ok(()) - } - CoordinatorRequest::SignedNode { - node, - node_signature, - request, - } => { - validate_node(node, &format!("{path}.node"))?; - validate_node_signature(node_signature, &format!("{path}.node_signature"))?; - validate_coordinator_request(request, &format!("{path}.request")) - } - CoordinatorRequest::ReportNodeCapabilities { - tenant, - project, - node, - artifact_locations, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_artifact_array(artifact_locations, &format!("{path}.artifact_locations")) - } - CoordinatorRequest::RevokeNodeCredential { - tenant, - project, - actor_user, - node, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_node(node, &format!("{path}.node")) - } - CoordinatorRequest::ScheduleTask { - tenant, - project, - required_artifacts, - prefer_node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; - validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) - } - CoordinatorRequest::LaunchTask { - tenant, - project, - actor_user, - actor_agent, - agent_signature, - task_spec, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; - validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; - if let Some(signature) = agent_signature { - validate_agent_signature(signature, &format!("{path}.agent_signature"))?; - } - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - CoordinatorRequest::LaunchChildTask { - tenant, - project, - process, - node, - parent_task, - task_spec, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(parent_task, &format!("{path}.parent_task"))?; - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - CoordinatorRequest::JoinChildTask { - tenant, - project, - process, - node, - parent_task, - task, - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(parent_task, &format!("{path}.parent_task"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::PollTaskAssignment { - tenant, - project, - node, - } - | CoordinatorRequest::PollArtifactTransfer { - tenant, - project, - node, - } - | CoordinatorRequest::CompleteSourcePreparation { - tenant, - project, - node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node")) - } - CoordinatorRequest::UploadArtifactTransferChunk { - tenant, - project, - node, - transfer_id, - artifact, - .. - } - | CoordinatorRequest::FailArtifactTransfer { - tenant, - project, - node, - transfer_id, - artifact, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(transfer_id, &format!("{path}.transfer_id"), 256)?; - validate_artifact(artifact, &format!("{path}.artifact")) - } - CoordinatorRequest::RequestRendezvous { - scope, - source, - destination, - .. - } => { - validate_data_plane_scope(scope, &format!("{path}.scope"))?; - validate_node_endpoint(source, &format!("{path}.source"))?; - validate_node_endpoint(destination, &format!("{path}.destination")) - } - CoordinatorRequest::RequestSourcePreparation { - tenant, project, .. - } => validate_tenant_project(tenant, project, path), - CoordinatorRequest::StartProcess { - tenant, - project, - actor_user, - actor_agent, - agent_signature, - process, - launch_attempt, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; - validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; - if let Some(signature) = agent_signature { - validate_agent_signature(signature, &format!("{path}.agent_signature"))?; - } - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - CoordinatorRequest::ReconnectNode { - tenant, - project, - node, - process, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_process(process, &format!("{path}.process")) - } - CoordinatorRequest::CancelTask { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::PollTaskControl { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::PollDebugCommand { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::ReportDebugProbeHit { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportTaskLog { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportVfsMetadata { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::TaskCompleted { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportDebugState { - tenant, - project, - process, - node, - task, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::CancelProcess { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::DebugAttach { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::SetDebugBreakpoints { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::InspectDebugBreakpoints { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::ResumeDebugEpoch { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::InspectDebugEpoch { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::ListTaskSnapshots { - tenant, - project, - actor_user, - process, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process")) - } - CoordinatorRequest::ListRecentLogs { - tenant, - project, - actor_user, - process, - task, - limit, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - if let Some(task) = task { - validate_task_instance(task, &format!("{path}.task"))?; - } - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::AbortProcess { - tenant, - project, - actor_user, - process, - launch_attempt, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - CoordinatorRequest::ListProcesses { - tenant, - project, - actor_user, - } - | CoordinatorRequest::QuotaStatus { - tenant, - project, - actor_user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListProcessSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 100) - } - CoordinatorRequest::RestartTask { - tenant, - project, - actor_user, - process, - task, - .. - } - | CoordinatorRequest::ResolveTaskFailure { - tenant, - project, - actor_user, - process, - task, - .. - } - | CoordinatorRequest::JoinTask { - tenant, - project, - actor_user, - process, - task, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::CreateDebugEpoch { - tenant, - project, - actor_user, - process, - stopped_task, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(stopped_task, &format!("{path}.stopped_task")) - } - CoordinatorRequest::ListTaskEvents { - tenant, - project, - actor_user, - process, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_process(process.as_deref(), &format!("{path}.process")) - } - CoordinatorRequest::RenderOperatorPanel { - tenant, - project, - process, - actor_user, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::SubmitPanelEvent { - tenant, - project, - process, - widget_id, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_external_token(widget_id, &format!("{path}.widget_id"), 256) - } - CoordinatorRequest::ListArtifacts { - tenant, - project, - actor_user, - process, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_process(process.as_deref(), &format!("{path}.process"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::CreateArtifactDownloadLink { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::OpenArtifactDownloadStream { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::RevokeArtifactDownloadLink { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::GetArtifact { - tenant, - project, - actor_user, - artifact, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_artifact(artifact, &format!("{path}.artifact")) - } - CoordinatorRequest::ExportArtifactToNode { - tenant, - project, - actor_user, - artifact, - receiver_node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_artifact(artifact, &format!("{path}.artifact"))?; - validate_node(receiver_node, &format!("{path}.receiver_node")) - } - } -} - -fn validate_authenticated_request( - request: &AuthenticatedCoordinatorRequest, - path: &str, -) -> Result<(), String> { - match request { - AuthenticatedCoordinatorRequest::AuthStatus - | AuthenticatedCoordinatorRequest::RevokeCliSession - | AuthenticatedCoordinatorRequest::ListProjects - | AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } - | AuthenticatedCoordinatorRequest::ListNodeDescriptors - | AuthenticatedCoordinatorRequest::ListProcesses - | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), - AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => { - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => { - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 100) - } - AuthenticatedCoordinatorRequest::CreateProject { project, .. } - | AuthenticatedCoordinatorRequest::SelectProject { project } => { - validate_project(project, &format!("{path}.project")) - } - AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { - agent, public_key, .. - } - | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { - agent, public_key, .. - } => { - validate_agent(agent, &format!("{path}.agent"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Ok(()), - AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => { - validate_agent(agent, &format!("{path}.agent")) - } - AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => { - validate_node(node, &format!("{path}.node")) - } - AuthenticatedCoordinatorRequest::StartProcess { - process, - launch_attempt, - .. - } - | AuthenticatedCoordinatorRequest::AbortProcess { - process, - launch_attempt, - } => { - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - AuthenticatedCoordinatorRequest::ScheduleTask { - required_artifacts, - prefer_node, - .. - } => { - validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; - validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) - } - AuthenticatedCoordinatorRequest::LaunchTask { task_spec, .. } => { - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - AuthenticatedCoordinatorRequest::CancelProcess { process } - | AuthenticatedCoordinatorRequest::DebugAttach { process } - | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, .. } - | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } - | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, .. } - | AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, .. } - | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { - validate_process(process, &format!("{path}.process")) - } - AuthenticatedCoordinatorRequest::ListRecentLogs { - process, - task, - limit, - .. - } => { - validate_process(process, &format!("{path}.process"))?; - if let Some(task) = task { - validate_task_instance(task, &format!("{path}.task"))?; - } - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } - | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } - | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(task, &format!("{path}.task")) - } - AuthenticatedCoordinatorRequest::CreateDebugEpoch { - process, - stopped_task, - .. - } => { - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(stopped_task, &format!("{path}.stopped_task")) - } - AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { - validate_optional_process(process.as_deref(), &format!("{path}.process")) - } - AuthenticatedCoordinatorRequest::ListArtifacts { - process, - cursor, - limit, - } => { - validate_optional_process(process.as_deref(), &format!("{path}.process"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } - | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } - | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } - | AuthenticatedCoordinatorRequest::GetArtifact { artifact } => { - validate_artifact(artifact, &format!("{path}.artifact")) - } - AuthenticatedCoordinatorRequest::ExportArtifactToNode { - artifact, - receiver_node, - .. - } => { - validate_artifact(artifact, &format!("{path}.artifact"))?; - validate_node(receiver_node, &format!("{path}.receiver_node")) - } - } -} - -fn validate_task_spec(task_spec: &TaskSpec, path: &str) -> Result<(), String> { - validate_existing_id( - &task_spec.tenant, - &format!("{path}.tenant"), - TenantId::try_new, - )?; - validate_existing_id( - &task_spec.project, - &format!("{path}.project"), - ProjectId::try_new, - )?; - validate_existing_id( - &task_spec.process, - &format!("{path}.process"), - ProcessId::try_new, - )?; - validate_existing_id( - &task_spec.task_definition, - &format!("{path}.task_definition"), - TaskDefinitionId::try_new, - )?; - validate_existing_id( - &task_spec.task_instance, - &format!("{path}.task_instance"), - TaskInstanceId::try_new, - )?; - if let Some(environment_id) = &task_spec.environment_id { - validate_external_token(environment_id, &format!("{path}.environment_id"), 128)?; - } - for (index, artifact) in task_spec.required_artifacts.iter().enumerate() { - validate_existing_id( - artifact, - &format!("{path}.required_artifacts[{index}]"), - ArtifactId::try_new, - )?; - } - for (argument_index, argument) in task_spec.args.iter().enumerate() { - validate_task_boundary_value(argument, &format!("{path}.args[{argument_index}]"))?; - } - Ok(()) -} - -fn validate_task_boundary_value(value: &TaskBoundaryValue, path: &str) -> Result<(), String> { - match value { - TaskBoundaryValue::Artifact(artifact) => validate_existing_id( - &artifact.id, - &format!("{path}.artifact.id"), - ArtifactId::try_new, - ), - TaskBoundaryValue::Structured(structured) => { - for (index, handle) in structured.handles.iter().enumerate() { - if let TaskBoundaryHandle::Artifact(artifact) = handle { - validate_existing_id( - &artifact.id, - &format!("{path}.handles[{index}].artifact.id"), - ArtifactId::try_new, - )?; - } - } - Ok(()) - } - TaskBoundaryValue::SmallJson(_) - | TaskBoundaryValue::SourceSnapshot(_) - | TaskBoundaryValue::Blob(_) - | TaskBoundaryValue::VfsManifest(_) => Ok(()), - } -} - -fn validate_data_plane_scope(scope: &DataPlaneScope, path: &str) -> Result<(), String> { - validate_existing_id(&scope.tenant, &format!("{path}.tenant"), TenantId::try_new)?; - validate_existing_id( - &scope.project, - &format!("{path}.project"), - ProjectId::try_new, - )?; - validate_existing_id( - &scope.process, - &format!("{path}.process"), - ProcessId::try_new, - )?; - if let DataPlaneObject::Artifact(artifact) = &scope.object { - validate_existing_id( - artifact, - &format!("{path}.object.artifact"), - ArtifactId::try_new, - )?; - } - validate_external_token( - &scope.authorization_subject, - &format!("{path}.authorization_subject"), - 512, - ) -} - -fn validate_node_endpoint(endpoint: &NodeEndpoint, path: &str) -> Result<(), String> { - validate_existing_id(&endpoint.node, &format!("{path}.node"), NodeId::try_new)?; - validate_external_token( - &endpoint.advertised_addr, - &format!("{path}.advertised_addr"), - 512, - ) -} - -fn validate_node_signature(signature: &NodeSignedRequest, path: &str) -> Result<(), String> { - validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; - validate_external_token(&signature.signature, &format!("{path}.signature"), 512) -} - -fn validate_agent_signature(signature: &AgentSignedRequest, path: &str) -> Result<(), String> { - validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; - validate_external_token(&signature.signature, &format!("{path}.signature"), 512) -} - -fn validate_tenant_project(tenant: &str, project: &str, path: &str) -> Result<(), String> { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_project(project, &format!("{path}.project")) -} - -fn validate_artifact_array(values: &[String], path: &str) -> Result<(), String> { - for (index, value) in values.iter().enumerate() { - validate_artifact(value, &format!("{path}[{index}]"))?; - } - Ok(()) -} - -fn validate_optional_user(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_user(value, path)) -} - -fn validate_optional_agent(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_agent(value, path)) -} - -fn validate_optional_node(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_node(value, path)) -} - -fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_process(value, path)) -} - -fn validate_optional_cursor(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_external_token(value, path, 256)) -} - -fn validate_page_limit(value: u32, path: &str, maximum: u32) -> Result<(), String> { - if value == 0 || value > maximum { - return Err(format!( - "malformed pagination limit {path}: expected 1 through {maximum}, received {value}" - )); - } - Ok(()) -} - -fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) -} - -fn validate_tenant(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, TenantId::try_new) -} - -fn validate_project(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ProjectId::try_new) -} - -fn validate_user(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, UserId::try_new) -} - -fn validate_agent(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, AgentId::try_new) -} - -fn validate_node(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, NodeId::try_new) -} - -fn validate_process(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ProcessId::try_new) -} - -fn validate_task_instance(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, TaskInstanceId::try_new) -} - -fn validate_artifact(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ArtifactId::try_new) -} - -fn validate_launch_attempt(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, LaunchAttemptId::try_new) -} - -fn validate_external_id( - value: &str, - path: &str, - parser: impl FnOnce(String) -> Result, -) -> Result<(), String> -where - E: std::fmt::Display, -{ - parser(value.to_owned()) - .map(|_| ()) - .map_err(|error| format!("malformed external identifier {path}: {error}")) -} - -fn validate_existing_id( - value: &T, - path: &str, - parser: impl FnOnce(String) -> Result, -) -> Result<(), String> -where - T: std::fmt::Display, - E: std::fmt::Display, -{ - validate_external_id(&value.to_string(), path, parser) -} - -fn validate_external_token(value: &str, path: &str, max_bytes: usize) -> Result<(), String> { - clusterflux_core::validate_opaque_token(value, max_bytes) - .map_err(|error| format!("malformed external token {path}: {error}")) -} - -#[cfg(test)] -mod external_identifier_tests { - use super::*; - - fn valid_task_spec() -> TaskSpec { - TaskSpec { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - task_definition: TaskDefinitionId::from("compile"), - task_instance: TaskInstanceId::from("compile-1"), - dispatch: clusterflux_core::TaskDispatch::CoordinatorNodeWasm { - export: Some("compile".to_owned()), - abi: clusterflux_core::WasmExportAbi::TaskV1, - }, - environment_id: Some("linux-rootless".to_owned()), - environment: None, - environment_digest: None, - required_capabilities: Default::default(), - dependency_cache: None, - source_snapshot: None, - required_artifacts: vec![ArtifactId::from("input-artifact")], - args: Vec::new(), - vfs_epoch: 1, - failure_policy: Default::default(), - bundle_digest: None, - } - } - - fn validation_error(value: serde_json::Value) -> String { - match serde_json::from_value::(value) { - Ok(request) => request - .validate_external_identifiers() - .expect_err("request should contain one malformed external identifier"), - Err(error) => error.to_string(), - } - } - - #[test] - fn nested_authenticated_identifiers_are_validated() { - let request = CoordinatorRequest::Authenticated { - session_secret: "cli_session_valid".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "bad process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - }, - }; - let error = request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("request.request.process")); - } - - #[test] - fn opaque_secrets_are_bounded_as_tokens_instead_of_object_ids() { - let request = CoordinatorRequest::Authenticated { - session_secret: "opaque secret/+==".to_owned(), - request: AuthenticatedCoordinatorRequest::AuthStatus, - }; - request.validate_external_identifiers().unwrap(); - - let mut request = request; - let CoordinatorRequest::Authenticated { session_secret, .. } = &mut request else { - unreachable!() - }; - *session_secret = "bad\0secret".to_owned(); - let error = request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.session_secret")); - } - - #[test] - fn real_protocol_variants_reject_exactly_one_malformed_nested_identifier() { - let launch = CoordinatorRequest::LaunchTask { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: Some("user".to_owned()), - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - task_spec: valid_task_spec(), - wait_for_node: false, - artifact_path: "/vfs/artifacts/output".to_owned(), - wasm_module_base64: "AGFzbQEAAAA=".to_owned(), - }; - - let mut malformed_definition = serde_json::to_value(&launch).unwrap(); - malformed_definition["task_spec"]["task_definition"] = - serde_json::Value::String("bad task definition!".to_owned()); - let error = validation_error(malformed_definition); - assert!(error.contains("TaskDefinitionId is invalid")); - - let mut malformed_artifact = serde_json::to_value(&launch).unwrap(); - malformed_artifact["task_spec"]["required_artifacts"][0] = - serde_json::Value::String("bad artifact!".to_owned()); - let error = validation_error(malformed_artifact); - assert!(error.contains("ArtifactId is invalid")); - - let rendezvous = CoordinatorRequest::RequestRendezvous { - scope: DataPlaneScope { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - object: DataPlaneObject::Artifact(ArtifactId::from("artifact")), - authorization_subject: "node-a-to-node-b".to_owned(), - }, - source: NodeEndpoint { - node: NodeId::from("node-a"), - advertised_addr: "node-a.invalid:4433".to_owned(), - public_key_fingerprint: Digest::sha256("node-a"), - }, - destination: NodeEndpoint { - node: NodeId::from("node-b"), - advertised_addr: "node-b.invalid:4433".to_owned(), - public_key_fingerprint: Digest::sha256("node-b"), - }, - direct_connectivity: true, - failure_reason: String::new(), - }; - let mut malformed_endpoint = serde_json::to_value(rendezvous).unwrap(); - malformed_endpoint["destination"]["node"] = - serde_json::Value::String("bad node!".to_owned()); - let error = validation_error(malformed_endpoint); - assert!(error.contains("NodeId is invalid")); - } - - #[test] - fn signed_and_authenticated_real_variants_validate_scoped_ids_and_tokens() { - let cases = [ - CoordinatorRequest::Authenticated { - session_secret: "session-secret".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "bad process!".to_owned(), - launch_attempt: Some("attempt".to_owned()), - }, - }, - CoordinatorRequest::StartProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: Some("bad agent!".to_owned()), - agent_public_key_fingerprint: Some(Digest::sha256("agent")), - agent_signature: Some(AgentSignedRequest { - nonce: "agent-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - process: "process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - restart: false, - }, - CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "bad node!".to_owned(), - node_signature: Some(NodeSignedRequest { - nonce: "node-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - }, - CoordinatorRequest::SignedNode { - node: "node".to_owned(), - node_signature: NodeSignedRequest { - nonce: "node-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }, - request: Box::new(CoordinatorRequest::PollTaskAssignment { - tenant: "tenant".to_owned(), - project: "bad project!".to_owned(), - node: "node".to_owned(), - }), - }, - CoordinatorRequest::AbortProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - process: "process".to_owned(), - launch_attempt: Some("bad attempt!".to_owned()), - }, - ]; - - for request in cases { - let error = request.validate_external_identifiers().unwrap_err(); - assert!( - error.contains("malformed external identifier"), - "unexpected validation error for {request:?}: {error}" - ); - } - } - - #[test] - fn signed_request_nonces_are_validated_as_opaque_tokens() { - let agent_request = CoordinatorRequest::StartProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: Some("agent".to_owned()), - agent_public_key_fingerprint: Some(Digest::sha256("agent")), - agent_signature: Some(AgentSignedRequest { - nonce: String::new(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - process: "process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - restart: false, - }; - let error = agent_request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.agent_signature.nonce")); - - let node_request = CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - node_signature: Some(NodeSignedRequest { - nonce: String::new(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - }; - let error = node_request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.node_signature.nonce")); - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum AuthenticatedCoordinatorRequest { @@ -1858,12 +568,6 @@ pub enum AuthenticatedCoordinatorRequest { ttl_seconds: u64, }, ListNodeDescriptors, - ListNodeSummaries { - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, RevokeNodeCredential { node: String, }, @@ -1900,12 +604,6 @@ pub enum AuthenticatedCoordinatorRequest { launch_attempt: Option, }, ListProcesses, - ListProcessSummaries { - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, QuotaStatus, RestartTask { process: String, @@ -1923,8 +621,6 @@ pub enum AuthenticatedCoordinatorRequest { }, SetDebugBreakpoints { process: String, - #[serde(default)] - revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { @@ -1950,15 +646,6 @@ pub enum AuthenticatedCoordinatorRequest { ListTaskSnapshots { process: String, }, - ListRecentLogs { - process: String, - #[serde(default)] - task: Option, - #[serde(default)] - after_sequence: Option, - #[serde(default = "default_log_page_limit")] - limit: u32, - }, JoinTask { process: String, task: String, @@ -1969,17 +656,6 @@ pub enum AuthenticatedCoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, - ListArtifacts { - #[serde(default)] - process: Option, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, - GetArtifact { - artifact: String, - }, OpenArtifactDownloadStream { artifact: String, max_bytes: u64, @@ -2002,14 +678,6 @@ fn default_download_ttl_seconds() -> u64 { 900 } -fn default_page_limit() -> u32 { - 50 -} - -fn default_log_page_limit() -> u32 { - 100 -} - fn default_node_enrollment_ttl_seconds() -> u64 { 900 } diff --git a/crates/clusterflux-coordinator/src/service/protocol/responses.rs b/crates/clusterflux-coordinator/src/service/protocol/responses.rs index 8245536..54655b7 100644 --- a/crates/clusterflux-coordinator/src/service/protocol/responses.rs +++ b/crates/clusterflux-coordinator/src/service/protocol/responses.rs @@ -183,121 +183,6 @@ pub struct VirtualProcessStatus { pub coordinator_epoch: u64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeSummary { - pub id: NodeId, - pub display_name: String, - pub online: bool, - pub stale: bool, - pub last_seen_epoch_seconds: Option, - pub capabilities: NodeCapabilities, - pub direct_connectivity: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessLifecycleState { - Active, - RecentTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessActivityState { - Running, - WaitingForNode, - WaitingForTask, - AwaitingAction, - DebugEpochPartial, - Cancelling, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessFinalResult { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochSummary { - pub epoch: u64, - pub command: String, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessSummary { - pub process: ProcessId, - pub lifecycle: ProcessLifecycleState, - pub activity: ProcessActivityState, - pub main_wait_state: Option, - pub started_at_epoch_seconds: u64, - pub ended_at_epoch_seconds: Option, - pub final_result: Option, - pub connected_nodes: Vec, - pub current_debug_epoch: Option, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactAvailability { - Available, - NodeOffline, - Unavailable, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactRetentionState { - NodeRetained, - ExplicitStorage, - Lost, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactSummary { - pub id: ArtifactId, - pub display_path: String, - pub display_name: String, - pub process: ProcessId, - pub producer_task: TaskInstanceId, - pub safe_node: Option, - pub digest: Digest, - pub size_bytes: u64, - pub availability: ArtifactAvailability, - pub downloadable_now: bool, - pub retention_state: ArtifactRetentionState, - pub explicit_storage: bool, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskLogStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogEntry { - pub sequence: u64, - pub process: ProcessId, - pub task: TaskInstanceId, - pub stream: TaskLogStream, - pub text: String, - pub server_timestamp_epoch_seconds: u64, - pub truncated: bool, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum SourcePreparationDisposition { Pending { reason: String }, @@ -328,7 +213,7 @@ pub enum CoordinatorResponse { manual_review: bool, sanitized_reason: Option, next_actions: Vec, - sensitive_moderation_details_exposed: bool, + private_moderation_details_exposed: bool, signup_failure_details_exposed: bool, }, AdminStatus { @@ -397,11 +282,6 @@ pub enum CoordinatorResponse { descriptors: Vec, actor: UserId, }, - NodeSummaries { - nodes: Vec, - next_cursor: Option, - actor: UserId, - }, NodeCredentialRevoked { node: NodeId, tenant: TenantId, @@ -493,11 +373,6 @@ pub enum CoordinatorResponse { processes: Vec, actor: UserId, }, - ProcessSummaries { - processes: Vec, - next_cursor: Option, - actor: UserId, - }, QuotaStatus { tenant: TenantId, project: ProjectId, @@ -555,7 +430,6 @@ pub enum CoordinatorResponse { DebugBreakpoints { process: ProcessId, actor: UserId, - revision: u64, probe_symbols: Vec, hit_epoch: Option, hit_task: Option, @@ -608,17 +482,6 @@ pub enum CoordinatorResponse { stderr_tail: String, backpressured: bool, }, - TaskLogChunkRecorded { - process: ProcessId, - task: TaskInstanceId, - sequence: Option, - next_offset: u64, - }, - RecentLogs { - entries: Vec, - next_sequence: Option, - history_truncated: bool, - }, VfsMetadataRecorded { process: ProcessId, task: TaskInstanceId, @@ -655,13 +518,6 @@ pub enum CoordinatorResponse { ArtifactDownloadLink { link: DownloadLink, }, - Artifacts { - artifacts: Vec, - next_cursor: Option, - }, - Artifact { - artifact: ArtifactSummary, - }, ArtifactDownloadLinkRevoked { link: DownloadLink, }, @@ -686,24 +542,6 @@ pub enum CoordinatorResponse { artifact_size_bytes: u64, }, Error { - #[serde(flatten)] - error: clusterflux_core::ApiError, + message: String, }, } - -impl CoordinatorResponse { - pub fn error(request_id: impl Into, message: impl Into) -> Self { - Self::Error { - error: clusterflux_core::ApiError::from_message(request_id, message), - } - } - - pub fn service_error( - request_id: impl Into, - error: &crate::service::CoordinatorServiceError, - ) -> Self { - Self::Error { - error: error.api_error(request_id), - } - } -} diff --git a/crates/clusterflux-coordinator/src/service/quota.rs b/crates/clusterflux-coordinator/src/service/quota.rs index 78b65ae..6fcd90f 100644 --- a/crates/clusterflux-coordinator/src/service/quota.rs +++ b/crates/clusterflux-coordinator/src/service/quota.rs @@ -260,6 +260,22 @@ impl CoordinatorQuota { self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds) } + pub(super) fn can_charge_log_bytes( + &self, + tenant: &TenantId, + project: &ProjectId, + bytes: u64, + now_epoch_seconds: u64, + ) -> Result<(), LimitError> { + self.can_charge( + tenant, + project, + LimitKind::LogBytes, + bytes, + now_epoch_seconds, + ) + } + pub(super) fn charge_log_bytes( &mut self, tenant: &TenantId, diff --git a/crates/clusterflux-coordinator/src/service/relay.rs b/crates/clusterflux-coordinator/src/service/relay.rs index b79db3d..cbe92dd 100644 --- a/crates/clusterflux-coordinator/src/service/relay.rs +++ b/crates/clusterflux-coordinator/src/service/relay.rs @@ -65,16 +65,6 @@ pub struct ArtifactRelayUsage { pub egress_bytes: u64, pub abandoned_or_failed_bytes: u64, pub reserved_bytes: u64, - pub lifetime_ingress_bytes: u64, - pub lifetime_egress_bytes: u64, - pub lifetime_abandoned_or_failed_bytes: u64, - pub completed_transfers: u64, - pub failed_transfers: u64, - pub cancelled_transfers: u64, - pub expired_transfers: u64, - pub tracked_scopes: usize, - pub max_active_global: usize, - pub max_tracked_scopes: usize, } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -87,20 +77,6 @@ pub struct ArtifactRelayDurableState { pub ingress_used: u64, pub egress_used: u64, pub abandoned_or_failed_used: u64, - #[serde(default)] - pub lifetime_ingress_bytes: u64, - #[serde(default)] - pub lifetime_egress_bytes: u64, - #[serde(default)] - pub lifetime_abandoned_or_failed_bytes: u64, - #[serde(default)] - pub completed_transfers: u64, - #[serde(default)] - pub failed_transfers: u64, - #[serde(default)] - pub cancelled_transfers: u64, - #[serde(default)] - pub expired_transfers: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -169,13 +145,6 @@ pub(super) struct ArtifactRelayLedger { ingress_used: u64, egress_used: u64, abandoned_or_failed_used: u64, - lifetime_ingress_bytes: u64, - lifetime_egress_bytes: u64, - lifetime_abandoned_or_failed_bytes: u64, - completed_transfers: u64, - failed_transfers: u64, - cancelled_transfers: u64, - expired_transfers: u64, } impl Default for ArtifactRelayLedger { @@ -196,13 +165,6 @@ impl ArtifactRelayLedger { ingress_used: 0, egress_used: 0, abandoned_or_failed_used: 0, - lifetime_ingress_bytes: 0, - lifetime_egress_bytes: 0, - lifetime_abandoned_or_failed_bytes: 0, - completed_transfers: 0, - failed_transfers: 0, - cancelled_transfers: 0, - expired_transfers: 0, } } @@ -256,13 +218,6 @@ impl ArtifactRelayLedger { ingress_used: state.ingress_used, egress_used: state.egress_used, abandoned_or_failed_used: state.abandoned_or_failed_used, - lifetime_ingress_bytes: state.lifetime_ingress_bytes, - lifetime_egress_bytes: state.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes, - completed_transfers: state.completed_transfers, - failed_transfers: state.failed_transfers, - cancelled_transfers: state.cancelled_transfers, - expired_transfers: state.expired_transfers, } } @@ -305,13 +260,6 @@ impl ArtifactRelayLedger { ingress_used: self.ingress_used, egress_used: self.egress_used, abandoned_or_failed_used: self.abandoned_or_failed_used, - lifetime_ingress_bytes: self.lifetime_ingress_bytes, - lifetime_egress_bytes: self.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, - completed_transfers: self.completed_transfers, - failed_transfers: self.failed_transfers, - cancelled_transfers: self.cancelled_transfers, - expired_transfers: self.expired_transfers, } } @@ -543,11 +491,9 @@ impl ArtifactRelayLedger { if ingress { reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes); self.ingress_used = self.ingress_used.saturating_add(bytes); - self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes); } else { reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes); self.egress_used = self.egress_used.saturating_add(bytes); - self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes); } let project_key = ( reservation.scope.tenant.clone(), @@ -608,29 +554,10 @@ impl ArtifactRelayLedger { pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) { if let Some(reservation) = self.reservations.remove(key) { if reason != RelayFinishReason::Completed { - let abandoned_or_failed_bytes = reservation - .ingress_bytes - .saturating_add(reservation.egress_bytes); self.abandoned_or_failed_used = self .abandoned_or_failed_used - .saturating_add(abandoned_or_failed_bytes); - self.lifetime_abandoned_or_failed_bytes = self - .lifetime_abandoned_or_failed_bytes - .saturating_add(abandoned_or_failed_bytes); - } - match reason { - RelayFinishReason::Completed => { - self.completed_transfers = self.completed_transfers.saturating_add(1); - } - RelayFinishReason::Failed => { - self.failed_transfers = self.failed_transfers.saturating_add(1); - } - RelayFinishReason::Cancelled => { - self.cancelled_transfers = self.cancelled_transfers.saturating_add(1); - } - RelayFinishReason::Expired => { - self.expired_transfers = self.expired_transfers.saturating_add(1); - } + .saturating_add(reservation.ingress_bytes) + .saturating_add(reservation.egress_bytes); } } } @@ -653,18 +580,6 @@ impl ArtifactRelayLedger { ingress_bytes: self.ingress_used, egress_bytes: self.egress_used, abandoned_or_failed_bytes: self.abandoned_or_failed_used, - lifetime_ingress_bytes: self.lifetime_ingress_bytes, - lifetime_egress_bytes: self.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, - completed_transfers: self.completed_transfers, - failed_transfers: self.failed_transfers, - cancelled_transfers: self.cancelled_transfers, - expired_transfers: self.expired_transfers, - tracked_scopes: self.project_used.len() - + self.tenant_used.len() - + self.account_used.len(), - max_active_global: self.configuration.max_active_global, - max_tracked_scopes: self.configuration.max_tracked_scopes, reserved_bytes: self .reservations .values() @@ -673,68 +588,3 @@ impl ArtifactRelayLedger { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() { - let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited()); - ledger - .reserve( - "transfer".to_owned(), - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - 16, - 16, - 100, - 1, - ) - .unwrap(); - ledger.charge_ingress("transfer", 24, 1).unwrap(); - ledger.charge_egress("transfer", 24, 1).unwrap(); - ledger.finish("transfer", RelayFinishReason::Completed); - - let usage = ledger.usage(); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - assert_eq!(usage.completed_transfers, 1); - - ledger.prepare_period(u64::MAX); - let usage = ledger.usage(); - assert_eq!(usage.ingress_bytes, 0); - assert_eq!(usage.egress_bytes, 0); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - - let restored = ArtifactRelayLedger::from_durable( - ArtifactRelayConfiguration::unlimited(), - ledger.durable_state(), - ); - let usage = restored.usage(); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - assert_eq!(usage.completed_transfers, 1); - } - - #[test] - fn older_durable_relay_state_defaults_new_metrics() { - let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({ - "reservations": {}, - "period": 1, - "project_used": [], - "tenant_used": [], - "account_used": [], - "ingress_used": 3, - "egress_used": 4, - "abandoned_or_failed_used": 2 - })) - .unwrap(); - - assert_eq!(state.lifetime_ingress_bytes, 0); - assert_eq!(state.lifetime_egress_bytes, 0); - assert_eq!(state.completed_transfers, 0); - } -} diff --git a/crates/clusterflux-coordinator/src/service/routing.rs b/crates/clusterflux-coordinator/src/service/routing.rs index e05a670..abceab7 100644 --- a/crates/clusterflux-coordinator/src/service/routing.rs +++ b/crates/clusterflux-coordinator/src/service/routing.rs @@ -5,9 +5,6 @@ impl CoordinatorService { &mut self, request: CoordinatorRequest, ) -> Result { - request - .validate_external_identifiers() - .map_err(CoordinatorServiceError::Protocol)?; self.pump_main_runtime_commands(); let request_payload = serde_json::to_value(&request).map_err(|error| { CoordinatorServiceError::Protocol(format!( @@ -45,7 +42,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - sensitive_moderation_details_exposed: false, + private_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -274,17 +271,9 @@ impl CoordinatorService { enrollment_grant, ), CoordinatorRequest::NodeHeartbeat { - tenant, - project, node, node_signature, - } => self.handle_node_heartbeat( - tenant, - project, - node, - node_signature, - &request_payload_digest, - ), + } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest), CoordinatorRequest::SignedNode { node, node_signature, @@ -298,13 +287,6 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_node_descriptors(tenant, project, actor_user), - CoordinatorRequest::ListNodeSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::RevokeNodeCredential { tenant, project, @@ -428,13 +410,6 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_processes(tenant, project, actor_user), - CoordinatorRequest::ListProcessSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::QuotaStatus { tenant, project, @@ -480,8 +455,7 @@ impl CoordinatorService { CoordinatorRequest::PollDebugCommand { .. } | CoordinatorRequest::ReportDebugState { .. } | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(), - CoordinatorRequest::ReportTaskLog { .. } - | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(), + CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ListTaskEvents { @@ -496,23 +470,6 @@ impl CoordinatorService { actor_user, process, } => self.handle_list_task_snapshots(tenant, project, actor_user, process), - CoordinatorRequest::ListRecentLogs { - tenant, - project, - actor_user, - process, - task, - after_sequence, - limit, - } => self.handle_list_recent_logs( - tenant, - project, - actor_user, - process, - task, - after_sequence, - limit, - ), CoordinatorRequest::JoinTask { tenant, project, @@ -559,20 +516,6 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), - CoordinatorRequest::ListArtifacts { - tenant, - project, - actor_user, - process, - cursor, - limit, - } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit), - CoordinatorRequest::GetArtifact { - tenant, - project, - actor_user, - artifact, - } => self.handle_get_artifact(tenant, project, actor_user, artifact), CoordinatorRequest::OpenArtifactDownloadStream { tenant, project, @@ -655,7 +598,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - sensitive_moderation_details_exposed: false, + private_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -742,14 +685,6 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), - AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self - .handle_list_node_summaries( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - cursor, - limit, - ), AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self .handle_revoke_node_credential( context.tenant.as_str().to_owned(), @@ -801,14 +736,6 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), - AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self - .handle_list_process_summaries( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - cursor, - limit, - ), AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -864,20 +791,6 @@ impl CoordinatorService { actor.as_str().to_owned(), process, ), - AuthenticatedCoordinatorRequest::ListRecentLogs { - process, - task, - after_sequence, - limit, - } => self.handle_list_recent_logs( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - process, - task, - after_sequence, - limit, - ), AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -897,24 +810,6 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), - AuthenticatedCoordinatorRequest::ListArtifacts { - process, - cursor, - limit, - } => self.handle_list_artifacts( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - process, - cursor, - limit, - ), - AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - artifact, - ), AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, max_bytes, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index 40f633a..e6e3878 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -1,6 +1,6 @@ -use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId}; +use clusterflux_core::{NodeId, NodeSignedRequest}; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError}; @@ -12,24 +12,22 @@ impl CoordinatorService { request: CoordinatorRequest, ) -> Result { let request_kind = signed_node_request_kind(&request)?; - let request_scope = signed_node_request_scope(&request)?; + let request_node = signed_node_request_node(&request)?; let request_payload = serde_json::to_value(&request).map_err(|error| { CoordinatorServiceError::Protocol(format!( "failed to canonicalize signed node request: {error}" )) })?; let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload); - let signed_node = NodeId::try_new(signed_node).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}")) - })?; - if request_scope.node != signed_node { + let signed_node = NodeId::new(signed_node); + if request_node != signed_node { return Err(CoordinatorError::Unauthorized( "signed node request node does not match the wrapped request node".to_owned(), ) .into()); } self.authenticate_node_request( - &request_scope, + &signed_node, Some(node_signature), request_kind, &payload_digest, @@ -146,26 +144,11 @@ impl CoordinatorService { provider, source_snapshot, ), - CoordinatorRequest::RequestRendezvous { - scope, - source, - destination, - direct_connectivity, - failure_reason, - } => self.handle_request_rendezvous( - scope, - source, - destination, - direct_connectivity, - failure_reason, - ), CoordinatorRequest::ReconnectNode { - tenant, - project, node, process, epoch, - } => self.handle_reconnect_node(tenant, project, node, process, epoch), + } => self.handle_reconnect_node(node, process, epoch), CoordinatorRequest::PollTaskControl { tenant, project, @@ -253,29 +236,6 @@ impl CoordinatorService { stderr_truncated, backpressured, ), - CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - process, - node, - task, - stream, - offset, - source_bytes, - text, - truncated, - } => self.handle_report_task_log_chunk( - tenant, - project, - process, - node, - task, - stream, - offset, - source_bytes, - text, - truncated, - ), CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -362,14 +322,12 @@ fn signed_node_request_kind( CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"), CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"), CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"), - CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"), CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"), CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"), CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"), CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"), CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"), CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"), - CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"), CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"), CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"), _ => Err(CoordinatorError::Unauthorized( @@ -379,135 +337,29 @@ fn signed_node_request_kind( } } -fn signed_node_request_scope( +fn signed_node_request_node( request: &CoordinatorRequest, -) -> Result { +) -> Result { match request { - CoordinatorRequest::ReportNodeCapabilities { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollTaskAssignment { - tenant, - project, - node, - } - | CoordinatorRequest::PollArtifactTransfer { - tenant, - project, - node, - } - | CoordinatorRequest::UploadArtifactTransferChunk { - tenant, - project, - node, - .. - } - | CoordinatorRequest::FailArtifactTransfer { - tenant, - project, - node, - .. - } - | CoordinatorRequest::LaunchChildTask { - tenant, - project, - node, - .. - } - | CoordinatorRequest::JoinChildTask { - tenant, - project, - node, - .. - } - | CoordinatorRequest::CompleteSourcePreparation { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReconnectNode { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollTaskControl { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollDebugCommand { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportDebugState { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportDebugProbeHit { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportTaskLog { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportVfsMetadata { - tenant, - project, - node, - .. - } - | CoordinatorRequest::TaskCompleted { - tenant, - project, - node, - .. - } => node_scope_from_strings(tenant, project, node), - CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new( - scope.tenant.clone(), - scope.project.clone(), - source.node.clone(), - )), + CoordinatorRequest::ReportNodeCapabilities { node, .. } + | CoordinatorRequest::PollTaskAssignment { node, .. } + | CoordinatorRequest::PollArtifactTransfer { node, .. } + | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } + | CoordinatorRequest::FailArtifactTransfer { node, .. } + | CoordinatorRequest::LaunchChildTask { node, .. } + | CoordinatorRequest::JoinChildTask { node, .. } + | CoordinatorRequest::CompleteSourcePreparation { node, .. } + | CoordinatorRequest::ReconnectNode { node, .. } + | CoordinatorRequest::PollTaskControl { node, .. } + | CoordinatorRequest::PollDebugCommand { node, .. } + | CoordinatorRequest::ReportDebugState { node, .. } + | CoordinatorRequest::ReportDebugProbeHit { node, .. } + | CoordinatorRequest::ReportTaskLog { node, .. } + | CoordinatorRequest::ReportVfsMetadata { node, .. } + | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())), _ => Err(CoordinatorError::Unauthorized( "signed_node envelope only accepts node-originated coordinator requests".to_owned(), ) .into()), } } - -fn node_scope_from_strings( - tenant: &str, - project: &str, - node: &str, -) -> Result { - let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}")) - })?; - let project = ProjectId::try_new(project.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}")) - })?; - let node = NodeId::try_new(node.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}")) - })?; - Ok(NodeScopeKey::new(tenant, project, node)) -} diff --git a/crates/clusterflux-coordinator/src/service/summaries.rs b/crates/clusterflux-coordinator/src/service/summaries.rs deleted file mode 100644 index 1e5ad92..0000000 --- a/crates/clusterflux-coordinator/src/service/summaries.rs +++ /dev/null @@ -1,565 +0,0 @@ -use std::collections::BTreeSet; -use std::time::Instant; - -use clusterflux_core::{ - ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, - TenantId, UserId, -}; - -use super::keys::{process_control_key, ProcessControlKey}; -use super::{ - ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse, - CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary, - NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, - TaskAttemptState, -}; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(super) struct StoredProcessSummary { - pub(super) started_at_epoch_seconds: u64, - pub(super) ended_at_epoch_seconds: Option, - pub(super) final_result: Option, - pub(super) connected_nodes: Vec, - pub(super) main_task_definition: Option, - pub(super) main_task_instance: Option, - pub(super) main_terminal_state: Option, - pub(super) order: u64, -} - -impl CoordinatorService { - pub(super) fn handle_list_node_summaries( - &mut self, - tenant: String, - project: String, - actor_user: String, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let actor = UserId::new(actor_user); - let cursor = cursor.as_deref(); - let mut nodes = self - .node_descriptors - .iter() - .filter(|(scope, descriptor)| { - scope.tenant == tenant - && scope.project == project - && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor) - }) - .map(|(scope, descriptor)| { - let online = self.node_is_live(scope); - NodeSummary { - id: descriptor.id.clone(), - display_name: descriptor.id.as_str().to_owned(), - online, - stale: !online, - last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(), - capabilities: descriptor.capabilities.clone(), - direct_connectivity: descriptor.direct_connectivity, - } - }) - .collect::>(); - nodes.sort_by(|left, right| left.id.cmp(&right.id)); - let has_more = nodes.len() > limit as usize; - nodes.truncate(limit as usize); - let next_cursor = has_more - .then(|| nodes.last().map(|node| node.id.as_str().to_owned())) - .flatten(); - Ok(CoordinatorResponse::NodeSummaries { - nodes, - next_cursor, - actor, - }) - } - - pub(super) fn record_process_started( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - now_epoch_seconds: u64, - ) { - let key = process_control_key(tenant, project, process); - if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) { - logs.retain(|entry| &entry.process != process); - if logs.is_empty() { - self.recent_logs.remove(&(tenant.clone(), project.clone())); - } - } - self.recent_log_dropped_through.remove(&key); - self.recent_log_accounted_bytes.retain( - |(entry_tenant, entry_project, entry_process, _, _), _| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.recent_log_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, _, _)| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.recent_log_quota_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, _, _)| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.process_summary_order - .retain(|retained| retained != &key); - self.evict_process_summaries_for_project(tenant, project); - self.evict_process_summaries_total(); - let order = self.next_process_summary_order; - self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); - self.process_summaries.insert( - key.clone(), - StoredProcessSummary { - started_at_epoch_seconds: now_epoch_seconds, - ended_at_epoch_seconds: None, - final_result: None, - connected_nodes: Vec::new(), - main_task_definition: None, - main_task_instance: None, - main_terminal_state: None, - order, - }, - ); - self.process_summary_order.push_back(key); - } - - pub(super) fn record_process_terminal( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - final_result: ProcessFinalResult, - now_epoch_seconds: u64, - ) { - let key = process_control_key(tenant, project, process); - let connected_nodes = self - .coordinator - .active_process(tenant, project, process) - .map(|active| active.connected_nodes.iter().cloned().collect()) - .unwrap_or_default(); - let order = self.next_process_summary_order; - let entry = self - .process_summaries - .entry(key.clone()) - .or_insert_with(|| { - self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); - self.process_summary_order.push_back(key.clone()); - StoredProcessSummary { - started_at_epoch_seconds: now_epoch_seconds, - ended_at_epoch_seconds: None, - final_result: None, - connected_nodes: Vec::new(), - main_task_definition: None, - main_task_instance: None, - main_terminal_state: None, - order, - } - }); - entry.ended_at_epoch_seconds = Some(now_epoch_seconds); - entry.final_result = Some(final_result); - entry.connected_nodes = connected_nodes; - } - - pub(super) fn record_main_terminal_state( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task_definition: TaskDefinitionId, - task_instance: TaskInstanceId, - terminal_state: super::TaskTerminalState, - ) { - let key = process_control_key(tenant, project, process); - if !self.process_summaries.contains_key(&key) { - self.record_process_started( - tenant, - project, - process, - self.liveness_now_epoch_seconds(), - ); - } - if let Some(summary) = self.process_summaries.get_mut(&key) { - summary.main_task_definition = Some(task_definition); - summary.main_task_instance = Some(task_instance); - summary.main_terminal_state = Some(terminal_state); - } - } - - pub(super) fn handle_list_process_summaries( - &mut self, - tenant: String, - project: String, - actor_user: String, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let actor = UserId::new(actor_user); - let cursor = parse_order_cursor(cursor.as_deref(), "process")?; - let mut stored = self - .process_summaries - .iter() - .filter(|((entry_tenant, entry_project, _), summary)| { - entry_tenant == &tenant - && entry_project == &project - && cursor.is_none_or(|cursor| summary.order < cursor) - }) - .map(|(key, summary)| (key.clone(), summary.clone())) - .collect::>(); - stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order)); - let has_more = stored.len() > limit as usize; - stored.truncate(limit as usize); - let processes = stored - .into_iter() - .map(|(key, stored)| self.process_summary_from_stored(&key, stored)) - .collect::>(); - let next_cursor = has_more - .then(|| processes.last().map(|process| process.order_cursor.clone())) - .flatten(); - Ok(CoordinatorResponse::ProcessSummaries { - processes, - next_cursor, - actor, - }) - } - - fn process_summary_from_stored( - &self, - key: &ProcessControlKey, - stored: StoredProcessSummary, - ) -> ProcessSummary { - let (tenant, project, process) = key; - let active = self.coordinator.active_process(tenant, project, process); - let process_key = process_control_key(tenant, project, process); - let main_wait_state = active.and_then(|_| { - if self.pending_task_launches.iter().any(|pending| { - &pending.tenant == tenant - && &pending.project == project - && &pending.process == process - }) { - Some("waiting_for_node".to_owned()) - } else if self - .main_runtime - .is_waiting_for_task(tenant, project, process) - { - Some("waiting_for_task".to_owned()) - } else { - None - } - }); - let current_debug_epoch = self.debug_epoch_summary(&process_key); - let awaiting_action = self.task_attempts.iter().any( - |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { - attempt_tenant == tenant - && attempt_project == project - && attempt_process == process - && attempts.iter().any(|attempt| { - attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction - }) - }, - ); - let activity = if let Some(result) = &stored.final_result { - match result { - ProcessFinalResult::Completed => ProcessActivityState::Completed, - ProcessFinalResult::Failed => ProcessActivityState::Failed, - ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled, - } - } else if self.process_cancellations.contains(&process_key) { - ProcessActivityState::Cancelling - } else if current_debug_epoch - .as_ref() - .is_some_and(|epoch| epoch.partially_frozen) - { - ProcessActivityState::DebugEpochPartial - } else if awaiting_action { - ProcessActivityState::AwaitingAction - } else { - match main_wait_state.as_deref() { - Some("waiting_for_node") => ProcessActivityState::WaitingForNode, - Some("waiting_for_task") => ProcessActivityState::WaitingForTask, - _ => ProcessActivityState::Running, - } - }; - let connected_nodes = active - .map(|active| active.connected_nodes.iter().cloned().collect()) - .unwrap_or(stored.connected_nodes); - ProcessSummary { - process: process.clone(), - lifecycle: if active.is_some() { - ProcessLifecycleState::Active - } else { - ProcessLifecycleState::RecentTerminal - }, - activity, - main_wait_state, - started_at_epoch_seconds: stored.started_at_epoch_seconds, - ended_at_epoch_seconds: stored.ended_at_epoch_seconds, - final_result: stored.final_result, - connected_nodes, - current_debug_epoch, - order_cursor: format!("process:{}", stored.order), - } - } - - fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option { - let runtime = self.debug_epoch_runtime.get(key)?; - let acknowledgements = runtime.acknowledgements.values().collect::>(); - let all_acknowledged = !runtime.expected.is_empty() - && runtime - .expected - .iter() - .all(|participant| runtime.acknowledgements.contains_key(participant)); - let fully_frozen = runtime.command == "freeze" - && all_acknowledged - && acknowledgements - .iter() - .all(|ack| ack.state == DebugAcknowledgementState::Frozen); - let freeze_deadline_elapsed = - runtime.command == "freeze" && Instant::now() >= runtime.deadline; - let frozen_count = acknowledgements - .iter() - .filter(|ack| ack.state == DebugAcknowledgementState::Frozen) - .count(); - let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen; - let fully_resumed = runtime.command == "resume" - && all_acknowledged - && acknowledgements - .iter() - .all(|ack| ack.state == DebugAcknowledgementState::Running); - let failed = acknowledgements - .iter() - .any(|ack| ack.state == DebugAcknowledgementState::Failed) - || (freeze_deadline_elapsed - && runtime - .expected - .iter() - .any(|participant| !runtime.acknowledgements.contains_key(participant))); - Some(DebugEpochSummary { - epoch: runtime.epoch, - command: runtime.command.clone(), - fully_frozen, - partially_frozen, - fully_resumed, - failed, - }) - } - - pub(super) fn handle_list_artifacts( - &mut self, - tenant: String, - project: String, - actor_user: String, - process: Option, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let process = process.map(ProcessId::new); - if let Some(process) = &process { - self.authorize_task_event_process_scope(&tenant, &project, process)?; - } - let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?; - let mut metadata = self - .artifact_registry - .metadata_for_project(&tenant, &project) - .filter(|metadata| { - process - .as_ref() - .is_none_or(|process| &metadata.process == process) - && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor) - }) - .cloned() - .collect::>(); - metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch)); - let has_more = metadata.len() > limit as usize; - metadata.truncate(limit as usize); - let artifacts = metadata - .into_iter() - .map(|metadata| self.artifact_summary(metadata)) - .collect::>(); - let next_cursor = has_more - .then(|| { - artifacts - .last() - .map(|artifact| artifact.order_cursor.clone()) - }) - .flatten(); - Ok(CoordinatorResponse::Artifacts { - artifacts, - next_cursor, - }) - } - - pub(super) fn handle_get_artifact( - &mut self, - tenant: String, - project: String, - actor_user: String, - artifact: String, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let artifact = ArtifactId::new(artifact); - let metadata = self - .artifact_registry - .metadata(&tenant, &project, &artifact) - .cloned() - .ok_or(clusterflux_core::DownloadError::NotFound)?; - Ok(CoordinatorResponse::Artifact { - artifact: self.artifact_summary(metadata), - }) - } - - fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary { - let live_retaining_nodes = metadata - .retaining_nodes - .iter() - .filter(|node| { - self.node_is_live(&crate::NodeScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - node, - )) - }) - .cloned() - .collect::>(); - let safe_node = live_retaining_nodes - .iter() - .next() - .cloned() - .or_else(|| metadata.retaining_nodes.iter().next().cloned()); - let explicit_storage = !metadata.explicit_locations.is_empty(); - let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage; - let availability = if downloadable_now { - ArtifactAvailability::Available - } else if !metadata.retaining_nodes.is_empty() { - ArtifactAvailability::NodeOffline - } else { - ArtifactAvailability::Unavailable - }; - let retention_state = if explicit_storage { - ArtifactRetentionState::ExplicitStorage - } else if !metadata.retaining_nodes.is_empty() { - ArtifactRetentionState::NodeRetained - } else { - ArtifactRetentionState::Lost - }; - let display_suffix = metadata.id.as_str().replace(':', "/"); - let display_path = format!("/vfs/artifacts/{display_suffix}"); - let display_name = display_suffix - .rsplit('/') - .next() - .unwrap_or(metadata.id.as_str()) - .to_owned(); - ArtifactSummary { - id: metadata.id, - display_path, - display_name, - process: metadata.process, - producer_task: metadata.producer_task, - safe_node, - digest: metadata.digest, - size_bytes: metadata.size, - availability, - downloadable_now, - retention_state, - explicit_storage, - order_cursor: format!("artifact:{}", metadata.flushed_epoch), - } - } - - fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) { - while self - .process_summaries - .keys() - .filter(|(entry_tenant, entry_project, _)| { - entry_tenant == tenant && entry_project == project - }) - .count() - >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT - { - let candidate = self.process_summary_order.iter().find(|key| { - &key.0 == tenant - && &key.1 == project - && self - .process_summaries - .get(*key) - .is_some_and(|summary| summary.final_result.is_some()) - }); - let Some(candidate) = candidate.cloned() else { - break; - }; - self.remove_process_summary_state(&candidate); - } - } - - fn evict_process_summaries_total(&mut self) { - while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL { - let candidate = self.process_summary_order.iter().find(|key| { - self.process_summaries - .get(*key) - .is_some_and(|summary| summary.final_result.is_some()) - }); - let Some(candidate) = candidate.cloned() else { - break; - }; - self.remove_process_summary_state(&candidate); - } - } - - fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) { - self.process_summaries.remove(key); - self.recent_log_dropped_through.remove(key); - if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) { - logs.retain(|entry| entry.process != key.2); - if logs.is_empty() { - self.recent_logs.remove(&(key.0.clone(), key.1.clone())); - } - } - self.recent_log_accounted_bytes - .retain(|(tenant, project, process, _, _), _| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.recent_log_truncated_streams - .retain(|(tenant, project, process, _, _)| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.recent_log_quota_truncated_streams - .retain(|(tenant, project, process, _, _)| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.process_summary_order - .retain(|retained| retained != key); - } -} - -fn parse_order_cursor( - cursor: Option<&str>, - expected_kind: &str, -) -> Result, CoordinatorServiceError> { - cursor - .map(|cursor| { - let (kind, order) = cursor.split_once(':').ok_or_else(|| { - CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - )) - })?; - if kind != expected_kind { - return Err(CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - ))); - } - order.parse::().map_err(|_| { - CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - )) - }) - }) - .transpose() -} diff --git a/crates/clusterflux-coordinator/src/service/tcp.rs b/crates/clusterflux-coordinator/src/service/tcp.rs index 2b6e9c9..d519f34 100644 --- a/crates/clusterflux-coordinator/src/service/tcp.rs +++ b/crates/clusterflux-coordinator/src/service/tcp.rs @@ -79,15 +79,17 @@ impl CoordinatorService { continue; } let response = match decode_wire_request(&line) { - Ok((request_id, request)) => { - match authorize_client_request(&request, authority_mode) - .and_then(|()| self.handle_request(request)) - { - Ok(response) => response, - Err(err) => CoordinatorResponse::service_error(request_id, &err), - } - } - Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), + Ok(request) => match authorize_client_request(&request, authority_mode) + .and_then(|()| self.handle_request(request)) + { + Ok(response) => response, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -112,19 +114,25 @@ fn handle_shared_stream( continue; } let response = match decode_wire_request(&line) { - Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) { + Ok(request) => match authorize_client_request(&request, authority_mode) { Ok(()) => match service.lock() { Ok(mut service) => match service.handle_request(request) { Ok(response) => response, - Err(err) => CoordinatorResponse::service_error(request_id, &err), + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(_) => CoordinatorResponse::Error { + message: "coordinator service lock poisoned".to_owned(), }, - Err(_) => { - CoordinatorResponse::error(request_id, "coordinator service lock poisoned") - } }, - Err(err) => CoordinatorResponse::service_error(request_id, &err), + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), }, - Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -138,27 +146,12 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato Ok((listener, addr)) } -fn decode_wire_request( - line: &str, -) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> { +fn decode_wire_request(line: &str) -> Result { serde_json::from_str::(line)? - .into_parts() + .into_request() .map_err(CoordinatorServiceError::Protocol) } -fn wire_request_id_hint(line: &str) -> String { - serde_json::from_str::(line) - .ok() - .and_then(|value| { - value - .get("request_id") - .and_then(|value| value.as_str()) - .map(str::to_owned) - }) - .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok()) - .unwrap_or_else(|| "unavailable".to_owned()) -} - fn authorize_client_request( request: &CoordinatorRequest, authority_mode: ClientAuthorityMode, @@ -187,21 +180,15 @@ fn authorize_client_request( } | CoordinatorRequest::AdminStatus { .. } | CoordinatorRequest::SuspendTenant { .. } => Ok(()), - _ => Err(crate::CoordinatorError::Unauthorized( + _ => Err(CoordinatorServiceError::Protocol( "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority" .to_owned(), - ) - .into()), + )), } } #[cfg(test)] mod transport_boundary_tests { - use std::io::{BufRead as _, BufReader}; - - use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId}; - use serde_json::json; - use super::*; #[test] @@ -210,99 +197,4 @@ mod transport_boundary_tests { let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err(); assert!(error.to_string().contains("restricted to loopback")); } - - #[test] - fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() { - let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); - let mut coordinator = CoordinatorService::new(11); - coordinator - .issue_cli_session( - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - "healthy-session", - None, - ) - .unwrap(); - let shared = Arc::new(Mutex::new(coordinator)); - let server_shared = Arc::clone(&shared); - let server = std::thread::spawn(move || { - let (stream, _) = listener.accept().unwrap(); - handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap(); - }); - - let mut stream = TcpStream::connect(addr).unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - for (index, malformed_process) in [ - String::new(), - " ".to_owned(), - "bad\0process".to_owned(), - "bad process!".to_owned(), - "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1), - ] - .into_iter() - .enumerate() - { - let malformed = coordinator_wire_request( - format!("malformed-{index}"), - json!({ - "type": "authenticated", - "session_secret": "healthy-session", - "request": { - "type": "abort_process", - "process": malformed_process, - "launch_attempt": "valid-attempt" - } - }), - ); - serde_json::to_writer(&mut stream, &malformed).unwrap(); - stream.write_all(b"\n").unwrap(); - stream.flush().unwrap(); - - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { error } = - serde_json::from_str::(&line).unwrap() - else { - panic!("malformed identifier request unexpectedly succeeded"); - }; - assert!( - error.message.contains("malformed external identifier") - && error.message.contains("request.request.process"), - "unexpected malformed identifier response: {}", - error.message - ); - assert_eq!(error.request_id, format!("malformed-{index}")); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); - - let valid = coordinator_wire_request( - format!("healthy-{index}"), - json!({ - "type": "authenticated", - "session_secret": "healthy-session", - "request": { "type": "auth_status" } - }), - ); - serde_json::to_writer(&mut stream, &valid).unwrap(); - stream.write_all(b"\n").unwrap(); - stream.flush().unwrap(); - - line.clear(); - reader.read_line(&mut line).unwrap(); - assert!( - matches!( - serde_json::from_str::(&line).unwrap(), - CoordinatorResponse::AuthStatus { - authenticated: true, - .. - } - ), - "valid authenticated traffic failed after malformed request {index}" - ); - } - - stream.shutdown(std::net::Shutdown::Both).unwrap(); - server.join().unwrap(); - assert!(!shared.is_poisoned()); - } } diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index 339860d..43215d7 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -12,8 +12,8 @@ use clusterflux_core::{ AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability, DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements, LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits, - SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, - TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult, + SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec, + VfsPath, WasmExportAbi, }; use serde_json::json; @@ -49,35 +49,6 @@ fn test_admin_request( ) } -fn enroll_test_node( - service: &mut CoordinatorService, - tenant: &str, - project: &str, - node: &str, - public_key: &str, -) { - let response = service - .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { - tenant: tenant.to_owned(), - project: project.to_owned(), - actor_user: "test-user".to_owned(), - ttl_seconds: 900, - }) - .unwrap(); - let CoordinatorResponse::NodeEnrollmentGrantCreated { grant, .. } = response else { - panic!("expected node enrollment grant"); - }; - service - .handle_request(CoordinatorRequest::ExchangeNodeEnrollmentGrant { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - public_key: public_key.to_owned(), - enrollment_grant: grant, - }) - .unwrap(); -} - #[test] fn runtime_service_uses_memory_only_when_database_url_is_absent() { let service = CoordinatorService::new_with_database_url(1, None).unwrap(); @@ -92,7 +63,7 @@ fn runtime_service_uses_memory_only_when_database_url_is_absent() { } #[test] -fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() { +fn postgres_backed_runtime_service_survives_restart_without_live_process_state() { let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else { return; }; @@ -113,21 +84,14 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() None, ) .unwrap(); - enroll_test_node( - &mut first, - "tenant-pg", - "project-pg", - "node-pg", - &test_node_public_key("node-pg"), - ); - let duplicate_private_key = test_node_private_key("node-pg-other-scope"); - enroll_test_node( - &mut first, - "tenant-pg-other", - "project-pg-other", - "node-pg", - &node_ed25519_public_key_from_private_key(&duplicate_private_key).unwrap(), - ); + first + .handle_request(CoordinatorRequest::AttachNode { + tenant: "tenant-pg".to_owned(), + project: "project-pg".to_owned(), + node: "node-pg".to_owned(), + public_key: test_node_public_key("node-pg"), + }) + .unwrap(); first .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), @@ -157,12 +121,8 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() )); let heartbeat = restarted .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg".to_owned(), - project: "project-pg".to_owned(), node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope( - "tenant-pg", - "project-pg", + node_signature: Some(signed_node_heartbeat( "node-pg", "postgres-restart-heartbeat", )), @@ -172,24 +132,6 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() heartbeat, CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } )); - let duplicate_heartbeat = restarted - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg-other".to_owned(), - project: "project-pg-other".to_owned(), - node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-pg-other", - "project-pg-other", - "node-pg", - &duplicate_private_key, - "postgres-restart-heartbeat", - )), - }) - .unwrap(); - assert!(matches!( - duplicate_heartbeat, - CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } - )); let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), @@ -200,49 +142,6 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() panic!("expected process list"); }; assert!(processes.is_empty()); - - restarted - .handle_request(CoordinatorRequest::RevokeNodeCredential { - tenant: "tenant-pg".to_owned(), - project: "project-pg".to_owned(), - actor_user: "user-pg".to_owned(), - node: "node-pg".to_owned(), - }) - .unwrap(); - drop(restarted); - - let mut after_revocation = - CoordinatorService::new_with_database_url(43, Some(&database_url)).unwrap(); - assert!(after_revocation - .coordinator - .node_identity( - &TenantId::from("tenant-pg"), - &ProjectId::from("project-pg"), - &NodeId::from("node-pg"), - ) - .is_none()); - assert!(after_revocation - .coordinator - .node_identity( - &TenantId::from("tenant-pg-other"), - &ProjectId::from("project-pg-other"), - &NodeId::from("node-pg"), - ) - .is_some()); - after_revocation - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg-other".to_owned(), - project: "project-pg-other".to_owned(), - node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-pg-other", - "project-pg-other", - "node-pg", - &duplicate_private_key, - "post-revocation-restart", - )), - }) - .expect("the duplicate node in the other scope must remain authenticated"); } fn linux_capabilities() -> NodeCapabilities { @@ -494,191 +393,6 @@ fn register_test_task_assignment( )); } -fn service_with_completed_main_and_final_child( - failure_policy: clusterflux_core::TaskFailurePolicy, -) -> CoordinatorService { - let mut service = CoordinatorService::new(83); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let node = NodeId::from("worker"); - let task = TaskInstanceId::from("final-child"); - - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - public_key: test_node_public_key(node.as_str()), - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: true, - online: true, - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: process.to_string(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - process: process.to_string(), - epoch: 83, - }) - .unwrap(); - - let mut task_spec = test_task_spec_instance( - tenant.as_str(), - project.as_str(), - process.as_str(), - "child-definition", - task.as_str(), - 83, - [], - ); - task_spec.failure_policy = failure_policy; - let assignment = TaskAssignment { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task: task.clone(), - node: node.clone(), - epoch: 83, - artifact_path: "/vfs/artifacts/final-child.bin".to_owned(), - task_spec: task_spec.clone(), - wasm_module_base64: test_wasm_module_base64(), - }; - service - .capture_task_restart_checkpoint(&assignment) - .unwrap(); - service - .begin_task_attempt( - &task_spec, - Some(node.clone()), - Some(&assignment.artifact_path), - false, - ) - .unwrap(); - service - .active_tasks - .insert(task_control_key(&tenant, &project, &process, &node, &task)); - service - .task_assignments - .entry((tenant.clone(), project.clone(), node.clone())) - .or_default() - .push_back(assignment); - service - .debug_epochs - .insert(process_control_key(&tenant, &project, &process), 11); - service.debug_breakpoints.insert( - process_control_key(&tenant, &project, &process), - super::debug::DebugBreakpointPlan { - actor: UserId::from("user"), - revision: 1, - probe_symbols: BTreeSet::from(["child-probe".to_owned()]), - hit_epoch: None, - hit_task: None, - hit_probe_symbol: None, - }, - ); - service.debug_commands.insert( - task_control_key(&tenant, &project, &process, &node, &task), - super::debug::DebugPendingCommand { - epoch: 11, - command: "continue".to_owned(), - }, - ); - let panel_key = super::keys::panel_stop_key(&tenant, &project, &process); - service.panel_snapshots.insert( - panel_key.clone(), - PanelState { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - widgets: BTreeMap::new(), - program_ui_events_enabled: false, - control_plane_actions: Vec::new(), - }, - ); - service.stopped_panels.insert(panel_key); - service.record_task_completion_event(TaskCompletionEvent { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - node: NodeId::from("coordinator-main"), - executor: TaskExecutor::CoordinatorMain, - task_definition: TaskDefinitionId::from("build"), - task: TaskInstanceId::from("main"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - service - .coordinator - .grant_project_debug(tenant, project, UserId::from("user")); - service -} - -fn complete_terminal_matrix_child( - service: &mut CoordinatorService, - terminal_state: TaskTerminalState, -) { - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "terminal-matrix".to_owned(), - node: "worker".to_owned(), - task: "final-child".to_owned(), - terminal_state: Some(terminal_state), - status_code: Some(1), - stdout_bytes: 0, - stderr_bytes: 4, - stdout_tail: String::new(), - stderr_tail: "boom".to_owned(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }) - .unwrap(); -} - fn test_agent_public_key() -> String { agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap() } @@ -741,21 +455,7 @@ fn test_node_public_key(node: &str) -> String { } fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest { - signed_node_heartbeat_in_scope("tenant", "project", node, nonce) -} - -fn signed_node_heartbeat_in_scope( - tenant: &str, - project: &str, - node: &str, - nonce: &str, -) -> NodeSignedRequest { - let payload = json!({ - "type": "node_heartbeat", - "tenant": tenant, - "project": project, - "node": node - }); + let payload = json!({"type": "node_heartbeat", "node": node}); signed_node_request_with_private_key( node, &test_node_private_key(node), @@ -770,22 +470,7 @@ fn signed_node_heartbeat_with_private_key( private_key: &str, nonce: &str, ) -> NodeSignedRequest { - signed_node_heartbeat_in_scope_with_private_key("tenant", "project", node, private_key, nonce) -} - -fn signed_node_heartbeat_in_scope_with_private_key( - tenant: &str, - project: &str, - node: &str, - private_key: &str, - nonce: &str, -) -> NodeSignedRequest { - let payload = json!({ - "type": "node_heartbeat", - "tenant": tenant, - "project": project, - "node": node - }); + let payload = json!({"type": "node_heartbeat", "node": node}); signed_node_request_with_private_key( node, private_key, @@ -814,34 +499,6 @@ fn signed_node_request_with_private_key( } fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { - let node = match &request { - CoordinatorRequest::ReportNodeCapabilities { node, .. } - | CoordinatorRequest::PollTaskAssignment { node, .. } - | CoordinatorRequest::PollArtifactTransfer { node, .. } - | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } - | CoordinatorRequest::FailArtifactTransfer { node, .. } - | CoordinatorRequest::LaunchChildTask { node, .. } - | CoordinatorRequest::JoinChildTask { node, .. } - | CoordinatorRequest::CompleteSourcePreparation { node, .. } - | CoordinatorRequest::ReconnectNode { node, .. } - | CoordinatorRequest::PollTaskControl { node, .. } - | CoordinatorRequest::PollDebugCommand { node, .. } - | CoordinatorRequest::ReportDebugState { node, .. } - | CoordinatorRequest::ReportDebugProbeHit { node, .. } - | CoordinatorRequest::ReportTaskLog { node, .. } - | CoordinatorRequest::ReportTaskLogChunk { node, .. } - | CoordinatorRequest::ReportVfsMetadata { node, .. } - | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), - CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), - _ => panic!("test helper only signs node-originated requests"), - }; - signed_node_request_auto_with_private_key(request, &test_node_private_key(&node)) -} - -fn signed_node_request_auto_with_private_key( - request: CoordinatorRequest, - private_key: &str, -) -> CoordinatorRequest { let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap()); let (node, request_kind) = match &request { CoordinatorRequest::ReportNodeCapabilities { node, .. } => { @@ -864,9 +521,6 @@ fn signed_node_request_auto_with_private_key( CoordinatorRequest::CompleteSourcePreparation { node, .. } => { (node.clone(), "complete_source_preparation") } - CoordinatorRequest::RequestRendezvous { source, .. } => { - (source.node.to_string(), "request_rendezvous") - } CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"), CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"), CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"), @@ -875,9 +529,6 @@ fn signed_node_request_auto_with_private_key( (node.clone(), "report_debug_probe_hit") } CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"), - CoordinatorRequest::ReportTaskLogChunk { node, .. } => { - (node.clone(), "report_task_log_chunk") - } CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"), CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"), _ => panic!("test helper only signs node-originated requests"), @@ -889,7 +540,7 @@ fn signed_node_request_auto_with_private_key( node: node.clone(), node_signature: signed_node_request_with_private_key( &node, - private_key, + &test_node_private_key(&node), request_kind, &payload_digest, &format!("node-request-{nonce}"), @@ -1295,8 +946,6 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), node: "session-node".to_owned(), process: "vp-session".to_owned(), epoch: 7, @@ -1503,7 +1152,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { suspended, disabled, sanitized_reason, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1524,7 +1173,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(!suspended); assert!(!disabled); assert!(sanitized_reason.is_none()); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); for (tenant, policy_name, expected_status, expected_reason) in [ @@ -1560,7 +1209,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { manual_review, sanitized_reason, next_actions, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1582,7 +1231,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); } @@ -1721,7 +1370,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { disabled, sanitized_reason, next_actions, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1744,7 +1393,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); let create = service @@ -2435,7 +2084,7 @@ fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() } #[test] -fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() { +fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::LogBytes, 4); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); @@ -2470,8 +2119,6 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2494,37 +2141,23 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() backpressured: false, }) .unwrap(); - let truncated = service + let denied = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "task".to_owned(), - stdout_bytes: 4, - stderr_bytes: 1, - stdout_tail: "outx".to_owned(), - stderr_tail: "e".to_owned(), + stdout_bytes: 1, + stderr_bytes: 0, + stdout_tail: "x".to_owned(), + stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, backpressured: true, }) - .unwrap(); - let CoordinatorResponse::TaskLogRecorded { - stdout_tail, - stdout_bytes: 4, - .. - } = truncated - else { - panic!("expected a successful truncated task-log report"); - }; - assert_eq!(stdout_tail, "[log output truncated at project log quota]"); - assert!(service - .recent_logs - .get(&(TenantId::from("tenant"), ProjectId::from("project"))) - .unwrap() - .iter() - .any(|entry| entry.text.contains("project log quota") && entry.truncated)); + .unwrap_err(); + assert!(denied.to_string().contains("LogBytes")); assert_eq!( service .quota @@ -2533,140 +2166,6 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() ); } -#[test] -fn log_quota_exhaustion_cannot_strand_task_completion_or_artifact_publication() { - let mut limits = ResourceLimits::unlimited(); - limits.limits.insert(LimitKind::LogBytes, 4); - let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); - let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( - 7, - "test-admin-token", - None, - quota, - ) - .unwrap(); - service.set_server_time(30); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - public_key: test_node_public_key("node"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process".to_owned(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - process: "process".to_owned(), - epoch: 7, - }) - .unwrap(); - register_test_task_assignment( - &mut service, - "tenant", - "project", - "process", - "node", - "compile", - "compile-one", - 7, - ); - service.record_task_completion_event(TaskCompletionEvent { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - node: NodeId::from("coordinator-main"), - executor: TaskExecutor::CoordinatorMain, - task_definition: TaskDefinitionId::from("build"), - task: TaskInstanceId::from("main"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "process".to_owned(), - node: "node".to_owned(), - task: "compile-one".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 64, - stderr_bytes: 0, - stdout_tail: "the-real-final-tail".to_owned(), - stderr_tail: String::new(), - stdout_truncated: true, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/result.bin".to_owned()), - artifact_digest: Some(Digest::sha256("artifact bytes")), - artifact_size_bytes: Some(14), - result: None, - }) - .unwrap(); - - let event = service - .task_events - .iter() - .find(|event| event.task == TaskInstanceId::from("compile-one")) - .unwrap(); - assert_eq!( - event.stdout_tail, - "[log output truncated at project log quota]" - ); - assert!(event.stdout_truncated); - assert!(!service - .active_tasks - .iter() - .any(|key| key.4 == TaskInstanceId::from("compile-one"))); - assert!(service - .coordinator - .active_process( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ProcessId::from("process"), - ) - .is_none()); - assert!(matches!( - service - .handle_request(CoordinatorRequest::GetArtifact { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - artifact: "result.bin".to_owned(), - }) - .unwrap(), - CoordinatorResponse::Artifact { .. } - )); -} - #[test] fn service_attaches_node_starts_process_and_records_scoped_task_event() { let mut service = CoordinatorService::new(7); @@ -2715,8 +2214,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")), }) @@ -2731,8 +2228,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2917,8 +2412,6 @@ fn service_revokes_node_credentials_and_live_descriptors() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")), }) @@ -2938,201 +2431,6 @@ fn service_revokes_node_credentials_and_live_descriptors() { assert!(descriptors.is_empty()); } -#[test] -fn duplicate_node_ids_are_isolated_across_identity_replay_liveness_and_revocation() { - let mut service = CoordinatorService::new(19); - let node = "shared-node"; - let private_a = test_node_private_key("tenant-a-shared-node"); - let private_b = test_node_private_key("tenant-b-shared-node"); - let private_c = test_node_private_key("tenant-a-other-project-shared-node"); - let public_a = node_ed25519_public_key_from_private_key(&private_a).unwrap(); - let public_b = node_ed25519_public_key_from_private_key(&private_b).unwrap(); - let public_c = node_ed25519_public_key_from_private_key(&private_c).unwrap(); - - for (tenant, project, public_key) in [ - ("tenant-a", "project-a", public_a), - ("tenant-b", "project-b", public_b), - ("tenant-a", "project-c", public_c), - ] { - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - public_key, - }) - .unwrap(); - } - assert_eq!( - service - .coordinator - .node_identity_count_for_tenant(&TenantId::from("tenant-a")), - 2 - ); - assert_eq!( - service - .coordinator - .node_identity_count_for_tenant(&TenantId::from("tenant-b")), - 1 - ); - - for (index, (tenant, project, private_key)) in [ - ("tenant-a", "project-a", private_a.as_str()), - ("tenant-b", "project-b", private_b.as_str()), - ("tenant-a", "project-c", private_c.as_str()), - ] - .into_iter() - .enumerate() - { - service.set_server_time(100 + index as u64); - service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - tenant, - project, - node, - private_key, - "same-nonce", - )), - }) - .unwrap(); - service - .handle_request(signed_node_request_auto_with_private_key( - CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: vec![Digest::sha256(format!( - "cache-{tenant}-{project}" - ))], - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: index != 1, - online: true, - }, - private_key, - )) - .unwrap(); - } - - let replay = service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-a", - "project-a", - node, - &private_a, - "same-nonce", - )), - }) - .unwrap_err(); - assert!(replay.to_string().contains("nonce")); - - let forged = service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-b".to_owned(), - project: "project-b".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-b", - "project-b", - node, - &private_a, - "forged-cross-scope", - )), - }) - .unwrap_err(); - assert!(forged.to_string().contains("signature")); - - let scope_a = crate::NodeScopeKey::new( - TenantId::from("tenant-a"), - ProjectId::from("project-a"), - NodeId::from(node), - ); - let scope_b = crate::NodeScopeKey::new( - TenantId::from("tenant-b"), - ProjectId::from("project-b"), - NodeId::from(node), - ); - let scope_c = crate::NodeScopeKey::new( - TenantId::from("tenant-a"), - ProjectId::from("project-c"), - NodeId::from(node), - ); - assert!(service.node_descriptors.contains_key(&scope_a)); - assert!(service.node_descriptors.contains_key(&scope_b)); - assert!(service.node_descriptors.contains_key(&scope_c)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_a)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_b)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_c)); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_a], 100); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_b], 101); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_c], 102); - assert!(service.node_descriptors[&scope_a].direct_connectivity); - assert!(!service.node_descriptors[&scope_b].direct_connectivity); - assert!(service.node_descriptors[&scope_c].direct_connectivity); - assert!(service - .node_replay_nonces - .contains_key(&(scope_a.clone(), "same-nonce".to_owned()))); - assert!(service - .node_replay_nonces - .contains_key(&(scope_b.clone(), "same-nonce".to_owned()))); - assert!(service - .node_replay_nonces - .contains_key(&(scope_c.clone(), "same-nonce".to_owned()))); - - service - .handle_request(CoordinatorRequest::RevokeNodeCredential { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - actor_user: "user-a".to_owned(), - node: node.to_owned(), - }) - .unwrap(); - assert!(service - .coordinator - .node_identity( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - &NodeId::from(node), - ) - .is_none()); - assert!(service - .coordinator - .node_identity( - &TenantId::from("tenant-b"), - &ProjectId::from("project-b"), - &NodeId::from(node), - ) - .is_some()); - assert!(!service.node_descriptors.contains_key(&scope_a)); - assert!(service.node_descriptors.contains_key(&scope_b)); - assert!(service.node_descriptors.contains_key(&scope_c)); - - service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-b".to_owned(), - project: "project-b".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-b", - "project-b", - node, - &private_b, - "post-other-scope-revocation", - )), - }) - .expect("revoking tenant A's duplicate node must not affect tenant B"); -} - #[test] fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() { let mut service = CoordinatorService::new(7); @@ -3160,8 +2458,6 @@ fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -3456,7 +2752,6 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { }, ); let CoordinatorResponse::DebugBreakpoints { - revision, probe_symbols, hit_epoch, .. @@ -3466,37 +2761,15 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), - revision: 1, probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()], }) .unwrap() else { panic!("expected debug breakpoints response"); }; - assert_eq!(revision, 1); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); assert_eq!(hit_epoch, None); - let CoordinatorResponse::DebugBreakpoints { - revision, - probe_symbols, - .. - } = service - .handle_request(CoordinatorRequest::SetDebugBreakpoints { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - process: "process".to_owned(), - revision: 0, - probe_symbols: vec!["clusterflux.probe.stale".to_owned()], - }) - .unwrap() - else { - panic!("expected stale breakpoint response"); - }; - assert_eq!(revision, 1); - assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); - let CoordinatorResponse::DebugProbeHit { breakpoint_matched, debug_epoch, @@ -4044,8 +3317,6 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() { for node in ["node-a", "node-b"] { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: node.to_owned(), process: "process".to_owned(), epoch: 7, @@ -4312,622 +3583,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { assert!(!service.main_runtime.controls.contains_key(&process_key)); } -#[test] -fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() { - let mut service = CoordinatorService::new(31); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("process-main-before-child"); - let child = TaskInstanceId::from("child-active"); - let process_key = process_control_key(&tenant, &project, &process); - - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: "worker".to_owned(), - public_key: test_node_public_key("worker"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: process.to_string(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: "worker".to_owned(), - process: process.to_string(), - epoch: 31, - }) - .unwrap(); - register_test_task_assignment( - &mut service, - tenant.as_str(), - project.as_str(), - process.as_str(), - "worker", - "child-definition", - child.as_str(), - 31, - ); - - let main = TaskInstanceId::from("main-instance"); - service.main_runtime.controls.insert( - process_key.clone(), - super::main_runtime::CoordinatorMainControl { - task_definition: TaskDefinitionId::from("build"), - task_instance: main.clone(), - abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), - debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), - launch_id: 1, - }, - ); - service.debug_epochs.insert(process_key.clone(), 9); - - service.record_coordinator_main_completion( - super::main_runtime::MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main, - epoch: 31, - launch_id: 1, - }, - Ok(WasmTaskResult::completed( - TaskInstanceId::from("main-instance"), - TaskBoundaryValue::SmallJson(json!("main completed")), - )), - ); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(service - .active_tasks - .iter() - .any(|(_, _, retained_process, _, task)| { - retained_process == &process && task == &child - })); - let blocked_next = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process-too-early".to_owned(), - restart: false, - }) - .unwrap_err(); - assert!(blocked_next - .to_string() - .contains("already has active virtual process")); - - let artifact_bytes = b"child artifact survives terminal cleanup"; - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: child.to_string(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: artifact_bytes.len() as u64, - stderr_bytes: 0, - stdout_tail: "child completed".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/child-output".to_owned()), - artifact_digest: Some(Digest::sha256(artifact_bytes)), - artifact_size_bytes: Some(artifact_bytes.len() as u64), - result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.debug_epochs.contains_key(&process_key)); - let CoordinatorResponse::TaskEvents { events } = service - .handle_request(CoordinatorRequest::ListTaskEvents { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: Some(process.to_string()), - }) - .unwrap() - else { - panic!("expected retained task events"); - }; - assert_eq!(events.len(), 2); - assert!(events.iter().any(|event| { - event.executor == TaskExecutor::CoordinatorMain - && event.terminal_state == TaskTerminalState::Completed - })); - assert!(events.iter().any(|event| { - event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes)) - })); - let metadata = service - .artifact_registry - .metadata(&tenant, &project, &ArtifactId::from("child-output")) - .expect("artifact metadata must survive terminal cleanup"); - assert_eq!(metadata.process, process); - assert_eq!(metadata.digest, Digest::sha256(artifact_bytes)); - - let next = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process-after-cleanup".to_owned(), - restart: false, - }) - .unwrap(); - assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. })); -} - -#[test] -fn completed_main_terminal_matrix_retires_after_failed_or_cancelled_final_child() { - for terminal_state in [TaskTerminalState::Failed, TaskTerminalState::Cancelled] { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::FailFast, - ); - complete_terminal_matrix_child(&mut service, terminal_state.clone()); - - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - assert!( - service - .coordinator - .active_process(&tenant, &project, &process) - .is_none(), - "{terminal_state:?} final child left the process slot active" - ); - assert!(!service - .debug_epochs - .contains_key(&process_control_key(&tenant, &project, &process))); - assert!(!service - .debug_breakpoints - .contains_key(&process_control_key(&tenant, &project, &process))); - assert!(service.debug_commands.keys().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - assert!(!service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - assert!(service.active_tasks.iter().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - assert!(!service - .main_runtime - .controls - .contains_key(&process_control_key(&tenant, &project, &process))); - let join = service.task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ); - assert_eq!( - join.state, - match terminal_state { - TaskTerminalState::Failed => TaskJoinState::Failed, - TaskTerminalState::Cancelled => TaskJoinState::Cancelled, - TaskTerminalState::Completed => unreachable!(), - } - ); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-terminal".to_owned(), - restart: false, - }) - .expect("the terminal outcome must release the one-process project slot"); - } -} - -#[test] -fn completed_main_unpolled_final_assignment_completion_retires_process() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let node = NodeId::from("worker"); - let assignment_key = (tenant.clone(), project.clone(), node); - assert_eq!( - service - .task_assignments - .get(&assignment_key) - .map_or(0, VecDeque::len), - 1 - ); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: "final-child".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 2, - stderr_bytes: 0, - stdout_tail: "42".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: Some(TaskBoundaryValue::SmallJson(json!(42))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.task_assignments.contains_key(&assignment_key)); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-unpolled-completion".to_owned(), - restart: false, - }) - .expect("unpolled terminal completion must release the one-process slot"); -} - -#[test] -fn completed_main_retires_from_authoritative_state_after_event_history_rotates() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - - for index in 0..=MAX_TASK_EVENTS_PER_PROCESS { - service.record_task_completion_event(TaskCompletionEvent { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - node: NodeId::from("worker"), - executor: TaskExecutor::Node, - task_definition: TaskDefinitionId::from("historical"), - task: TaskInstanceId::new(format!("historical-{index}")), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - } - assert!( - service.task_events.iter().all(|event| { - event.process != process || event.executor != TaskExecutor::CoordinatorMain - }), - "the regression requires bounded history to have rotated the main event" - ); - - complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - let summary = service - .process_summaries - .get(&process_control_key(&tenant, &project, &process)) - .expect("the terminal process summary must remain authoritative"); - assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed)); - assert_eq!( - summary.main_terminal_state, - Some(TaskTerminalState::Completed) - ); -} - -#[test] -fn completed_main_await_operator_blocks_retirement_until_each_resolution() { - for resolution in [ - TaskFailureResolution::AcceptFailure, - TaskFailureResolution::Cancel, - ] { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::AwaitOperator, - ); - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let process_key = process_control_key(&tenant, &project, &process); - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(service.debug_breakpoints.contains_key(&process_key)); - assert!(service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - let attempt = service - .task_attempts - .get(&super::keys::task_restart_key( - &tenant, - &project, - &process, - &TaskInstanceId::from("final-child"), - )) - .and_then(|attempts| attempts.iter().rev().find(|attempt| attempt.current)) - .unwrap(); - assert_eq!(attempt.state, TaskAttemptState::FailedAwaitingAction); - - service - .handle_request(CoordinatorRequest::ResolveTaskFailure { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: process.to_string(), - task: "final-child".to_owned(), - resolution, - }) - .unwrap(); - - assert!( - service - .coordinator - .active_process(&tenant, &project, &process) - .is_none(), - "{resolution:?} left the process slot active" - ); - assert!(!service.debug_epochs.contains_key(&process_key)); - assert!(!service.debug_breakpoints.contains_key(&process_key)); - assert!(!service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - assert!(service.debug_commands.keys().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - let join = service.task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ); - assert_eq!( - join.state, - match resolution { - TaskFailureResolution::AcceptFailure => TaskJoinState::Failed, - TaskFailureResolution::Cancel => TaskJoinState::Cancelled, - } - ); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-resolution".to_owned(), - restart: false, - }) - .expect("operator resolution must release the one-process project slot"); - } -} - -#[test] -fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::AwaitOperator, - ); - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let task = TaskInstanceId::from("final-child"); - - let CoordinatorResponse::TaskRestart { accepted, .. } = service - .handle_request(CoordinatorRequest::RestartTask { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: process.to_string(), - task: task.to_string(), - replacement_bundle: None, - }) - .unwrap() - else { - panic!("expected task restart"); - }; - assert!(accepted); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: task.to_string(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 2, - stderr_bytes: 0, - stdout_tail: "ok".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: Some(TaskBoundaryValue::SmallJson(json!("ok"))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - let attempts = service - .task_attempts - .get(&super::keys::task_restart_key( - &tenant, &project, &process, &task, - )) - .unwrap(); - assert!( - attempts - .iter() - .any(|attempt| !attempt.current - && attempt.state == TaskAttemptState::FailedAwaitingAction) - ); - assert!(attempts - .iter() - .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed)); - assert_eq!( - service - .process_summaries - .get(&process_control_key(&tenant, &project, &process)) - .and_then(|summary| summary.final_result.clone()), - Some(ProcessFinalResult::Completed), - "a successful current retry must override the superseded failed attempt" - ); - assert_eq!( - service - .task_join_result(tenant, project, process, task) - .state, - TaskJoinState::Completed - ); -} - -#[test] -fn completed_main_failed_child_does_not_abort_another_active_child() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - register_test_task_assignment( - &mut service, - "tenant", - "project", - "terminal-matrix", - "worker", - "other-child-definition", - "other-child", - 83, - ); - - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let other_key = task_control_key( - &tenant, - &project, - &process, - &NodeId::from("worker"), - &TaskInstanceId::from("other-child"), - ); - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.active_tasks.contains(&other_key)); - assert!(!service.task_aborts.contains(&other_key)); - assert_eq!( - service - .task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ) - .state, - TaskJoinState::Failed - ); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: "other-child".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.active_tasks.contains(&other_key)); -} - #[test] fn quiescent_cooperative_cancel_releases_slot_immediately() { let mut service = CoordinatorService::new(17); @@ -5059,8 +3714,6 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 17, @@ -5370,7 +4023,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_tenant.to_string().contains("does not exist")); + assert!(cross_tenant.to_string().contains("tenant mismatch")); let cross_project = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { @@ -5382,7 +4035,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_project.to_string().contains("does not exist")); + assert!(cross_project.to_string().contains("project mismatch")); let cross_tenant_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5395,7 +4048,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_open.to_string().contains("does not exist")); + assert!(cross_tenant_open.to_string().contains("tenant mismatch")); let cross_project_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5408,7 +4061,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_open.to_string().contains("does not exist")); + assert!(cross_project_open.to_string().contains("project mismatch")); let guessed = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5664,8 +4317,6 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -5710,7 +4361,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_retry.to_string().contains("does not exist")); + assert!(cross_tenant_retry.to_string().contains("tenant mismatch")); let cross_project_retry = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5723,7 +4374,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_retry.to_string().contains("does not exist")); + assert!(cross_project_retry.to_string().contains("project mismatch")); } #[test] @@ -5876,29 +4527,6 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() { #[test] fn windows_task_events_share_the_virtual_process_scope() { let mut service = CoordinatorService::new(7); - service.record_task_completion_event(TaskCompletionEvent { - tenant: TenantId::from("other-tenant"), - project: ProjectId::from("other-project"), - process: ProcessId::from("other-process"), - node: NodeId::from("other-node"), - executor: super::TaskExecutor::Node, - task_definition: TaskDefinitionId::from("other-task"), - task: TaskInstanceId::from("other-task"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), @@ -5999,28 +4627,6 @@ fn windows_task_events_share_the_virtual_process_scope() { #[test] fn service_schedules_task_across_reported_node_descriptors() { let mut service = CoordinatorService::new(7); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - node: "other-node".to_owned(), - public_key: test_node_public_key("other-node"), - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - node: "other-node".to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: false, - online: true, - }) - .unwrap(); for node in ["cold-node", "warm-node"] { service .handle_request(CoordinatorRequest::AttachNode { @@ -6232,8 +4838,6 @@ fn coordinator_side_task_launch_queues_worker_assignment() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker-linux".to_owned(), process: "vp-control".to_owned(), epoch, @@ -6555,8 +5159,6 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 41, @@ -6884,8 +5486,6 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant-soak".to_owned(), - project: "project-soak".to_owned(), node: "worker-soak".to_owned(), process: "vp-soak".to_owned(), epoch: 73, @@ -6952,29 +5552,14 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { } for index in 0..2_000 { - service.node_replay_nonces.insert( - ( - crate::NodeScopeKey::new( - TenantId::from("tenant-soak"), - ProjectId::from("project-soak"), - NodeId::from("worker-soak"), - ), - format!("expired-{index}"), - ), - 0, - ); + service + .node_replay_nonces + .insert((NodeId::from("worker-soak"), format!("expired-{index}")), 0); } service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-soak".to_owned(), - project: "project-soak".to_owned(), node: "worker-soak".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope( - "tenant-soak", - "project-soak", - "worker-soak", - "post-expiry-prune", - )), + node_signature: Some(signed_node_heartbeat("worker-soak", "post-expiry-prune")), }) .unwrap(); @@ -7027,14 +5612,7 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { let retained_soak_nonces = service .node_replay_nonces .keys() - .filter(|(scope, _)| { - scope - == &crate::NodeScopeKey::new( - TenantId::from("tenant-soak"), - ProjectId::from("project-soak"), - NodeId::from("worker-soak"), - ) - }) + .filter(|(node, _)| node == &NodeId::from("worker-soak")) .count(); assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS); @@ -7105,8 +5683,6 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 12, @@ -7227,8 +5803,6 @@ fn coordinator_rejects_named_environment_without_requirements() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "uncached-worker".to_owned(), process: "vp-environment".to_owned(), epoch, @@ -7308,52 +5882,6 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { #[test] fn coordinator_side_task_launch_can_wait_for_capable_worker() { let mut service = CoordinatorService::new(11); - let CoordinatorResponse::ProcessStarted { - epoch: other_epoch, .. - } = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "other-vp-wait".to_owned(), - restart: false, - }) - .unwrap() - else { - panic!("expected unrelated process start"); - }; - let CoordinatorResponse::TaskQueued { - queued_tasks: other_queued_tasks, - .. - } = service - .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { - task_spec: test_task_spec( - "other-tenant", - "other-project", - "other-vp-wait", - "other-compile", - other_epoch, - [Capability::Command], - ), - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - actor_user: Some("other-user".to_owned()), - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - wait_for_node: true, - artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(), - wasm_module_base64: test_wasm_module_base64(), - }) - .unwrap() - else { - panic!("expected unrelated queued task launch"); - }; - assert_eq!(other_queued_tasks, 1); let CoordinatorResponse::ProcessStarted { launch_attempt: None, epoch, @@ -7556,172 +6084,6 @@ fn service_meters_rendezvous_before_direct_transfer_plan() { assert!(quota.to_string().contains("RendezvousAttempt")); } -#[test] -fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() { - let mut service = CoordinatorService::new(7); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node-a".to_owned(), - public_key: test_node_public_key("node-a"), - }) - .unwrap(); - - let response = service - .handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous { - scope: data_plane_scope("project"), - source: endpoint("node-a"), - destination: endpoint("node-b"), - direct_connectivity: true, - failure_reason: String::new(), - }) - .unwrap(); - - let CoordinatorResponse::RendezvousPlan { plan, .. } = response else { - panic!("expected signed rendezvous plan"); - }; - assert_eq!(plan.source.node, NodeId::from("node-a")); - assert_eq!(plan.destination.node, NodeId::from("node-b")); -} - -#[test] -fn signed_hostile_artifact_paths_return_errors_and_the_same_service_stays_healthy() { - let mut service = CoordinatorService::new(27); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - public_key: test_node_public_key("node"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "hostile-path-process".to_owned(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - process: "hostile-path-process".to_owned(), - epoch: 27, - }) - .unwrap(); - - let invalid_metadata = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "metadata-task".to_owned(), - artifact_path: Some("/vfs/artifacts/bad artifact!".to_owned()), - artifact_digest: Some(Digest::sha256("bad")), - artifact_size_bytes: Some(3), - large_bytes_uploaded: false, - }) - .unwrap_err(); - assert!( - invalid_metadata - .to_string() - .contains("invalid VFS artifact path"), - "unexpected error: {invalid_metadata}" - ); - - let valid_metadata = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "metadata-task".to_owned(), - artifact_path: Some("/vfs/artifacts/valid-artifact".to_owned()), - artifact_digest: Some(Digest::sha256("valid")), - artifact_size_bytes: Some(5), - large_bytes_uploaded: false, - }) - .unwrap(); - assert!(matches!( - valid_metadata, - CoordinatorResponse::VfsMetadataRecorded { .. } - )); - - register_test_task_assignment( - &mut service, - "tenant", - "project", - "hostile-path-process", - "node", - "child", - "child-instance", - 27, - ); - let invalid_completion = service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "child-instance".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/repeated//component".to_owned()), - artifact_digest: Some(Digest::sha256("bad-completion")), - artifact_size_bytes: Some(0), - result: None, - }) - .unwrap_err(); - assert!( - invalid_completion - .to_string() - .contains("invalid VFS artifact path"), - "unexpected error: {invalid_completion}" - ); - - let valid_completion = service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "child-instance".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/valid-completion".to_owned()), - artifact_digest: Some(Digest::sha256("valid-completion")), - artifact_size_bytes: Some(0), - result: None, - }) - .unwrap(); - assert!(matches!( - valid_completion, - CoordinatorResponse::TaskRecorded { .. } - )); -} - #[test] fn service_rejects_task_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); @@ -7769,7 +6131,7 @@ fn service_rejects_task_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("outside")); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant-b".to_owned(), @@ -7834,7 +6196,7 @@ fn service_rejects_node_capability_report_outside_enrollment_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("tenant/project scope")); assert!(service.node_descriptors.is_empty()); } @@ -7860,7 +6222,7 @@ fn service_rejects_source_preparation_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("tenant/project scope")); } #[test] @@ -7869,8 +6231,6 @@ fn service_rejects_unknown_node_heartbeat() { let error = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "missing".to_owned(), node_signature: None, }) @@ -7893,8 +6253,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let unsigned = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: None, }) @@ -7904,8 +6262,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let wrong_private_key = test_node_private_key("other-node"); let wrong_signature = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat_with_private_key( "node", @@ -7919,8 +6275,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let signed = signed_node_heartbeat("node", "fresh-node-heartbeat"); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed.clone()), }) @@ -7935,8 +6289,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let replay = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed), }) @@ -8007,8 +6359,6 @@ fn service_stream_accepts_multiple_requests_on_one_connection() { write_coordinator_wire_request( &mut stream, &CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")), }, @@ -8067,16 +6417,12 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { error } = + let CoordinatorResponse::Error { message } = serde_json::from_str::(&line).unwrap() else { panic!("expected strict body-authority denial"); }; - assert!(error - .message - .contains("request-body identity fields are not authority")); - assert_eq!(error.request_id, "strict-stream-forged"); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden); + assert!(message.contains("request-body identity fields are not authority")); write_coordinator_wire_request( &mut stream, @@ -8132,14 +6478,10 @@ fn service_stream_rejects_invalid_versioned_envelope_metadata() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); let response = serde_json::from_str::(&line).unwrap(); - let CoordinatorResponse::Error { error } = response else { + let CoordinatorResponse::Error { message } = response else { panic!("expected invalid wire envelope response"); }; - assert!(error - .message - .contains("operation attach_node does not match payload operation ping")); - assert_eq!(error.request_id, "bad-operation"); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); + assert!(message.contains("operation attach_node does not match payload operation ping")); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); @@ -8236,879 +6578,3 @@ fn coordinator_generates_and_bounds_node_enrollment_grants() { assert_ne!(first_grant, second_grant); assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60); } - -#[test] -fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret) in [ - ("tenant-a", "project-a", "user-a", "session-a"), - ("tenant-b", "project-b", "user-b", "session-b"), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - } - service.set_server_time(100); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: "process-one".to_owned(), - restart: false, - }, - }) - .unwrap(); - - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected process summaries"); - }; - assert_eq!(processes.len(), 1); - assert_eq!(processes[0].process, ProcessId::from("process-one")); - assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active); - assert_eq!(processes[0].activity, ProcessActivityState::Running); - assert_eq!(processes[0].started_at_epoch_seconds, 100); - assert!(next_cursor.is_none()); - - let CoordinatorResponse::ProcessSummaries { processes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 10, - }, - }) - .unwrap() - else { - panic!("expected scoped process summaries"); - }; - assert!(processes.is_empty()); - - service.set_server_time(120); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "process-one".to_owned(), - launch_attempt: None, - }, - }) - .unwrap(); - let CoordinatorResponse::ProcessSummaries { processes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 10, - }, - }) - .unwrap() - else { - panic!("expected terminal process summary"); - }; - assert_eq!( - processes[0].lifecycle, - ProcessLifecycleState::RecentTerminal - ); - assert_eq!(processes[0].activity, ProcessActivityState::Cancelled); - assert_eq!( - processes[0].final_result, - Some(ProcessFinalResult::Cancelled) - ); - assert_eq!(processes[0].ended_at_epoch_seconds, Some(120)); - - service.set_server_time(130); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: "process-two".to_owned(), - restart: false, - }, - }) - .unwrap(); - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor: Some(cursor), - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected first process summary page"); - }; - assert_eq!(processes[0].process, ProcessId::from("process-two")); - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor: None, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: Some(cursor), - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected final process summary page"); - }; - assert_eq!(processes[0].process, ProcessId::from("process-one")); - - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 101, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("100")); -} - -#[test] -fn process_summary_eviction_releases_live_log_accounting_state() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant-summary-bound"); - let project = ProjectId::from("project-summary-bound"); - let task = TaskInstanceId::from("task"); - - for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT { - let process = ProcessId::new(format!("process-{index:03}")); - service.record_process_started(&tenant, &project, &process, index as u64); - service.record_process_terminal( - &tenant, - &project, - &process, - ProcessFinalResult::Completed, - index as u64 + 1, - ); - let key = ( - tenant.clone(), - project.clone(), - process, - task.clone(), - "stdout".to_owned(), - ); - service.recent_log_accounted_bytes.insert(key.clone(), 10); - service.recent_log_truncated_streams.insert(key); - } - - let evicted = ProcessId::from("process-000"); - service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000); - - assert!(!service.process_summaries.contains_key(&( - tenant.clone(), - project.clone(), - evicted.clone() - ))); - assert!(!service.recent_log_accounted_bytes.keys().any( - |(entry_tenant, entry_project, process, _, _)| { - entry_tenant == &tenant && entry_project == &project && process == &evicted - } - )); - assert!(!service.recent_log_truncated_streams.iter().any( - |(entry_tenant, entry_project, process, _, _)| { - entry_tenant == &tenant && entry_project == &project && process == &evicted - } - )); -} - -#[test] -fn web_node_summaries_are_scoped_paginated_and_hard_bounded() { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - "session", - None, - ) - .unwrap(); - for node in ["node-a", "node-b", "node-c"] { - enroll_test_node( - &mut service, - "tenant", - "project", - node, - &test_node_public_key(node), - ); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: true, - online: true, - }) - .unwrap(); - } - - let CoordinatorResponse::NodeSummaries { - nodes, - next_cursor: Some(cursor), - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected first node-summary page"); - }; - assert_eq!( - nodes - .iter() - .map(|node| node.id.as_str()) - .collect::>(), - ["node-a", "node-b"] - ); - - let CoordinatorResponse::NodeSummaries { - nodes, - next_cursor: None, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: Some(cursor), - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected final node-summary page"); - }; - assert_eq!(nodes.len(), 1); - assert_eq!(nodes[0].id, NodeId::from("node-c")); - - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 201, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("200")); -} - -#[test] -fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret, node) in [ - ("tenant-a", "project-a", "user-a", "session-a", "node-a"), - ("tenant-b", "project-b", "user-b", "session-b", "node-b"), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - enroll_test_node( - &mut service, - tenant, - project, - node, - &test_node_public_key(node), - ); - } - service.set_server_time(100); - for (tenant, project, node) in [ - ("tenant-a", "project-a", "node-a"), - ("tenant-b", "project-b", "node-b"), - ] { - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: vec!["shared-artifact".to_owned()], - direct_connectivity: true, - online: false, - }) - .unwrap(); - } - let CoordinatorResponse::NodeSummaries { nodes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected node summaries"); - }; - assert_eq!(nodes.len(), 1); - assert_eq!(nodes[0].id, NodeId::from("node-a")); - assert!(nodes[0].online); - assert!(!nodes[0].stale); - assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); - assert_eq!(nodes[0].capabilities.os, Os::Linux); - for (tenant, project, node, digest) in [ - ("tenant-a", "project-a", "node-a", "tenant-a-bytes"), - ("tenant-b", "project-b", "node-b", "tenant-b-bytes"), - ] { - service.artifact_registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("shared-artifact"), - tenant: TenantId::from(tenant), - project: ProjectId::from(project), - process: ProcessId::from("process-one"), - producer_task: TaskInstanceId::from("task-one"), - retaining_node: NodeId::from(node), - digest: Digest::sha256(digest), - size: digest.len() as u64, - }); - } - service.artifact_registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("second-artifact"), - tenant: TenantId::from("tenant-a"), - project: ProjectId::from("project-a"), - process: ProcessId::from("process-two"), - producer_task: TaskInstanceId::from("task-two"), - retaining_node: NodeId::from("node-a"), - digest: Digest::sha256("second"), - size: 6, - }); - - let CoordinatorResponse::Artifacts { - artifacts, - next_cursor: Some(cursor), - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListArtifacts { - process: None, - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected first artifact page"); - }; - assert_eq!(artifacts.len(), 1); - assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact")); - assert_eq!(artifacts[0].availability, ArtifactAvailability::Available); - let CoordinatorResponse::Artifacts { - artifacts, - next_cursor: None, - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListArtifacts { - process: None, - cursor: Some(cursor), - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected final artifact page"); - }; - assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact")); - assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes")); - - let cross_tenant = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "tenant-b-only".to_owned(), - }, - }) - .unwrap_err(); - assert!(cross_tenant.to_string().contains("does not exist")); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected tenant-b artifact"); - }; - assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes")); - - service.set_server_time(131); - let CoordinatorResponse::NodeSummaries { nodes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected stale node summary"); - }; - assert!(!nodes[0].online); - assert!(nodes[0].stale); - assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected offline artifact metadata"); - }; - assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline); - assert!(!artifact.downloadable_now); - - service - .artifact_registry - .sync_to_explicit_store( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - &ArtifactId::from("shared-artifact"), - "store://tenant-a/shared-artifact", - ) - .unwrap(); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected explicitly retained artifact metadata"); - }; - assert_eq!(artifact.availability, ArtifactAvailability::Available); - assert_eq!( - artifact.retention_state, - ArtifactRetentionState::ExplicitStorage - ); - assert!(artifact.downloadable_now); -} - -#[test] -fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret, node, process) in [ - ( - "tenant-a", - "project-a", - "user-a", - "session-a", - "node-a", - "process-shared", - ), - ( - "tenant-b", - "project-b", - "user-b", - "session-b", - "node-b", - "process-b", - ), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - enroll_test_node( - &mut service, - tenant, - project, - node, - &test_node_public_key(node), - ); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: secret.to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: process.to_owned(), - restart: false, - }, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - process: process.to_owned(), - epoch: 7, - }) - .unwrap(); - } - service.set_server_time(100); - let first = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 0, - source_bytes: 5, - text: "hello".to_owned(), - truncated: false, - }) - .unwrap(); - let CoordinatorResponse::TaskLogChunkRecorded { - sequence: Some(first_sequence), - next_offset: 5, - .. - } = first - else { - panic!("expected first live log sequence"); - }; - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 5, - "live bytes must be charged when accepted" - ); - let retry = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 0, - source_bytes: 5, - text: "hello".to_owned(), - truncated: false, - }) - .unwrap(); - assert!(matches!( - retry, - CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. } - )); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 5, - "a retried chunk must not be charged twice" - ); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 8, - source_bytes: 2, - text: "ok".to_owned(), - truncated: false, - }) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 10, - "a gap and the delivered bytes must both count toward source-byte usage" - ); - - let CoordinatorResponse::RecentLogs { - entries, - next_sequence: Some(cursor), - history_truncated: false, - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected first recent-log page"); - }; - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].sequence, first_sequence); - assert_eq!(entries[0].text, "hello"); - assert!(entries[1].text.contains("3 bytes")); - assert!(entries[1].truncated); - let CoordinatorResponse::RecentLogs { entries, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: Some(cursor), - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected second recent-log page"); - }; - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].text, "ok"); - - service - .handle_report_task_log( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - 12, - 0, - "hello???okZZ".to_owned(), - String::new(), - false, - false, - false, - ) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 12, - "the final summary must charge only source bytes not already charged live" - ); - assert_eq!( - service - .recent_logs - .get(&(TenantId::from("tenant-a"), ProjectId::from("project-a"))) - .unwrap() - .back() - .unwrap() - .text, - "ZZ", - "final-tail reconciliation must append only the nonduplicating suffix" - ); - service - .handle_report_task_log( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - 12, - 0, - "hello???okZZ".to_owned(), - String::new(), - false, - false, - false, - ) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 12, - "replayed final accounting must be idempotent" - ); - - let marker = service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - 12, - 0, - "[log output truncated at node capture limit]".to_owned(), - true, - ) - .unwrap(); - assert!(matches!( - marker, - CoordinatorResponse::TaskLogChunkRecorded { - sequence: Some(_), - next_offset: 12, - .. - } - )); - let repeated_marker = service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - 12, - 0, - "[log output truncated at node capture limit]".to_owned(), - true, - ) - .unwrap(); - assert!(matches!( - repeated_marker, - CoordinatorResponse::TaskLogChunkRecorded { - sequence: None, - next_offset: 12, - .. - } - )); - - let cross_tenant = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 10, - }, - }) - .unwrap_err(); - assert!(cross_tenant.to_string().contains("outside")); - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 201, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("200")); - - service - .handle_report_task_log_chunk( - "tenant-b".to_owned(), - "project-b".to_owned(), - "process-b".to_owned(), - "node-b".to_owned(), - "task-b".to_owned(), - TaskLogStream::Stderr, - 0, - 1, - "b".to_owned(), - false, - ) - .unwrap(); - for offset in 10..310 { - service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - offset, - 1, - "x".to_owned(), - false, - ) - .unwrap(); - } - let tenant_a_logs = - &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))]; - assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS); - let tenant_b_logs = - &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))]; - assert_eq!(tenant_b_logs.len(), 1); - assert_eq!(tenant_b_logs[0].text, "b"); - let CoordinatorResponse::RecentLogs { - entries, - history_truncated, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected bounded recent-log response"); - }; - assert_eq!(entries.len(), 200); - assert!(history_truncated); -} diff --git a/crates/clusterflux-coordinator/src/service/wire_protocol.rs b/crates/clusterflux-coordinator/src/service/wire_protocol.rs index 07aec2a..4884415 100644 --- a/crates/clusterflux-coordinator/src/service/wire_protocol.rs +++ b/crates/clusterflux-coordinator/src/service/wire_protocol.rs @@ -1,4 +1,4 @@ -use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE}; +use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -12,12 +12,8 @@ pub enum CoordinatorWireRequest { impl CoordinatorWireRequest { pub fn into_request(self) -> Result { - self.into_parts().map(|(_, request)| request) - } - - pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { match self { - Self::Envelope(envelope) => envelope.into_parts(), + Self::Envelope(envelope) => envelope.into_request(), } } } @@ -36,10 +32,6 @@ pub struct CoordinatorRequestEnvelope { impl CoordinatorRequestEnvelope { pub fn into_request(self) -> Result { - self.into_parts().map(|(_, request)| request) - } - - pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE { return Err(format!( "unsupported coordinator wire request type {}; expected {}", @@ -52,9 +44,9 @@ impl CoordinatorRequestEnvelope { self.protocol_version, COORDINATOR_PROTOCOL_VERSION )); } - RequestId::try_new(self.request_id.clone()) - .map_err(|error| format!("malformed coordinator wire request_id: {error}"))?; - self.payload.validate_external_identifiers()?; + if self.request_id.trim().is_empty() { + return Err("coordinator wire request_id must be non-empty".to_owned()); + } let payload_operation = self.payload.operation()?; if self.operation != payload_operation { return Err(format!( @@ -62,6 +54,6 @@ impl CoordinatorRequestEnvelope { self.operation, payload_operation )); } - Ok((self.request_id, self.payload)) + Ok(self.payload) } } diff --git a/crates/clusterflux-core/src/api_error.rs b/crates/clusterflux-core/src/api_error.rs deleted file mode 100644 index b6625cf..0000000 --- a/crates/clusterflux-core/src/api_error.rs +++ /dev/null @@ -1,296 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::fmt; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ApiErrorCode { - Unauthenticated, - SessionExpired, - AccountSuspended, - Forbidden, - ValidationError, - NotFound, - Conflict, - ActiveProcessExists, - NodeOffline, - NoCapableNode, - TaskNotRestartable, - ArtifactUnavailable, - ArtifactLimitExceeded, - QuotaExceeded, - TemporaryCapacity, - DebugEpochPartial, - InternalError, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ApiErrorCategory { - Authentication, - Authorization, - Validation, - State, - Availability, - Resource, - Internal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ApiError { - pub code: ApiErrorCode, - pub category: ApiErrorCategory, - pub message: String, - pub retryable: bool, - pub request_id: String, -} - -impl fmt::Display for ApiError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "{} (code {:?}, request {})", - self.message, self.code, self.request_id - ) - } -} - -impl std::error::Error for ApiError {} - -impl ApiError { - pub fn new( - code: ApiErrorCode, - category: ApiErrorCategory, - message: impl Into, - retryable: bool, - request_id: impl Into, - ) -> Self { - Self { - code, - category, - message: message.into(), - retryable, - request_id: request_id.into(), - } - } - - pub fn from_message(request_id: impl Into, message: impl Into) -> Self { - let request_id = request_id.into(); - let message = message.into(); - let normalized = message.to_ascii_lowercase(); - let (code, category, retryable) = if normalized.contains("session credential has expired") - || normalized.contains("session expired") - { - ( - ApiErrorCode::SessionExpired, - ApiErrorCategory::Authentication, - false, - ) - } else if normalized.contains("tenant is suspended") - || normalized.contains("account is suspended") - || normalized.contains("suspended by hosted") - { - ( - ApiErrorCode::AccountSuspended, - ApiErrorCategory::Authorization, - false, - ) - } else if normalized.contains("session credential") - || normalized.contains("no authenticated") - || normalized.contains("not authenticated") - || normalized.contains("credential is not") - { - ( - ApiErrorCode::Unauthenticated, - ApiErrorCategory::Authentication, - false, - ) - } else if normalized.contains("project already has active virtual process") { - ( - ApiErrorCode::ActiveProcessExists, - ApiErrorCategory::State, - false, - ) - } else if normalized.contains("no capable node") { - ( - ApiErrorCode::NoCapableNode, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("node offline") - || normalized.contains("node is not live") - || normalized.contains("source node is not connected") - || normalized.contains("direct connectivity unavailable") - { - ( - ApiErrorCode::NodeOffline, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("restart") - && (normalized.contains("not restartable") - || normalized.contains("requires whole") - || normalized.contains("clean boundary")) - { - ( - ApiErrorCode::TaskNotRestartable, - ApiErrorCategory::State, - false, - ) - } else if normalized.contains("artifact") - && (normalized.contains("exceeds download limit") - || normalized.contains("download session limit") - || normalized.contains("artifact limit")) - { - ( - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCategory::Resource, - false, - ) - } else if normalized.contains("artifact") - && (normalized.contains("does not exist") - || normalized.contains("not found") - || normalized.contains("unknown artifact")) - { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } else if normalized.contains("artifact") - && (normalized.contains("unavailable") || normalized.contains("retention")) - { - ( - ApiErrorCode::ArtifactUnavailable, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("resource limit") - || normalized.contains("quota") - || normalized.contains("limit exceeded") - { - ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ) - } else if normalized.contains("capacity") - || normalized.contains("temporarily full") - || normalized.contains("replay window is full") - { - ( - ApiErrorCode::TemporaryCapacity, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("partial debug epoch") - || normalized.contains("debug epoch is partially") - { - ( - ApiErrorCode::DebugEpochPartial, - ApiErrorCategory::State, - true, - ) - } else if normalized.contains("malformed") - || normalized.contains("invalid ") - || normalized.contains("protocol") - || normalized.contains("unknown field") - || normalized.contains("missing field") - || normalized.contains("must ") - { - ( - ApiErrorCode::ValidationError, - ApiErrorCategory::Validation, - false, - ) - } else if normalized.contains("outside") - || normalized.contains("unauthorized") - || normalized.contains("denied") - || normalized.contains("requires an authenticated") - || normalized.contains("may only") - { - ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ) - } else if normalized.contains("not found") - || normalized.contains("does not exist") - || normalized.contains("unknown ") - { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } else if normalized.contains("already") - || normalized.contains("conflict") - || normalized.contains("requires an active") - { - (ApiErrorCode::Conflict, ApiErrorCategory::State, false) - } else { - ( - ApiErrorCode::InternalError, - ApiErrorCategory::Internal, - false, - ) - }; - Self::new(code, category, message, retryable, request_id) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn required_machine_codes_are_stably_serialized() { - let required = [ - ApiErrorCode::Unauthenticated, - ApiErrorCode::SessionExpired, - ApiErrorCode::AccountSuspended, - ApiErrorCode::Forbidden, - ApiErrorCode::ValidationError, - ApiErrorCode::NotFound, - ApiErrorCode::Conflict, - ApiErrorCode::ActiveProcessExists, - ApiErrorCode::NodeOffline, - ApiErrorCode::NoCapableNode, - ApiErrorCode::TaskNotRestartable, - ApiErrorCode::ArtifactUnavailable, - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCode::QuotaExceeded, - ApiErrorCode::TemporaryCapacity, - ApiErrorCode::DebugEpochPartial, - ]; - let serialized = required - .into_iter() - .map(|code| serde_json::to_value(code).unwrap()) - .collect::>(); - assert_eq!( - serialized, - vec![ - "unauthenticated", - "session_expired", - "account_suspended", - "forbidden", - "validation_error", - "not_found", - "conflict", - "active_process_exists", - "node_offline", - "no_capable_node", - "task_not_restartable", - "artifact_unavailable", - "artifact_limit_exceeded", - "quota_exceeded", - "temporary_capacity", - "debug_epoch_partial", - ] - ); - } - - #[test] - fn message_classification_keeps_request_identity() { - let error = ApiError::from_message( - "request-17", - "CLI session credential has expired; run login again", - ); - assert_eq!(error.code, ApiErrorCode::SessionExpired); - assert_eq!(error.category, ApiErrorCategory::Authentication); - assert_eq!(error.request_id, "request-17"); - assert!(!error.retryable); - } -} diff --git a/crates/clusterflux-core/src/artifact.rs b/crates/clusterflux-core/src/artifact.rs index d9c89a3..f4f5311 100644 --- a/crates/clusterflux-core/src/artifact.rs +++ b/crates/clusterflux-core/src/artifact.rs @@ -10,7 +10,7 @@ use crate::{ const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32; const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60; -const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024; +const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -134,27 +134,6 @@ pub struct ArtifactMetadata { pub coordinator_has_large_bytes: bool, } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub struct ArtifactScopeKey { - pub tenant: TenantId, - pub project: ProjectId, - pub artifact: ArtifactId, -} - -impl ArtifactScopeKey { - pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self { - Self { - tenant, - project, - artifact, - } - } - - pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self { - Self::new(tenant.clone(), project.clone(), artifact.clone()) - } -} - #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArtifactFlush { pub id: ArtifactId, @@ -169,7 +148,7 @@ pub struct ArtifactFlush { #[derive(Clone, Debug, Default)] pub struct ArtifactRegistry { - artifacts: BTreeMap, + artifacts: BTreeMap, issued_download_links: BTreeMap, next_epoch: u64, } @@ -183,46 +162,42 @@ impl ArtifactRegistry { pub fn flush_metadata_bounded( &mut self, flush: ArtifactFlush, - pinned: &BTreeSet, + pinned: &BTreeSet, ) -> Result { - self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new()) - } - - pub fn flush_metadata_with_protected_processes( - &mut self, - flush: ArtifactFlush, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> Result { - let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id); - let replacing = self.artifacts.contains_key(&key); - let retained_for_project = self - .artifacts - .values() - .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project) - .count(); - let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT { - None - } else { - let mut protected_processes = protected_processes.clone(); - protected_processes.insert(flush.process.clone()); - Some( - self.project_metadata_eviction_candidate( - &flush.tenant, - &flush.project, - pinned, - &protected_processes, - ) + let replacing_existing = self.artifacts.contains_key(&flush.id); + while !replacing_existing + && self + .artifacts + .values() + .filter(|metadata| { + metadata.tenant == flush.tenant + && metadata.project == flush.project + && metadata.process == flush.process + }) + .count() + >= MAX_ARTIFACT_METADATA_PER_PROCESS + { + let candidate = self + .artifacts + .values() + .filter(|metadata| { + metadata.tenant == flush.tenant + && metadata.project == flush.project + && metadata.process == flush.process + && !pinned.contains(&metadata.id) + && !self.issued_download_links.values().any(|issued| { + issued.link.artifact == metadata.id + }) + }) + .min_by_key(|metadata| metadata.flushed_epoch) + .map(|metadata| metadata.id.clone()) .ok_or_else(|| { - format!( - "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \ - exhausted by active or retained artifacts" - ) - })?, - ) - }; - - self.next_epoch = self.next_epoch.saturating_add(1); + "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download" + .to_owned() + })?; + self.artifacts.remove(&candidate); + } + self.next_epoch += 1; let metadata = ArtifactMetadata { id: flush.id.clone(), tenant: flush.tenant, @@ -237,112 +212,36 @@ impl ArtifactRegistry { explicit_locations: Vec::new(), coordinator_has_large_bytes: false, }; - if let Some(eviction) = eviction { - self.artifacts.remove(&eviction); - } - self.artifacts.insert(key, metadata.clone()); + self.artifacts.insert(flush.id, metadata.clone()); Ok(metadata) } - pub fn enforce_project_metadata_limit( - &mut self, - tenant: &TenantId, - project: &ProjectId, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> usize { - let mut evicted = 0; - while self - .artifacts - .values() - .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) - .count() - > MAX_ARTIFACT_METADATA_PER_PROJECT - { - let candidate = self.project_metadata_eviction_candidate( - tenant, - project, - pinned, - protected_processes, - ); - let Some(candidate) = candidate else { - break; - }; - self.artifacts.remove(&candidate); - evicted += 1; - } - evicted - } - - fn project_metadata_eviction_candidate( - &self, - tenant: &TenantId, - project: &ProjectId, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> Option { - self.artifacts - .values() - .filter(|metadata| { - &metadata.tenant == tenant - && &metadata.project == project - && !pinned.contains(&ArtifactScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - &metadata.id, - )) - && !protected_processes.contains(&metadata.process) - && metadata.explicit_locations.is_empty() - && !self.issued_download_links.values().any(|issued| { - !issued.revoked - && issued.link.tenant == metadata.tenant - && issued.link.project == metadata.project - && issued.link.artifact == metadata.id - }) - }) - .min_by_key(|metadata| metadata.flushed_epoch) - .map(|metadata| { - ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id) - }) - } - pub fn sync_to_explicit_store( &mut self, - tenant: &TenantId, - project: &ProjectId, artifact: &ArtifactId, location: impl Into, ) -> Result<(), ArtifactUnavailable> { let metadata = self .artifacts - .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact)) + .get_mut(artifact) .ok_or(ArtifactUnavailable)?; metadata.explicit_locations.push(location.into()); Ok(()) } - pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) { - for metadata in self - .artifacts - .values_mut() - .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) - { + pub fn garbage_collect_node(&mut self, node: &NodeId) { + for metadata in self.artifacts.values_mut() { metadata.retaining_nodes.remove(node); } } pub fn reconcile_node_retention( &mut self, - tenant: &TenantId, - project: &ProjectId, node: &NodeId, retained_artifacts: &BTreeSet, ) { - for (key, metadata) in &mut self.artifacts { - if &key.tenant != tenant || &key.project != project { - continue; - } - if retained_artifacts.contains(&key.artifact) { + for (artifact, metadata) in &mut self.artifacts { + if retained_artifacts.contains(artifact) { metadata.retaining_nodes.insert(node.clone()); } else { metadata.retaining_nodes.remove(node); @@ -350,24 +249,8 @@ impl ArtifactRegistry { } } - pub fn metadata( - &self, - tenant: &TenantId, - project: &ProjectId, - artifact: &ArtifactId, - ) -> Option<&ArtifactMetadata> { - self.artifacts - .get(&ArtifactScopeKey::from_refs(tenant, project, artifact)) - } - - pub fn metadata_for_project<'a>( - &'a self, - tenant: &'a TenantId, - project: &'a ProjectId, - ) -> impl Iterator + 'a { - self.artifacts - .values() - .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project) + pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> { + self.artifacts.get(artifact) } pub fn download_action( @@ -378,11 +261,7 @@ impl ArtifactRegistry { ) -> Result { let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; let scope = Scope { tenant: metadata.tenant.clone(), @@ -436,11 +315,7 @@ impl ArtifactRegistry { self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; Ok(metadata.size) } @@ -458,11 +333,7 @@ impl ArtifactRegistry { if self .issued_download_links .values() - .filter(|issued| { - issued.link.tenant == context.tenant - && issued.link.project == context.project - && issued.link.artifact == *artifact - }) + .filter(|issued| issued.link.artifact == *artifact) .count() >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { @@ -474,11 +345,7 @@ impl ArtifactRegistry { let action = self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds); let policy_context_digest = @@ -531,11 +398,7 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.tenant != context.tenant - || issued.link.project != context.project - || issued.link.artifact != *artifact - || issued.link.actor != context.actor - { + if issued.link.artifact != *artifact || issued.link.actor != context.actor { return Err(DownloadError::InvalidToken); } self.download_action( @@ -571,9 +434,7 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.tenant != context.tenant - || issued.link.project != context.project - || issued.link.artifact != *artifact + if issued.link.artifact != *artifact || issued.link.max_bytes != policy.max_bytes || issued.link.actor != context.actor { @@ -591,11 +452,7 @@ impl ArtifactRegistry { } let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; if download_policy_context_digest(metadata, &action.source, policy) != issued.link.policy_context_digest @@ -620,11 +477,7 @@ impl ArtifactRegistry { ) -> Result<(), DownloadError> { let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &stream.link.tenant, - &stream.link.project, - &stream.link.artifact, - )) + .get(&stream.link.artifact) .ok_or(DownloadError::NotFound)?; if !source_is_available(metadata, &stream.link.source) { return Err(DownloadError::Unavailable); @@ -743,13 +596,7 @@ mod tests { #[test] fn flush_publishes_metadata_without_coordinator_bytes() { let registry = registry_with_artifact(); - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) - .unwrap(); + let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); assert!(!metadata.coordinator_has_large_bytes); assert_eq!(metadata.id, ArtifactId::from("artifact")); @@ -768,11 +615,7 @@ mod tests { #[test] fn unsynced_node_loss_surfaces_as_unavailable() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -793,20 +636,9 @@ mod tests { #[test] fn signed_node_retention_inventory_removes_garbage_collected_location() { let mut registry = registry_with_artifact(); - registry.reconcile_node_retention( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - &BTreeSet::new(), - ); + registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new()); - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) - .unwrap(); + let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); assert!(metadata.retaining_nodes.is_empty()); } @@ -816,26 +648,11 @@ mod tests { let mut registry = registry_with_artifact(); let node = NodeId::from("node"); let artifact = ArtifactId::from("artifact"); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &node, - ); + registry.garbage_collect_node(&node); - registry.reconcile_node_retention( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &node, - &BTreeSet::from([artifact.clone()]), - ); + registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()])); - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &artifact, - ) - .unwrap(); + let metadata = registry.metadata(&artifact).unwrap(); assert!(metadata.retaining_nodes.contains(&node)); } @@ -843,18 +660,9 @@ mod tests { fn explicit_user_storage_location_survives_node_retention_loss() { let mut registry = registry_with_artifact(); registry - .sync_to_explicit_store( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - "s3://bucket/app", - ) + .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app") .unwrap(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -875,11 +683,7 @@ mod tests { ); assert!( !registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) + .metadata(&ArtifactId::from("artifact")) .unwrap() .coordinator_has_large_bytes ); @@ -910,7 +714,7 @@ mod tests { ) .unwrap_err(); - assert_eq!(error, DownloadError::NotFound); + assert!(matches!(error, DownloadError::Unauthorized(_))); } #[test] @@ -930,206 +734,13 @@ mod tests { ) .unwrap_err(); - assert_eq!(error, DownloadError::NotFound); - } - - #[test] - fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() { - let artifact = ArtifactId::from("shared-artifact"); - let tenant_a = TenantId::from("tenant-a"); - let project_a = ProjectId::from("project-a"); - let tenant_b = TenantId::from("tenant-b"); - let project_b = ProjectId::from("project-b"); - let project_c = ProjectId::from("project-c"); - let node_a = NodeId::from("node-a"); - let node_b = NodeId::from("node-b"); - let node_c = NodeId::from("node-c"); - let mut registry = ArtifactRegistry::default(); - for (tenant, project, process, node, content, size) in [ - (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17), - (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29), - (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31), - ] { - registry.flush_metadata(ArtifactFlush { - id: artifact.clone(), - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::from(process), - producer_task: TaskInstanceId::from("producer"), - retaining_node: node.clone(), - digest: Digest::sha256(content), - size, - }); - } - - assert_eq!(registry.artifact_count(), 3); - assert_eq!( - registry - .metadata(&tenant_a, &project_a, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-a") - ); - assert_eq!( - registry - .metadata(&tenant_b, &project_b, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-b") - ); - assert_eq!( - registry - .metadata(&tenant_a, &project_c, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-c") - ); - registry.flush_metadata(ArtifactFlush { - id: artifact.clone(), - tenant: tenant_a.clone(), - project: project_a.clone(), - process: ProcessId::from("process-a"), - producer_task: TaskInstanceId::from("producer-repeat"), - retaining_node: node_a.clone(), - digest: Digest::sha256("bytes-a-repeat"), - size: 18, - }); - assert_eq!(registry.artifact_count(), 3); - assert_eq!( - registry - .metadata(&tenant_a, &project_a, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-a-repeat") - ); - assert_eq!( - registry - .metadata(&tenant_b, &project_b, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-b") - ); - - registry - .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a") - .unwrap(); - registry.garbage_collect_node(&tenant_a, &project_a, &node_a); - let context_a = AuthContext { - tenant: tenant_a.clone(), - project: project_a.clone(), - actor: Actor::User(UserId::from("user")), - }; - let context_b = AuthContext { - tenant: tenant_b.clone(), - project: project_b.clone(), - actor: Actor::User(UserId::from("user")), - }; - let policy = DownloadPolicy { max_bytes: 100 }; - assert_eq!( - registry - .download_action(&context_a, &artifact, &policy) - .unwrap() - .source, - StorageLocation::ExplicitStore("store://tenant-a".to_owned()) - ); - assert_eq!( - registry - .download_action(&context_b, &artifact, &policy) - .unwrap() - .source, - StorageLocation::RetainedNode(node_b) - ); - - let first_a = registry - .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60) - .unwrap(); - let first_b = registry - .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60) - .unwrap(); - assert_ne!( - first_a.scoped_token_digest, first_b.scoped_token_digest, - "the full artifact scope must contribute to download tokens" - ); - for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { - registry - .create_download_link( - &context_a, - &artifact, - &policy, - &format!("tenant-a-{index}"), - 10, - 60, - ) - .unwrap(); - } - assert!(matches!( - registry.create_download_link( - &context_a, - &artifact, - &policy, - "tenant-a-over-limit", - 10, - 60, - ), - Err(DownloadError::Usage(_)) - )); - registry - .create_download_link( - &context_b, - &artifact, - &policy, - "tenant-b-still-independent", - 10, - 60, - ) - .unwrap(); - registry - .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest) - .unwrap(); - - let limits = ResourceLimits { - limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]), - }; - let mut meter = ResourceMeter::default(); - registry - .open_download_stream( - DownloadStreamRequest { - context: &context_b, - artifact: &artifact, - policy: &policy, - presented_token_digest: &first_b.scoped_token_digest, - now_epoch_seconds: 11, - limits: &limits, - }, - &mut meter, - ) - .expect("revoking tenant A's link must not revoke tenant B's link"); - assert_eq!( - registry - .open_download_stream( - DownloadStreamRequest { - context: &context_b, - artifact: &artifact, - policy: &policy, - presented_token_digest: &first_a.scoped_token_digest, - now_epoch_seconds: 11, - limits: &limits, - }, - &mut meter, - ) - .unwrap_err(), - DownloadError::InvalidToken - ); + assert!(matches!(error, DownloadError::Unauthorized(_))); } #[test] fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -1406,200 +1017,51 @@ mod tests { } #[test] - fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() { + fn artifact_metadata_is_bounded_without_evicting_pins() { let mut registry = ArtifactRegistry::default(); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("artifact-0"), - tenant: TenantId::from("other-tenant"), - project: ProjectId::from("other-project"), - process: ProcessId::from("process"), - producer_task: TaskInstanceId::from("other-task"), - retaining_node: NodeId::from("other-node"), - digest: Digest::sha256("other-content"), - size: 1, - }); - for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { + let mut pinned = BTreeSet::new(); + for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS { let id = ArtifactId::new(format!("artifact-{index}")); - registry.flush_metadata(ArtifactFlush { - id, - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::new(if index == 3 { - "active-process-3".to_owned() - } else { - format!("completed-process-{index}") - }), - producer_task: TaskInstanceId::new(format!("task-{index}")), - retaining_node: NodeId::from("node"), - digest: Digest::sha256(format!("content-{index}")), - size: 1, - }); + pinned.insert(id.clone()); + registry + .flush_metadata_bounded( + ArtifactFlush { + id, + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), + producer_task: TaskInstanceId::new(format!("task-{index}")), + retaining_node: NodeId::from("node"), + digest: Digest::sha256(format!("content-{index}")), + size: 1, + }, + &pinned, + ) + .unwrap(); } - assert_eq!( - registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROJECT + 1 - ); - let pinned = BTreeSet::from([ArtifactScopeKey::new( - tenant.clone(), - project.clone(), - ArtifactId::from("artifact-0"), - )]); - registry - .sync_to_explicit_store( - &tenant, - &project, - &ArtifactId::from("artifact-1"), - "store://retained-export", - ) - .unwrap(); - let context = AuthContext { - tenant: tenant.clone(), - project: project.clone(), - actor: Actor::User(UserId::from("user")), - }; - registry - .create_download_link( - &context, - &ArtifactId::from("artifact-2"), - &DownloadPolicy { max_bytes: 1 }, - "active-download", - 10, - 60, - ) - .unwrap(); - let protected_processes = BTreeSet::from([ - ProcessId::from("active-process-3"), - ProcessId::from("active-process"), - ]); + assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); let next = ArtifactFlush { id: ArtifactId::from("artifact-next"), - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::from("active-process"), + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), producer_task: TaskInstanceId::from("task-next"), retaining_node: NodeId::from("node"), digest: Digest::sha256("next"), size: 1, }; - registry - .flush_metadata_with_protected_processes(next, &pinned, &protected_processes) - .unwrap(); - assert_eq!( - registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROJECT + 1 - ); - for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] { - assert!( - registry - .metadata(&tenant, &project, &ArtifactId::from(id)) - .is_some(), - "{id} was evicted despite being pinned, exported, downloaded, or live" - ); - } - assert!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-4")) - .is_none(), - "the oldest completed unprotected metadata should be evicted" - ); assert!(registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-next")) - .is_some()); - assert_eq!( - registry - .metadata( - &TenantId::from("other-tenant"), - &ProjectId::from("other-project"), - &ArtifactId::from("artifact-0"), - ) - .unwrap() - .digest, - Digest::sha256("other-content"), - "eviction and pins in one scope must not affect an equal ID in another scope" - ); - } - - #[test] - fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() { - let mut registry = ArtifactRegistry::default(); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let active_process = ProcessId::from("active-process"); - for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { - registry.flush_metadata(ArtifactFlush { - id: ArtifactId::new(format!("artifact-{index}")), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::new(format!("task-{index}")), - retaining_node: NodeId::from("node"), - digest: Digest::sha256(format!("content-{index}")), - size: 1, - }); - } - - let replacement = ArtifactFlush { - id: ArtifactId::from("artifact-0"), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::from("replacement-task"), - retaining_node: NodeId::from("node"), - digest: Digest::sha256("replacement"), - size: 2, - }; - registry - .flush_metadata_with_protected_processes( - replacement, - &BTreeSet::new(), - &BTreeSet::from([active_process.clone()]), - ) - .expect("replacement must remain possible at the metadata bound"); - assert_eq!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) - .unwrap() - .digest, - Digest::sha256("replacement") - ); - - let epoch_before_rejection = registry.next_epoch; - let result = registry.flush_metadata_with_protected_processes( - ArtifactFlush { - id: ArtifactId::from("artifact-rejected"), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::from("rejected-task"), - retaining_node: NodeId::from("node"), - digest: Digest::sha256("rejected"), - size: 3, - }, - &BTreeSet::new(), - &BTreeSet::from([active_process]), - ); - - assert!(result + .flush_metadata_bounded(next.clone(), &pinned) .unwrap_err() - .contains("exhausted by active or retained artifacts")); - assert_eq!(registry.next_epoch, epoch_before_rejection); - assert_eq!( - registry.metadata_for_project(&tenant, &project).count(), - MAX_ARTIFACT_METADATA_PER_PROJECT - ); + .contains("pinned")); + + pinned.remove(&ArtifactId::from("artifact-0")); + registry.flush_metadata_bounded(next, &pinned).unwrap(); + assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); + assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none()); assert!(registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected")) - .is_none()); - assert_eq!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) - .unwrap() - .digest, - Digest::sha256("replacement"), - "rejection must not modify existing metadata" - ); + .metadata(&ArtifactId::from("artifact-next")) + .is_some()); } #[test] @@ -1688,11 +1150,7 @@ mod tests { registry .stream_download_chunk(&mut stream, &limits, &mut meter, 16) .unwrap(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); assert_eq!( registry diff --git a/crates/clusterflux-core/src/auth.rs b/crates/clusterflux-core/src/auth.rs index 51686a2..61e99fb 100644 --- a/crates/clusterflux-core/src/auth.rs +++ b/crates/clusterflux-core/src/auth.rs @@ -364,13 +364,7 @@ pub fn agent_workflow_request_scope_from_payload( .get("task_instance") .and_then(Value::as_str) .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?; - ( - process, - Some( - TaskInstanceId::try_new(task) - .map_err(|error| format!("malformed launch_task task instance: {error}"))?, - ), - ) + (process, Some(TaskInstanceId::from(task))) } _ => { return Err(format!( @@ -379,13 +373,10 @@ pub fn agent_workflow_request_scope_from_payload( } }; AgentWorkflowRequestScope::new( - TenantId::try_new(tenant) - .map_err(|error| format!("malformed agent workflow tenant: {error}"))?, - ProjectId::try_new(project) - .map_err(|error| format!("malformed agent workflow project: {error}"))?, + TenantId::from(tenant), + ProjectId::from(project), request_kind, - ProcessId::try_new(process) - .map_err(|error| format!("malformed agent workflow process: {error}"))?, + ProcessId::from(process), task, ) } diff --git a/crates/clusterflux-core/src/ids.rs b/crates/clusterflux-core/src/ids.rs index 2e2744d..133c5f0 100644 --- a/crates/clusterflux-core/src/ids.rs +++ b/crates/clusterflux-core/src/ids.rs @@ -1,102 +1,18 @@ -#[cfg(not(target_arch = "wasm32"))] -use serde::{de::Error as _, Deserializer}; use serde::{Deserialize, Serialize}; -pub const MAX_EXTERNAL_ID_BYTES: usize = 255; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct OpaqueTokenError { - reason: String, -} - -impl std::fmt::Display for OpaqueTokenError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.reason) - } -} - -impl std::error::Error for OpaqueTokenError {} - -pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> { - let reason = if value.trim().is_empty() { - Some("value must not be empty or whitespace-only".to_owned()) - } else if value.len() > max_bytes { - Some(format!("value exceeds the {max_bytes}-byte limit")) - } else if value.chars().any(char::is_control) { - Some("control characters are forbidden".to_owned()) - } else { - None - }; - reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason })) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IdParseError { - id_type: &'static str, - reason: &'static str, -} - -impl IdParseError { - fn new(id_type: &'static str, reason: &'static str) -> Self { - Self { id_type, reason } - } -} - -impl std::fmt::Display for IdParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} is invalid: {}", self.id_type, self.reason) - } -} - -impl std::error::Error for IdParseError {} - -fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> { - if value.is_empty() || value.trim().is_empty() { - return Err(IdParseError::new( - id_type, - "value must not be empty or whitespace-only", - )); - } - if value.len() > MAX_EXTERNAL_ID_BYTES { - return Err(IdParseError::new( - id_type, - "value exceeds the 255-byte limit", - )); - } - if value.chars().any(char::is_control) { - return Err(IdParseError::new( - id_type, - "control characters are forbidden", - )); - } - if !value.bytes().all(|byte| { - byte.is_ascii_alphanumeric() - || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+') - }) { - return Err(IdParseError::new( - id_type, - "only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed", - )); - } - Ok(()) -} - macro_rules! id_type { ($name:ident) => { - #[cfg_attr(target_arch = "wasm32", derive(Deserialize))] - #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct $name(String); impl $name { - pub fn try_new(value: impl Into) -> Result { - let value = value.into(); - validate_id(&value, stringify!($name))?; - Ok(Self(value)) - } - - /// Constructs an identifier from a trusted, internally generated value. pub fn new(value: impl Into) -> Self { - Self::try_new(value).unwrap_or_else(|error| panic!("{error}")) + let value = value.into(); + assert!( + !value.trim().is_empty(), + concat!(stringify!($name), " cannot be empty") + ); + Self(value) } pub fn as_str(&self) -> &str { @@ -110,17 +26,6 @@ macro_rules! id_type { } } - #[cfg(not(target_arch = "wasm32"))] - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::try_new(value).map_err(D::Error::custom) - } - } - impl std::fmt::Display for $name { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.0) @@ -139,73 +44,3 @@ id_type!(TaskDefinitionId); id_type!(TaskInstanceId); id_type!(TenantId); id_type!(UserId); - -pub type DebugSessionId = ProcessId; -pub type RequestId = LaunchAttemptId; - -#[cfg(test)] -mod tests { - use super::*; - - fn assert_hostile_values(parse: impl Fn(String) -> Result) { - for value in [ - String::new(), - " ".to_owned(), - "bad\0id".to_owned(), - "bad id".to_owned(), - "x".repeat(MAX_EXTERNAL_ID_BYTES + 1), - ] { - assert!(parse(value).is_err()); - } - } - - #[test] - fn every_external_identifier_type_rejects_hostile_values() { - assert_hostile_values(AgentId::try_new); - assert_hostile_values(ArtifactId::try_new); - assert_hostile_values(DebugSessionId::try_new); - assert_hostile_values(NodeId::try_new); - assert_hostile_values(ProcessId::try_new); - assert_hostile_values(LaunchAttemptId::try_new); - assert_hostile_values(ProjectId::try_new); - assert_hostile_values(RequestId::try_new); - assert_hostile_values(TaskDefinitionId::try_new); - assert_hostile_values(TaskInstanceId::try_new); - assert_hostile_values(TenantId::try_new); - assert_hostile_values(UserId::try_new); - } - - #[test] - fn every_identifier_type_validates_during_deserialization() { - macro_rules! assert_deserialization { - ($id:ty) => { - assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok()); - let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err(); - assert!( - error.to_string().contains("is invalid"), - "{} produced an unexpected error: {error}", - stringify!($id) - ); - }; - } - - assert_deserialization!(AgentId); - assert_deserialization!(ArtifactId); - assert_deserialization!(NodeId); - assert_deserialization!(ProcessId); - assert_deserialization!(LaunchAttemptId); - assert_deserialization!(ProjectId); - assert_deserialization!(TaskDefinitionId); - assert_deserialization!(TaskInstanceId); - assert_deserialization!(TenantId); - assert_deserialization!(UserId); - } - - #[test] - fn opaque_tokens_are_bounded_without_using_identifier_syntax() { - validate_opaque_token("opaque secret/+==", 64).unwrap(); - assert!(validate_opaque_token("", 64).is_err()); - assert!(validate_opaque_token("bad\0token", 64).is_err()); - assert!(validate_opaque_token(&"x".repeat(65), 64).is_err()); - } -} diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index c9d5b77..a879f7a 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -1,4 +1,3 @@ -mod api_error; pub mod artifact; pub mod auth; pub mod bundle; @@ -19,11 +18,10 @@ pub mod transport; pub mod vfs; pub mod wire; -pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode}; pub use artifact::{ ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry, - ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, - DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation, + ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy, + DownloadStreamRequest, RetentionPolicy, StorageLocation, }; pub use auth::{ admin_request_proof, admin_request_proof_from_token_digest, @@ -68,9 +66,8 @@ pub use execution::{ WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION, }; pub use ids::{ - validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId, - NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId, - TenantId, UserId, MAX_EXTERNAL_ID_BYTES, + AgentId, ArtifactId, LaunchAttemptId, NodeId, ProcessId, ProjectId, TaskDefinitionId, + TaskInstanceId, TenantId, UserId, }; pub use limits::{ LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits, diff --git a/crates/clusterflux-core/src/vfs.rs b/crates/clusterflux-core/src/vfs.rs index 9f1b739..9e86e11 100644 --- a/crates/clusterflux-core/src/vfs.rs +++ b/crates/clusterflux-core/src/vfs.rs @@ -1,48 +1,18 @@ use std::collections::BTreeMap; -use serde::{de::Error as _, Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{Digest, NodeId, TaskInstanceId}; -pub const MAX_VFS_PATH_BYTES: usize = 4096; - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct VfsPath(String); impl VfsPath { pub fn new(path: impl Into) -> Result { let path = path.into(); if !path.starts_with("/vfs/") { - return Err(VfsError::invalid(path, "path must start with /vfs/")); - } - if path.len() > MAX_VFS_PATH_BYTES { - return Err(VfsError::invalid( - path, - "path exceeds the 4096-byte protocol limit", - )); - } - if path.chars().any(char::is_control) { - return Err(VfsError::invalid(path, "control characters are forbidden")); - } - if path.contains('\\') { - return Err(VfsError::invalid(path, "backslashes are forbidden")); - } - let relative = &path["/vfs/".len()..]; - if relative.is_empty() { - return Err(VfsError::invalid( - path, - "path after /vfs/ must not be empty", - )); - } - if relative - .split('/') - .any(|component| component.is_empty() || matches!(component, "." | "..")) - { - return Err(VfsError::invalid( - path, - "empty, '.', and '..' path components are forbidden", - )); + return Err(VfsError::InvalidPath(path)); } Ok(Self(path)) } @@ -52,16 +22,6 @@ impl VfsPath { } } -impl<'de> Deserialize<'de> for VfsPath { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let path = String::deserialize(deserializer)?; - Self::new(path).map_err(D::Error::custom) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VfsObject { pub path: VfsPath, @@ -103,18 +63,12 @@ pub enum ReuseDecision { #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum VfsError { - #[error("invalid VFS path {path:?}: {reason}")] - InvalidPath { path: String, reason: &'static str }, + #[error("VFS path must start with /vfs/: {0}")] + InvalidPath(String), #[error("path is not visible in the published VFS manifest: {0}")] NotVisible(String), } -impl VfsError { - fn invalid(path: String, reason: &'static str) -> Self { - Self::InvalidPath { path, reason } - } -} - #[derive(Clone, Debug)] pub struct VfsOverlay { task: TaskInstanceId, @@ -284,39 +238,4 @@ mod tests { assert_eq!(overlay.pending_len(), 0); } - - #[test] - fn vfs_path_rejects_the_complete_hostile_protocol_matrix() { - for invalid in [ - "vfs/artifacts/app".to_owned(), - "/vfs/".to_owned(), - "/vfs/artifacts//app".to_owned(), - "/vfs/artifacts/app/".to_owned(), - "/vfs/artifacts/./app".to_owned(), - "/vfs/artifacts/../app".to_owned(), - "/vfs/artifacts\\app".to_owned(), - "/vfs/artifacts/bad\0app".to_owned(), - format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)), - ] { - assert!( - VfsPath::new(&invalid).is_err(), - "hostile VFS path unexpectedly passed: {invalid:?}" - ); - assert!( - serde_json::from_value::(serde_json::json!(invalid)).is_err(), - "hostile VFS path unexpectedly deserialized" - ); - } - } - - #[test] - fn vfs_path_accepts_the_4096_byte_boundary() { - let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len())); - assert_eq!(path.len(), MAX_VFS_PATH_BYTES); - assert_eq!(VfsPath::new(&path).unwrap().as_str(), path); - - let overlong = format!("{path}x"); - assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1); - assert!(VfsPath::new(overlong).is_err()); - } } diff --git a/crates/clusterflux-dap/src/adapter.rs b/crates/clusterflux-dap/src/adapter.rs index 4e144ab..feeb1cb 100644 --- a/crates/clusterflux-dap/src/adapter.rs +++ b/crates/clusterflux-dap/src/adapter.rs @@ -55,7 +55,6 @@ enum AdapterEvent { }, BreakpointsUpdated { generation: u64, - revision: u64, result: std::result::Result<(), String>, }, } @@ -63,8 +62,6 @@ enum AdapterEvent { struct LaunchCompletion { state: AdapterState, local_runtime_session: Option, - breakpoint_revision: u64, - observation_diagnostics: Vec, } pub(crate) fn run_adapter() -> Result<()> { @@ -96,41 +93,16 @@ pub(crate) fn run_adapter() -> Result<()> { runtime_starting = false; match result { Ok(mut completion) => { - let requested_breakpoints = state.breakpoints.clone(); - let requested_revision = state.breakpoint_revision; - let installed_revision = completion.breakpoint_revision; - completion.state.breakpoints = requested_breakpoints; - completion.state.breakpoint_revision = requested_revision; - completion.state.breakpoints_installed = - installed_revision == requested_revision; + completion.state.breakpoints = state.breakpoints.clone(); + completion.state.breakpoints_installed = true; let previous_threads = state.threads.clone(); state = completion.state; emit_thread_lifecycle(&mut writer, &previous_threads, &state.threads)?; if let Some(session) = completion.local_runtime_session { _local_runtime_session = Some(session); } - for diagnostic in completion.observation_diagnostics { - writer.output("stderr", format!("{diagnostic}\n"))?; - } runtime_started = true; - if state.breakpoints_installed { - emit_verified_breakpoints(&mut writer, &state)?; - } else { - let breakpoint_state = state.clone(); - let breakpoint_tx = event_tx.clone(); - let revision = state.breakpoint_revision; - thread::spawn(move || { - let result = set_services_debug_breakpoints(&breakpoint_state) - .map_err(|error| { - format!("coordinator breakpoint update failed: {error:#}") - }); - let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated { - generation, - revision, - result, - }); - }); - } + emit_verified_breakpoints(&mut writer, &state)?; writer.output( "console", if state.session_mode == DapSessionMode::Attach { @@ -155,7 +127,15 @@ pub(crate) fn run_adapter() -> Result<()> { Err(message) => { runtime_started = false; writer.output("stderr", format!("{message}\n"))?; - writer.event("terminated", json!({ "restart": false }))?; + writer.event( + "stopped", + json!({ + "reason": "exception", + "description": message, + "threadId": MAIN_THREAD, + "allThreadsStopped": false, + }), + )?; } } continue; @@ -251,17 +231,8 @@ pub(crate) fn run_adapter() -> Result<()> { } continue; } - AdapterEvent::BreakpointsUpdated { - generation, - revision, - result, - } => { - if breakpoint_update_is_current( - generation, - runtime_generation, - revision, - state.breakpoint_revision, - ) { + AdapterEvent::BreakpointsUpdated { generation, result } => { + if generation == runtime_generation { match result { Ok(()) => { state.breakpoints_installed = true; @@ -335,16 +306,7 @@ pub(crate) fn run_adapter() -> Result<()> { .and_then(Value::as_bool) .unwrap_or(false); if let Some(process_id) = args.get("processId").and_then(Value::as_str) { - match clusterflux_core::ProcessId::try_new(process_id.to_owned()) { - Ok(process) => state.process = process, - Err(error) => { - writer.error_response( - &request, - format!("invalid DAP processId: {error}"), - )?; - continue; - } - } + state.process = clusterflux_core::ProcessId::new(process_id); } if command == "attach" { state.session_mode = DapSessionMode::Attach; @@ -376,8 +338,6 @@ pub(crate) fn run_adapter() -> Result<()> { requested_source_path, requested_lines, ); - state.breakpoint_revision = state.breakpoint_revision.saturating_add(1); - let breakpoint_revision = state.breakpoint_revision; let coordinator_backed = matches!( state.runtime_backend, RuntimeBackend::LocalServices | RuntimeBackend::LiveServices @@ -411,11 +371,8 @@ pub(crate) fn run_adapter() -> Result<()> { set_services_debug_breakpoints(&breakpoint_state).map_err(|error| { format!("coordinator breakpoint update failed: {error:#}") }); - let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated { - generation, - revision: breakpoint_revision, - result, - }); + let _ = breakpoint_tx + .send(AdapterEvent::BreakpointsUpdated { generation, result }); }); } } @@ -476,18 +433,11 @@ pub(crate) fn run_adapter() -> Result<()> { let mut completed_state = launch_state; let result = if completed_state.session_mode == DapSessionMode::Attach { attach_services_runtime(&completed_state) - .and_then(|record| { + .map(|record| { completed_state.apply_attach_record(record); - set_services_debug_breakpoints(&completed_state)?; - Ok(()) - }) - .map(|()| { - let breakpoint_revision = completed_state.breakpoint_revision; LaunchCompletion { state: completed_state, local_runtime_session: None, - breakpoint_revision, - observation_diagnostics: Vec::new(), } }) .map_err(|error| format!("services runtime attach failed: {error:#}")) @@ -496,16 +446,10 @@ pub(crate) fn run_adapter() -> Result<()> { RuntimeBackend::LocalServices => { run_local_services_runtime(&completed_state) .map(|(record, session)| { - let observation_diagnostics = - runtime_observation_diagnostics(&record); completed_state.apply_runtime_record(record); - let breakpoint_revision = - completed_state.breakpoint_revision; LaunchCompletion { state: completed_state, local_runtime_session: Some(session), - breakpoint_revision, - observation_diagnostics, } }) .map_err(|error| { @@ -515,16 +459,10 @@ pub(crate) fn run_adapter() -> Result<()> { RuntimeBackend::LiveServices => { run_live_services_runtime(&completed_state) .map(|record| { - let observation_diagnostics = - runtime_observation_diagnostics(&record); completed_state.apply_runtime_record(record); - let breakpoint_revision = - completed_state.breakpoint_revision; LaunchCompletion { state: completed_state, local_runtime_session: None, - breakpoint_revision, - observation_diagnostics, } }) .map_err(|error| { @@ -1067,29 +1005,6 @@ fn spawn_runtime_observer( }); } -pub(crate) fn breakpoint_update_is_current( - update_generation: u64, - runtime_generation: u64, - update_revision: u64, - current_revision: u64, -) -> bool { - update_generation == runtime_generation && update_revision == current_revision -} - -fn runtime_observation_diagnostics( - record: &crate::virtual_model::RuntimeLaunchRecord, -) -> Vec { - record - .node_report - .get("observation_diagnostics") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() -} - fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> { if !state.breakpoints_installed { return Ok(()); @@ -1208,12 +1123,10 @@ fn emit_runtime_outcome( )?; } RuntimeContinuationOutcome::Terminal(record) => { - let exit_code = record.status_code.unwrap_or(1); apply_runtime_record_with_thread_events(writer, state, record)?; if state.last_task_failed { writer.output("stderr", format!("{}\n", state.command_status))?; } - writer.event("exited", json!({ "exitCode": exit_code }))?; writer.event("terminated", json!({}))?; } } diff --git a/crates/clusterflux-dap/src/main.rs b/crates/clusterflux-dap/src/main.rs index b0a9c14..32336b2 100644 --- a/crates/clusterflux-dap/src/main.rs +++ b/crates/clusterflux-dap/src/main.rs @@ -22,7 +22,7 @@ use breakpoints::{ #[cfg(test)] use dap_protocol::{initialize_capabilities, read_message}; #[cfg(test)] -use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code}; +use runtime_client::{client_user_request, parse_task_restart_response}; #[cfg(test)] use variables::variables_response; #[cfg(test)] @@ -36,26 +36,6 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD}; use virtual_model::{process_id, RuntimeBackend}; fn main() -> Result<()> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux Debug Adapter Protocol server.\n\n\ - Usage: clusterflux-debug-dap\n\n\ - The adapter communicates over standard input and output.\n\n\ - Options:\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - [] => {} - [argument, ..] => anyhow::bail!("unknown argument: {argument}"), - } adapter::run_adapter() } diff --git a/crates/clusterflux-dap/src/runtime_client.rs b/crates/clusterflux-dap/src/runtime_client.rs index 9b483ac..192f931 100644 --- a/crates/clusterflux-dap/src/runtime_client.rs +++ b/crates/clusterflux-dap/src/runtime_client.rs @@ -422,31 +422,8 @@ fn launch_services_debug_entrypoint( } // The launch acknowledgement is the commit point. Failures while fetching // observation state after this line must not abort a process that is running. - let mut observation_diagnostics = Vec::new(); - let inject_post_commit_observation_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_POST_COMMIT_OBSERVATION_FAILURE").is_some(); - let task_snapshots = (if inject_post_commit_observation_failure { - Err(anyhow!("injected post-commit task observation failure")) - } else { - fetch_task_snapshots(coordinator, state) - }) - .unwrap_or_else(|error| { - observation_diagnostics.push(format!( - "initial task observation failed after main_launched: {error:#}" - )); - json!({ "snapshots": [] }) - }); - let (process_statuses, process_status) = (if inject_post_commit_observation_failure { - Err(anyhow!("injected post-commit process observation failure")) - } else { - fetch_current_process_status(coordinator, state) - }) - .unwrap_or_else(|error| { - observation_diagnostics.push(format!( - "initial process observation failed after main_launched: {error:#}" - )); - (json!({ "processes": [] }), None) - }); + let task_snapshots = fetch_task_snapshots(coordinator, state)?; + let (process_statuses, process_status) = fetch_current_process_status(coordinator, state)?; let node = process_status .as_ref() .and_then(|status| status.get("connected_nodes")) @@ -464,7 +441,6 @@ fn launch_services_debug_entrypoint( "process_status": process_status, "process_statuses": process_statuses, "task_snapshots": task_snapshots, - "observation_diagnostics": observation_diagnostics, }), task_events: json!({ "events": [] }), placed_task_launched: true, @@ -539,24 +515,7 @@ fn new_launch_attempt_id() -> String { fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> Result<()> { static INJECTED_INSTALL_FAILURE: AtomicBool = AtomicBool::new(false); - if std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION") - .ok() - .and_then(|value| value.parse::().ok()) - == Some(state.breakpoint_revision) - { - let delay_ms = std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(750); - std::thread::sleep(Duration::from_millis(delay_ms)); - } - let revision_failure = - std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION") - .ok() - .and_then(|value| value.parse::().ok()) - == Some(state.breakpoint_revision); - if (revision_failure - || std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some()) + if std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some() && !state.breakpoints.is_empty() && !INJECTED_INSTALL_FAILURE.swap(true, Ordering::SeqCst) { @@ -574,7 +533,6 @@ fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> "project": state.project_id, "actor_user": state.actor_user, "process": state.process.as_str(), - "revision": state.breakpoint_revision, "probe_symbols": state.requested_probe_symbols(), }), ), @@ -742,16 +700,14 @@ fn fetch_current_process_status( client_user_request( state, json!({ - "type": "list_process_summaries", + "type": "list_processes", "tenant": state.tenant, "project": state.project_id, "actor_user": state.actor_user, - "cursor": null, - "limit": 100, }), ), )?; - let summary = statuses + let current = statuses .get("processes") .and_then(Value::as_array) .and_then(|processes| { @@ -760,9 +716,6 @@ fn fetch_current_process_status( }) }) .cloned(); - let current = merge_active_process_status(state, summary, |request| { - coordinator_request(coordinator, request) - })?; Ok((statuses, current)) } @@ -773,15 +726,13 @@ fn fetch_current_process_status_in( let statuses = session.request(client_user_request( state, json!({ - "type": "list_process_summaries", + "type": "list_processes", "tenant": state.tenant, "project": state.project_id, "actor_user": state.actor_user, - "cursor": null, - "limit": 100, }), ))?; - let summary = statuses + let current = statuses .get("processes") .and_then(Value::as_array) .and_then(|processes| { @@ -790,51 +741,9 @@ fn fetch_current_process_status_in( }) }) .cloned(); - let current = merge_active_process_status(state, summary, |request| session.request(request))?; Ok((statuses, current)) } -fn merge_active_process_status( - state: &AdapterState, - summary: Option, - mut request: F, -) -> Result> -where - F: FnMut(Value) -> Result, -{ - let Some(mut summary) = summary else { - return Ok(None); - }; - if summary.get("lifecycle").and_then(Value::as_str) != Some("active") { - return Ok(Some(summary)); - } - let active_statuses = request(client_user_request( - state, - json!({ - "type": "list_processes", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - }), - ))?; - let active = active_statuses - .get("processes") - .and_then(Value::as_array) - .and_then(|processes| { - processes.iter().find(|process| { - process.get("process").and_then(Value::as_str) == Some(state.process.as_str()) - }) - }); - if let (Some(summary), Some(active)) = - (summary.as_object_mut(), active.and_then(Value::as_object)) - { - for (key, value) in active { - summary.entry(key.clone()).or_insert_with(|| value.clone()); - } - } - Ok(Some(summary)) -} - pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result { let coordinator = crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); @@ -908,7 +817,27 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result snapshots, Err(error) => { if !emit(RuntimeContinuationOutcome::Diagnostic(format!( @@ -1167,45 +1096,23 @@ pub(crate) fn observe_services_runtime( continue; } }; - let process_status_request = - if inject_process_status_failure && !process_status_failure_injected { - process_status_failure_injected = true; - Err(anyhow!("injected process-status transport failure")) - } else { - fetch_current_process_status_in(current_session, state) + let (process_statuses, process_status) = + match fetch_current_process_status_in(current_session, state) { + Ok(status) => status, + Err(error) => { + if !emit(RuntimeContinuationOutcome::Diagnostic(format!( + "runtime process observation failed: {error:#}; reconnecting in {} ms", + reconnect_delay.as_millis() + ))) { + return Ok(()); + } + session = None; + std::thread::sleep(reconnect_delay); + reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); + continue; + } }; - let (process_statuses, process_status) = match process_status_request { - Ok(status) => status, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime process observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; reconnect_delay = Duration::from_millis(100); - if !has_current_runtime_task(&task_snapshots) { - if let Some(mut outcome) = - terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) - { - if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { - record.node_report = json!({ - "terminal_event": record.node_report.get("terminal_event"), - "task_snapshots": task_snapshots, - "process_status": process_status, - "process_statuses": process_statuses, - }); - } - emit(outcome); - return Ok(()); - } - } if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) { let failed_task = failed_task.to_owned(); let failed_event = events @@ -1284,22 +1191,16 @@ pub(crate) fn observe_services_runtime( return Ok(()); } - let breakpoint_request = if inject_fallback_failure && fallback_failure_stage == 0 { - fallback_failure_stage = 1; - Err(anyhow!("injected breakpoint inspection transport failure")) - } else { - current_session.request(client_user_request( - state, - json!({ - "type": "inspect_debug_breakpoints", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - }), - )) - }; - let breakpoint = match breakpoint_request { + let breakpoint = match current_session.request(client_user_request( + state, + json!({ + "type": "inspect_debug_breakpoints", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + "process": state.process.as_str(), + }), + )) { Ok(breakpoint) => breakpoint, Err(inspect_error) => { // Main completion records its terminal event and clears ephemeral @@ -1307,64 +1208,19 @@ pub(crate) fn observe_services_runtime( // the event read above and this inspection request. Re-read the // durable event stream before treating the missing debug state as // an adapter failure. - let fallback_request = if inject_fallback_failure && fallback_failure_stage == 1 { - fallback_failure_stage = 2; - Err(anyhow!("injected fallback event transport failure")) - } else { - current_session.request(client_user_request( - state, - json!({ - "type": "list_task_events", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - }), - )) - }; - let events = match fallback_request { - Ok(events) => events, - Err(fallback_error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime breakpoint and fallback event observation failed: {inspect_error:#}; {fallback_error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if let Some(mut outcome) = - terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) - { - let task_snapshots = match fetch_task_snapshots_in(current_session, state) { - Ok(snapshots) => snapshots, - Err(snapshot_error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime fallback terminal snapshot observation failed: {snapshot_error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if !has_current_runtime_task(&task_snapshots) { - if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { - record.node_report = json!({ - "terminal_event": record.node_report.get("terminal_event"), - "task_snapshots": task_snapshots, - }); - } - emit(outcome); - return Ok(()); - } + let events = current_session.request(client_user_request( + state, + json!({ + "type": "list_task_events", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + "process": state.process.as_str(), + }), + ))?; + if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) { + emit(outcome); + return Ok(()); } if !emit(RuntimeContinuationOutcome::Diagnostic(format!( "runtime breakpoint observation failed: {inspect_error:#}; reconnecting in {} ms", @@ -1380,28 +1236,7 @@ pub(crate) fn observe_services_runtime( }; if let Some(epoch) = breakpoint.get("hit_epoch").and_then(Value::as_u64) { if epoch > previous_debug_epoch { - let frozen_request = - if inject_debug_epoch_wait_failure && !debug_epoch_wait_failure_injected { - debug_epoch_wait_failure_injected = true; - Err(anyhow!("injected Debug Epoch wait transport failure")) - } else { - wait_for_debug_epoch_state_at(&coordinator, state, epoch, true) - }; - let frozen = match frozen_request { - Ok(frozen) => frozen, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime Debug Epoch observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; + let frozen = wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)?; let node = frozen .get("acknowledgements") .and_then(Value::as_array) @@ -1517,126 +1352,68 @@ pub(crate) fn failed_awaiting_action_snapshot(task_snapshots: &Value) -> Option< }) } -fn has_current_runtime_task(task_snapshots: &Value) -> bool { - task_snapshots - .get("snapshots") - .and_then(Value::as_array) - .is_some_and(|snapshots| { - snapshots.iter().any(|snapshot| { - snapshot.get("current").and_then(Value::as_bool) == Some(true) - && matches!( - snapshot.get("state").and_then(Value::as_str), - Some("queued" | "running" | "failed_awaiting_action") - ) - }) - }) -} - -#[cfg(test)] -pub(crate) fn whole_process_status_code( - main_status_code: Option, - task_snapshots: &Value, -) -> Option { - let main_status_code = main_status_code?; - if main_status_code != 0 { - return Some(main_status_code); - } - - let snapshots = task_snapshots.get("snapshots").and_then(Value::as_array)?; - for snapshot in snapshots - .iter() - .filter(|snapshot| snapshot.get("current").and_then(Value::as_bool) == Some(true)) - { - match snapshot.get("state").and_then(Value::as_str) { - Some("completed") => {} - Some("failed" | "cancelled" | "failed_awaiting_action") => { - return Some( - snapshot - .get("status_code") - .and_then(Value::as_i64) - .and_then(|status| i32::try_from(status).ok()) - .filter(|status| *status != 0) - .unwrap_or(1), - ); - } - Some("queued" | "running") => return None, - _ => return Some(1), - } - } - Some(0) -} - fn terminal_runtime_outcome( coordinator: &str, - _state: &AdapterState, + state: &AdapterState, events: &Value, - process_status: Option<&Value>, ) -> Option { - let final_result = process_status? - .get("final_result") - .and_then(Value::as_str)?; - let status_code = match final_result { - "completed" => 0, - "failed" | "cancelled" => 1, - _ => return None, - }; let event = events .get("events") - .and_then(Value::as_array) - .and_then(|events| { - events.iter().rev().find(|event| { - event.get("executor").and_then(Value::as_str) == Some("coordinator_main") - }) - }); + .and_then(Value::as_array)? + .get(state.runtime_event_count..)? + .iter() + .rev() + .find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?; + let status_code = event + .get("status_code") + .and_then(Value::as_i64) + .map(|status| status as i32) + .or_else( + || match event.get("terminal_state").and_then(Value::as_str) { + Some("completed") => Some(0), + Some("failed" | "cancelled") => Some(1), + _ => None, + }, + ); Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord { coordinator: coordinator.to_owned(), node: event - .and_then(|event| event.get("node")) + .get("node") .and_then(Value::as_str) - .or_else(|| { - process_status - .and_then(|status| status.get("connected_nodes")) - .and_then(Value::as_array) - .and_then(|nodes| nodes.first()) - .and_then(Value::as_str) - }) .unwrap_or("coordinator-main") .to_owned(), - node_report: json!({ - "terminal_event": event, - "process_status": process_status, - }), + node_report: json!({ "terminal_event": event }), task_events: events.clone(), placed_task_launched: true, - status_code: Some(status_code), + status_code, stdout_bytes: event - .and_then(|event| event.get("stdout_bytes")) + .get("stdout_bytes") .and_then(Value::as_u64) .unwrap_or(0), stderr_bytes: event - .and_then(|event| event.get("stderr_bytes")) + .get("stderr_bytes") .and_then(Value::as_u64) .unwrap_or(0), stdout_tail: event - .and_then(|event| event.get("stdout_tail")) + .get("stdout_tail") .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stderr_tail: event - .and_then(|event| event.get("stderr_tail")) + .get("stderr_tail") .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stdout_truncated: event - .and_then(|event| event.get("stdout_truncated")) + .get("stdout_truncated") .and_then(Value::as_bool) .unwrap_or(false), stderr_truncated: event - .and_then(|event| event.get("stderr_truncated")) + .get("stderr_truncated") .and_then(Value::as_bool) .unwrap_or(false), artifact_path: event - .and_then(|event| event.get("artifact_path")) + .get("artifact_path") .and_then(Value::as_str) .map(str::to_owned), event_count: events @@ -1694,24 +1471,6 @@ mod transactional_launch_tests { use super::*; - #[test] - fn current_child_task_keeps_runtime_observation_alive_after_main_completion() { - assert!(has_current_runtime_task(&json!({ - "snapshots": [{ - "current": true, - "state": "running", - "task": "child" - }] - }))); - assert!(!has_current_runtime_task(&json!({ - "snapshots": [{ - "current": false, - "state": "completed", - "task": "main" - }] - }))); - } - #[test] fn inline_bundle_limit_is_checked_before_process_creation() { assert_eq!(MAX_CONTROL_FRAME_BYTES, 1024 * 1024); @@ -1722,53 +1481,6 @@ mod transactional_launch_tests { assert!(error.contains("no virtual process was created")); } - #[test] - fn durable_process_summary_is_terminal_authority_after_event_rotation() { - let state = AdapterState { - runtime_event_count: 10_000, - ..AdapterState::default() - }; - let completed = terminal_runtime_outcome( - "127.0.0.1:1", - &state, - &json!({ "events": [] }), - Some(&json!({ - "process": state.process.as_str(), - "lifecycle": "recent_terminal", - "final_result": "completed", - "connected_nodes": [] - })), - ) - .expect("the durable summary should terminate observation"); - let RuntimeContinuationOutcome::Terminal(completed) = completed else { - panic!("expected terminal outcome"); - }; - assert_eq!(completed.status_code, Some(0)); - - let failed = terminal_runtime_outcome( - "127.0.0.1:1", - &state, - &json!({ - "events": [{ - "executor": "coordinator_main", - "terminal_state": "completed", - "status_code": 0 - }] - }), - Some(&json!({ - "process": state.process.as_str(), - "lifecycle": "recent_terminal", - "final_result": "failed", - "connected_nodes": [] - })), - ) - .expect("the aggregate summary should override a successful main event"); - let RuntimeContinuationOutcome::Terminal(failed) = failed else { - panic!("expected terminal outcome"); - }; - assert_eq!(failed.status_code, Some(1)); - } - #[test] fn failed_debug_launch_reconnects_and_aborts_the_process() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs index add7be2..38dbc2f 100644 --- a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs +++ b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs @@ -173,9 +173,7 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result) -> Artifact { #[test] fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { - let project = std::env::temp_dir().join(format!( - "clusterflux-dap-wasm-locals-{}", - std::process::id() - )); - let src = project.join("src"); - fs::create_dir_all(&src).unwrap(); - fs::write( - src.join("lib.rs"), - "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n input + 1\n}\n", - ) - .unwrap(); - let mut state = AdapterState::default(); - state.project = project.to_string_lossy().into_owned(); + state.project = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures/runtime-conformance") + .to_string_lossy() + .into_owned(); state.source_path = "src/lib.rs".to_owned(); let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap(); state.threads.get_mut(&MAIN_THREAD).unwrap().line = source @@ -935,8 +888,6 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { assert!(!locals .iter() .any(|variable| variable["name"] == "wasm-local-diagnostic")); - - let _ = fs::remove_dir_all(project); } #[test] @@ -964,98 +915,6 @@ fn detects_current_failed_attempt_awaiting_operator_action() { ); } -#[test] -fn whole_process_terminal_status_covers_every_current_logical_task_attempt() { - let snapshots = |items: serde_json::Value| json!({ "snapshots": items }); - - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(0), - "a successful main and final child must succeed" - ); - for (state, status_code) in [("failed", 23), ("cancelled", 1)] { - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": state, - "status_code": status_code - } - ])) - ), - Some(status_code), - "a terminal child must determine the whole-process result" - ); - } - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "accepted-failure", - "current": true, - "state": "failed", - "command_state": "failure_accepted", - "status_code": 17 - } - ])) - ), - Some(17), - "accepting an AwaitOperator failure releases the process but does not turn failure into success" - ); - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "restarted", - "attempt_id": "attempt-1", - "current": false, - "state": "failed", - "status_code": 7 - }, - { - "task": "restarted", - "attempt_id": "attempt-2", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(0), - "a successful replacement attempt is authoritative over stale failed attempts" - ); - assert_eq!( - whole_process_status_code( - Some(9), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(9), - "a failed coordinator main cannot be masked by successful children" - ); -} - #[test] fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() { let mut state = AdapterState::default(); diff --git a/crates/clusterflux-dap/src/virtual_model.rs b/crates/clusterflux-dap/src/virtual_model.rs index 9ef76eb..0f72084 100644 --- a/crates/clusterflux-dap/src/virtual_model.rs +++ b/crates/clusterflux-dap/src/virtual_model.rs @@ -75,7 +75,6 @@ pub(crate) struct AdapterState { pub(crate) debug_probes: Vec, pub(crate) breakpoints: Vec, pub(crate) breakpoints_installed: bool, - pub(crate) breakpoint_revision: u64, pub(crate) threads: BTreeMap, } @@ -114,7 +113,7 @@ impl Default for AdapterState { project, source_path: "src/lib.rs".to_owned(), runtime_backend: RuntimeBackend::Simulated, - coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(), + coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(), tenant: TenantId::from("tenant"), project_id: ProjectId::from("project"), actor_user: UserId::from("dap"), @@ -134,7 +133,6 @@ impl Default for AdapterState { debug_probes: Vec::new(), breakpoints: Vec::new(), breakpoints_installed: false, - breakpoint_revision: 0, } } } @@ -200,24 +198,18 @@ impl AdapterState { } self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(&session.coordinator); - self.tenant = TenantId::try_new(session.tenant) - .map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?; - self.project_id = ProjectId::try_new(session.project) - .map_err(|error| anyhow!("invalid project in CLI session: {error}"))?; - self.actor_user = UserId::try_new(session.user) - .map_err(|error| anyhow!("invalid user in CLI session: {error}"))?; + self.tenant = TenantId::new(session.tenant); + self.project_id = ProjectId::new(session.project); + self.actor_user = UserId::new(session.user); self.client_session_secret = Some(session.session_secret); } else { self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint( coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"), ); if let Some(scope) = project_scope { - self.tenant = TenantId::try_new(scope.tenant) - .map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?; - self.project_id = ProjectId::try_new(scope.project) - .map_err(|error| anyhow!("invalid project in project scope: {error}"))?; - self.actor_user = UserId::try_new(scope.user) - .map_err(|error| anyhow!("invalid user in project scope: {error}"))?; + self.tenant = TenantId::new(scope.tenant); + self.project_id = ProjectId::new(scope.project); + self.actor_user = UserId::new(scope.user); } self.client_session_secret = None; } @@ -238,7 +230,6 @@ impl AdapterState { self.epoch = 0; self.breakpoints.clear(); self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated; - self.breakpoint_revision = 0; self.threads = if self.runtime_backend == RuntimeBackend::Simulated { launch_threads(&self.entry) } else { @@ -313,10 +304,7 @@ impl AdapterState { self.debug_all_threads_stopped = record.all_participants_frozen; self.epoch = record.debug_epoch.unwrap_or(self.epoch); self.coordinator_debug_epoch = record.debug_epoch; - self.stopped_task = record - .stopped_task - .as_deref() - .and_then(|task| TaskInstanceId::try_new(task).ok()); + self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new); self.stopped_probe_symbol = record.stopped_probe_symbol.clone(); if let Some(acknowledgements) = record .node_report @@ -333,12 +321,6 @@ impl AdapterState { else { continue; }; - let Ok(task_id) = TaskInstanceId::try_new(task) else { - continue; - }; - let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else { - continue; - }; let coordinator_main = acknowledgement.get("node").and_then(Value::as_str) == Some("coordinator-main"); let thread_id = if coordinator_main { @@ -348,19 +330,14 @@ impl AdapterState { .values() .find(|thread| thread.task.as_str() == task) .map(|thread| thread.id) - .unwrap_or_else(|| { - self.insert_runtime_thread( - task_id.clone(), - task_definition_id.clone(), - ) - }) + .unwrap_or_else(|| self.insert_runtime_thread(task, task_definition)) }; let thread = self .threads .get_mut(&thread_id) .expect("runtime thread was found or inserted"); - thread.task = task_id; - thread.task_definition = task_definition_id; + thread.task = TaskInstanceId::new(task); + thread.task_definition = TaskDefinitionId::new(task_definition); thread.runtime_stack_frames = acknowledgement .get("stack_frames") .and_then(Value::as_array) @@ -442,22 +419,18 @@ impl AdapterState { let _ = crate::view_state::write_view_state(self, &record); } - fn insert_runtime_thread( - &mut self, - task: TaskInstanceId, - task_definition: TaskDefinitionId, - ) -> i64 { + fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 { let id = self.allocate_runtime_thread_id(); let line = self .debug_probes .iter() .find(|probe| { - probe.task == task_definition || probe.function == task_definition.as_str() + probe.task.as_str() == task_definition || probe.function == task_definition }) .map(|probe| i64::from(probe.line_start)) .unwrap_or(1); - let definition_name = task_definition.as_str().replace(['_', '-'], " "); - let name = if task.as_str() == task_definition.as_str() { + let definition_name = task_definition.replace(['_', '-'], " "); + let name = if task == task_definition { definition_name } else { format!("{definition_name} ({task})") @@ -475,9 +448,9 @@ impl AdapterState { target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task, + task: TaskInstanceId::new(task), attempt_id: "unknown-attempt".to_owned(), - task_definition, + task_definition: TaskDefinitionId::new(task_definition), name, line, state: DebugRuntimeState::Running, @@ -743,12 +716,7 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId { ProcessId::new(format!("vp-{suffix}")) } -fn coordinator_main_thread( - entry: &str, - task: TaskInstanceId, - task_definition: TaskDefinitionId, -) -> VirtualThread { - let name = format!("{entry} coordinator main ({task})"); +fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread { VirtualThread { id: MAIN_THREAD, frame_id: 1_001, @@ -760,10 +728,10 @@ fn coordinator_main_thread( target_ref: 4_501, vfs_ref: 5_001, command_ref: 5_501, - task, + task: TaskInstanceId::from(task), attempt_id: "main:unknown".to_owned(), - task_definition, - name, + task_definition: TaskDefinitionId::from(task_definition), + name: format!("{entry} coordinator main ({task})"), line: 0, state: DebugRuntimeState::Running, freeze_supported: true, @@ -799,14 +767,10 @@ fn coordinator_threads_from_events( .expect("launch thread model should include main"); if let Some(status) = process_status { if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) { - if let Ok(task) = TaskInstanceId::try_new(task) { - main.task = task; - } + main.task = TaskInstanceId::from(task); } if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) { - if let Ok(definition) = TaskDefinitionId::try_new(definition) { - main.task_definition = definition; - } + main.task_definition = TaskDefinitionId::from(definition); } main.name = format!("{entry} coordinator main ({})", main.task); main.state = if status @@ -868,12 +832,6 @@ fn coordinator_threads_from_snapshots( .get("main_task_definition") .and_then(Value::as_str) .unwrap_or(entry); - let (Ok(task), Ok(task_definition)) = ( - TaskInstanceId::try_new(task), - TaskDefinitionId::try_new(task_definition), - ) else { - return threads; - }; let mut main = coordinator_main_thread(entry, task, task_definition); main.attempt_id = format!( "main:{}", @@ -928,12 +886,6 @@ fn coordinator_threads_from_snapshots( let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else { continue; }; - let Ok(task_id) = TaskInstanceId::try_new(task) else { - continue; - }; - let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else { - continue; - }; let attempt_id = snapshot .get("attempt_id") .and_then(Value::as_str) @@ -984,9 +936,9 @@ fn coordinator_threads_from_snapshots( target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: task_id, + task: TaskInstanceId::from(task), attempt_id, - task_definition: task_definition_id, + task_definition: TaskDefinitionId::from(task_definition), name: format!("{display_name} ({task}, attempt {short_attempt})"), line: snapshot .get("source_line") @@ -1033,8 +985,6 @@ fn coordinator_threads_from_snapshots( fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option { let task = event.get("task").and_then(Value::as_str)?; let task_definition = event.get("task_definition").and_then(Value::as_str)?; - let task_id = TaskInstanceId::try_new(task).ok()?; - let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?; let definition_name = task_definition.replace(['_', '-'], " "); let name = if task == task_definition { definition_name @@ -1080,13 +1030,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: task_id, + task: TaskInstanceId::from(task), attempt_id: event .get("attempt_id") .and_then(Value::as_str) .unwrap_or("unknown-attempt") .to_owned(), - task_definition: task_definition_id, + task_definition: TaskDefinitionId::from(task_definition), name, line: 0, state, diff --git a/crates/clusterflux-node/src/assignment_runner.rs b/crates/clusterflux-node/src/assignment_runner.rs index 54f5a00..0e6ce6d 100644 --- a/crates/clusterflux-node/src/assignment_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Read; use std::path::PathBuf; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -46,40 +46,6 @@ use validation::{ resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot, }; -#[derive(Debug)] -struct AssignmentExecutionError { - message: String, - stdout_source_bytes: u64, - stderr_source_bytes: u64, -} - -impl std::fmt::Display for AssignmentExecutionError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(&self.message) - } -} - -impl std::error::Error for AssignmentExecutionError {} - -fn execution_error_with_log_bytes( - message: impl Into, - stdout_source_bytes: &AtomicU64, - stderr_source_bytes: &AtomicU64, -) -> Box { - Box::new(AssignmentExecutionError { - message: message.into(), - stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed), - stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed), - }) -} - -pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) { - error - .downcast_ref::() - .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes)) - .unwrap_or((0, 0)) -} - pub(crate) fn run_verified_wasmtime_assignment( args: &Args, task: &RuntimeTask, @@ -129,8 +95,6 @@ pub(crate) fn run_verified_wasmtime_assignment( return Err("Wasm entrypoint assignment omitted its descriptor export".into()); } }; - let command_stdout_source_bytes = Arc::new(AtomicU64::new(0)); - let command_stderr_source_bytes = Arc::new(AtomicU64::new(0)); let (stdout, boundary_result) = match abi { WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => { let invocation = WasmTaskInvocation::new( @@ -138,28 +102,18 @@ pub(crate) fn run_verified_wasmtime_assignment( task_spec.task_instance.clone(), task_spec.args.clone(), ); - let result = WasmtimeTaskRuntime::new()? - .run_task_export_verified_with_task_host( + let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host( + &module, + expected_bundle_digest, + export, + &invocation, + Box::new(CoordinatorWasmTaskHost::new( + args, + task, + node_private_key, &module, - expected_bundle_digest, - export, - &invocation, - Box::new(CoordinatorWasmTaskHost::new( - args, - task, - node_private_key, - &module, - Arc::clone(&command_stdout_source_bytes), - Arc::clone(&command_stderr_source_bytes), - )?), - ) - .map_err(|error| { - execution_error_with_log_bytes( - error.to_string(), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })?; + )?), + )?; if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() { eprintln!( "clusterflux debug control: Wasm assignment returned for task {}", @@ -168,35 +122,17 @@ pub(crate) fn run_verified_wasmtime_assignment( } match result.outcome { WasmTaskOutcome::Completed => { - let boundary = result.result.ok_or_else(|| { - execution_error_with_log_bytes( - "completed Wasm task omitted result", - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })?; + let boundary = result.result.ok_or("completed Wasm task omitted result")?; ( - format!( - "{}\n", - serde_json::to_string(&boundary).map_err(|error| { - execution_error_with_log_bytes( - error.to_string(), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })? - ), + format!("{}\n", serde_json::to_string(&boundary)?), Some(boundary), ) } WasmTaskOutcome::Failed => { - return Err(execution_error_with_log_bytes( - result - .error - .unwrap_or_else(|| "Wasm task failed without an error".to_owned()), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - )) + return Err(result + .error + .unwrap_or_else(|| "Wasm task failed without an error".to_owned()) + .into()) } } } @@ -204,18 +140,12 @@ pub(crate) fn run_verified_wasmtime_assignment( let task_id = TaskInstanceId::new(task.task.clone()); let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone())); let manifest = artifacts.flush(); - let stdout_source_bytes = command_stdout_source_bytes - .load(Ordering::Relaxed) - .saturating_add(stdout.len() as u64); - let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed); Ok(( CommandOutput { virtual_thread: task_id, status_code: Some(0), stdout, stderr: String::new(), - stdout_source_bytes, - stderr_source_bytes, stdout_truncated: false, stderr_truncated: false, log_backpressured: false, @@ -246,8 +176,6 @@ struct CoordinatorWasmTaskHost { next_handle_id: u64, handles: Arc>>, command_status: Arc>>, - command_stdout_source_bytes: Arc, - command_stderr_source_bytes: Arc, cancellation_requested: Arc, abort_requested: Arc, debug_control: Arc, @@ -260,8 +188,6 @@ impl CoordinatorWasmTaskHost { parent: &RuntimeTask, node_private_key: &str, module: &[u8], - command_stdout_source_bytes: Arc, - command_stderr_source_bytes: Arc, ) -> Result> { let task_spec = parent .task_spec @@ -342,8 +268,6 @@ impl CoordinatorWasmTaskHost { next_handle_id: 1, handles, command_status, - command_stdout_source_bytes, - command_stderr_source_bytes, cancellation_requested, abort_requested, debug_control, @@ -369,16 +293,7 @@ impl CoordinatorWasmTaskHost { runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?; let execution = run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key); - let ( - terminal_state, - status_code, - stdout, - stderr, - stdout_source_bytes, - stderr_source_bytes, - result, - retained, - ) = match execution { + let (terminal_state, status_code, stdout, stderr, result, retained) = match execution { Ok((output, _manifest, result)) => { let retained = retained_result_artifact( self.args.project_root.as_deref(), @@ -391,40 +306,20 @@ impl CoordinatorWasmTaskHost { output.status_code, output.stdout, output.stderr, - output.stdout_source_bytes, - output.stderr_source_bytes, result, retained, ), - Err(error) => ( - "failed", - Some(1), - String::new(), - error.clone(), - output.stdout_source_bytes, - output - .stderr_source_bytes - .saturating_add(error.len() as u64), - None, - None, - ), + Err(error) => ("failed", Some(1), String::new(), error, None, None), } } - Err(error) => { - let (stdout_source_bytes, stderr_source_bytes) = - assignment_error_log_bytes(error.as_ref()); - let error = error.to_string(); - ( - "failed", - Some(1), - String::new(), - error.clone(), - stdout_source_bytes, - stderr_source_bytes.saturating_add(error.len() as u64), - None, - None, - ) - } + Err(error) => ( + "failed", + Some(1), + String::new(), + error.to_string(), + None, + None, + ), }; let artifact_path = retained .as_ref() @@ -444,8 +339,8 @@ impl CoordinatorWasmTaskHost { "task": runtime_task.task, "terminal_state": terminal_state, "status_code": status_code, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": stdout.len(), + "stderr_bytes": stderr.len(), "stdout_tail": stdout, "stderr_tail": stderr, "stdout_truncated": false, @@ -781,7 +676,6 @@ impl WasmTaskHost for CoordinatorWasmTaskHost { let mut runner = CoordinatorControlledProcessRunner::new( self, Duration::from_millis(request.timeout_ms), - configured_secrets.clone(), ); let output = LinuxRootlessPodmanBackend .execute_local_checkout_task( diff --git a/crates/clusterflux-node/src/assignment_runner/process_runner.rs b/crates/clusterflux-node/src/assignment_runner/process_runner.rs index 0e2da69..2b6dbd3 100644 --- a/crates/clusterflux-node/src/assignment_runner/process_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner/process_runner.rs @@ -1,83 +1,4 @@ use super::*; -use std::sync::mpsc::{self, Receiver, SyncSender}; - -struct LiveLogChunk { - stream: &'static str, - offset: u64, - source_bytes: u64, - bytes: Vec, - truncated: bool, -} - -fn append_bounded_tail(tail: &mut Vec, bytes: &[u8], maximum: usize) { - if maximum == 0 { - tail.clear(); - return; - } - if bytes.len() >= maximum { - tail.clear(); - tail.extend_from_slice(&bytes[bytes.len() - maximum..]); - return; - } - let overflow = tail - .len() - .saturating_add(bytes.len()) - .saturating_sub(maximum); - if overflow > 0 { - tail.drain(..overflow); - } - tail.extend_from_slice(bytes); -} - -fn redact_safe_live_log_prefix( - pending: &[u8], - configured_secrets: &[String], - final_chunk: bool, -) -> Option<(usize, String)> { - if pending.is_empty() { - return None; - } - let secret_bytes = configured_secrets - .iter() - .filter(|secret| secret.len() >= 4) - .map(String::as_bytes) - .collect::>(); - let maximum_secret_bytes = secret_bytes - .iter() - .map(|secret| secret.len()) - .max() - .unwrap_or(0); - if !final_chunk && pending.len() <= maximum_secret_bytes { - return None; - } - let mut consumed = if final_chunk { - pending.len() - } else { - pending.len() - maximum_secret_bytes - }; - loop { - let previous = consumed; - for secret in &secret_bytes { - for start in 0..=pending.len().saturating_sub(secret.len()) { - let end = start + secret.len(); - if start < consumed && end > consumed && &pending[start..end] == *secret { - consumed = end; - } - } - } - if consumed == previous { - break; - } - } - if consumed == 0 { - return None; - } - let text = redact_configured_values( - String::from_utf8_lossy(&pending[..consumed]).into_owned(), - configured_secrets, - ); - Some((consumed, text)) -} pub(super) struct CoordinatorControlledProcessRunner { pub(super) args: Args, @@ -86,20 +7,13 @@ pub(super) struct CoordinatorControlledProcessRunner { pub(super) node_private_key: String, pub(super) debug_control: Arc, pub(super) command_status: Arc>>, - pub(super) stdout_source_bytes: Arc, - pub(super) stderr_source_bytes: Arc, pub(super) timeout: Duration, - pub(super) configured_secrets: Vec, } impl CoordinatorControlledProcessRunner { const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1; - pub(super) fn new( - host: &CoordinatorWasmTaskHost, - timeout: Duration, - configured_secrets: Vec, - ) -> Self { + pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self { Self { args: host.args.clone(), process: host.process.clone(), @@ -107,10 +21,7 @@ impl CoordinatorControlledProcessRunner { node_private_key: host.node_private_key.clone(), debug_control: Arc::clone(&host.debug_control), command_status: Arc::clone(&host.command_status), - stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes), - stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes), timeout, - configured_secrets, } } @@ -290,18 +201,10 @@ impl CoordinatorControlledProcessRunner { fn drain_bounded( mut reader: impl Read + Send + 'static, maximum: usize, - stream: &'static str, - sender: SyncSender, - configured_secrets: Vec, - source_bytes_total: Arc, ) -> thread::JoinHandle, String>> { thread::spawn(move || { let mut captured = Vec::new(); let mut buffer = [0_u8; 16 * 1024]; - let stream_base = source_bytes_total.load(Ordering::Relaxed); - let mut source_bytes_read = 0_u64; - let mut pending_offset = stream_base; - let mut pending = Vec::new(); loop { let count = reader .read(&mut buffer) @@ -309,128 +212,12 @@ impl CoordinatorControlledProcessRunner { if count == 0 { break; } - let _ = source_bytes_total.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| Some(current.saturating_add(count as u64)), - ); - source_bytes_read = source_bytes_read.saturating_add(count as u64); - append_bounded_tail(&mut captured, &buffer[..count], maximum); - pending.extend_from_slice(&buffer[..count]); - if let Some((consumed, text)) = - redact_safe_live_log_prefix(&pending, &configured_secrets, false) - { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: pending_offset, - source_bytes: consumed as u64, - bytes: text.into_bytes(), - truncated: false, - }); - pending.drain(..consumed); - pending_offset = pending_offset.saturating_add(consumed as u64); - } - } - if let Some((consumed, text)) = - redact_safe_live_log_prefix(&pending, &configured_secrets, true) - { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: pending_offset, - source_bytes: consumed as u64, - bytes: text.into_bytes(), - truncated: false, - }); - } - if source_bytes_read > maximum as u64 { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: stream_base.saturating_add(source_bytes_read), - source_bytes: 0, - bytes: b"[log output truncated at node capture limit]".to_vec(), - truncated: true, - }); + let remaining = maximum.saturating_sub(captured.len()); + captured.extend_from_slice(&buffer[..count.min(remaining)]); } Ok(captured) }) } - - fn spawn_live_log_reporter(&self, receiver: Receiver) -> thread::JoinHandle<()> { - let args = self.args.clone(); - let process = self.process.clone(); - let task = self.task.clone(); - let node_private_key = self.node_private_key.clone(); - let configured_secrets = self.configured_secrets.clone(); - let command_status = Arc::clone(&self.command_status); - thread::spawn(move || { - let mut log_session = None; - let mut delivery_available = true; - while let Ok(chunk) = receiver.recv() { - if !delivery_available { - continue; - } - let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned(); - text = redact_configured_values(text, &configured_secrets); - let mut delivered = false; - for _ in 0..2 { - if log_session.is_none() { - log_session = CoordinatorSession::connect_with_timeouts( - &args.coordinator, - Duration::from_millis(500), - Duration::from_millis(500), - ) - .ok(); - } - let Some(session) = log_session.as_mut() else { - continue; - }; - let request = signed_node_request_json( - &args, - &node_private_key, - "report_task_log_chunk", - serde_json::json!({ - "type": "report_task_log_chunk", - "tenant": &args.tenant, - "project": &args.project, - "process": &process, - "node": &args.node, - "task": &task, - "stream": chunk.stream, - "offset": chunk.offset, - "source_bytes": chunk.source_bytes, - "text": &text, - "truncated": chunk.truncated, - }), - ); - let result = match request { - Ok(request) => session - .request(request) - .map(|_| ()) - .map_err(|error| error.to_string()), - Err(error) => Err(error.to_string()), - }; - match result { - Ok(()) => { - delivered = true; - break; - } - Err(_) => { - log_session = None; - } - } - } - if !delivered { - delivery_available = false; - if let Ok(mut current) = command_status.lock() { - *current = Some( - "live log delivery was interrupted; final bounded output remains available" - .to_owned(), - ); - } - } - } - }) - } } impl ProcessRunner for CoordinatorControlledProcessRunner { @@ -458,17 +245,12 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { command.program, command.args.join(" ") )); - let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64); let stdout = Self::drain_bounded( child .stdout .take() .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, - "stdout", - live_log_sender.clone(), - self.configured_secrets.clone(), - Arc::clone(&self.stdout_source_bytes), ); let stderr = Self::drain_bounded( child @@ -476,19 +258,11 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .take() .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, - "stderr", - live_log_sender, - self.configured_secrets.clone(), - Arc::clone(&self.stderr_source_bytes), ); - let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver); let mut session = match CoordinatorSession::connect(&self.args.coordinator) { Ok(session) => session, Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); - let _ = stdout.join(); - let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "establish execution control channel: {error}" ))); @@ -503,9 +277,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Ok(None) => {} Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); - let _ = stdout.join(); - let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(error.to_string())); } } @@ -518,7 +289,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "native command exceeded wall-clock timeout of {} ms", self.timeout.as_millis() @@ -533,7 +303,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Cancelled( "coordinator requested cancellation or abort".to_owned(), )); @@ -543,7 +312,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(error); } } @@ -592,9 +360,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .join() .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))? .map_err(BackendError::Command)?; - live_log_reporter - .join() - .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?; self.set_command_status(format!( "native command exited with status {:?}", status.code() @@ -606,52 +371,3 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { }) } } - -#[cfg(test)] -mod tests { - use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner}; - use std::io::Cursor; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{mpsc, Arc}; - - #[test] - fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() { - let secrets = vec!["correct-horse".to_owned()]; - let mut pending = b"prefix correct-".to_vec(); - let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap(); - pending.drain(..consumed); - pending.extend_from_slice(b"horse suffix"); - let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap(); - - let combined = format!("{first}{second}"); - assert_eq!(combined, "prefix [REDACTED] suffix"); - assert!(!combined.contains("correct-")); - assert!(!combined.contains("horse")); - } - - #[test] - fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() { - let source_bytes = Arc::new(AtomicU64::new(5)); - let (sender, receiver) = mpsc::sync_channel(8); - let reader = CoordinatorControlledProcessRunner::drain_bounded( - Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()), - 8, - "stdout", - sender, - Vec::new(), - Arc::clone(&source_bytes), - ); - let captured = reader.join().unwrap().unwrap(); - let chunks = receiver.into_iter().collect::>(); - - assert_eq!(captured, b"opqrstuv"); - assert_eq!(source_bytes.load(Ordering::Relaxed), 37); - assert_eq!( - chunks.iter().map(|chunk| chunk.source_bytes).sum::(), - 32 - ); - assert_eq!(chunks[0].offset, 5); - assert_eq!(chunks.last().unwrap().offset, 37); - assert!(chunks.iter().any(|chunk| chunk.truncated)); - } -} diff --git a/crates/clusterflux-node/src/assignment_runner/tests.rs b/crates/clusterflux-node/src/assignment_runner/tests.rs index d654d1f..4d2d864 100644 --- a/crates/clusterflux-node/src/assignment_runner/tests.rs +++ b/crates/clusterflux-node/src/assignment_runner/tests.rs @@ -48,10 +48,7 @@ fn test_controlled_runner( ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), - stdout_source_bytes: Arc::new(AtomicU64::new(0)), - stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout, - configured_secrets: Vec::new(), } } @@ -92,10 +89,7 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() { ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), - stdout_source_bytes: Arc::new(AtomicU64::new(0)), - stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout: Duration::from_secs(30), - configured_secrets: Vec::new(), }; let started = Instant::now(); let error = runner diff --git a/crates/clusterflux-node/src/command_runner.rs b/crates/clusterflux-node/src/command_runner.rs index 790b838..a1ebd33 100644 --- a/crates/clusterflux-node/src/command_runner.rs +++ b/crates/clusterflux-node/src/command_runner.rs @@ -1,6 +1,6 @@ use clusterflux_core::{ - CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay, - VfsPath, + CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, + VfsOverlay, VfsPath, }; use serde::{Deserialize, Serialize}; @@ -42,8 +42,6 @@ pub struct CommandOutput { pub status_code: Option, pub stdout: String, pub stderr: String, - pub stdout_source_bytes: u64, - pub stderr_source_bytes: u64, pub stdout_truncated: bool, pub stderr_truncated: bool, pub log_backpressured: bool, @@ -85,8 +83,6 @@ impl LocalCommandExecutor { &output.stderr, max_log_bytes, ); - let stdout_source_bytes = output.stdout.len() as u64; - let stderr_source_bytes = output.stderr.len() as u64; let staged_artifact = if let Some(path) = command.stage_stdout_as { Some(overlay.write( path, @@ -102,8 +98,6 @@ impl LocalCommandExecutor { status_code: output.status.code(), stdout: logs.stdout, stderr: logs.stderr, - stdout_source_bytes, - stderr_source_bytes, stdout_truncated: logs.stdout_truncated, stderr_truncated: logs.stderr_truncated, log_backpressured: logs.backpressured, @@ -113,20 +107,24 @@ impl LocalCommandExecutor { } pub(super) fn capture_command_logs( - _task: &TaskInstanceId, + task: &TaskInstanceId, stdout: &[u8], stderr: &[u8], max_log_bytes: usize, ) -> CapturedCommandLogs { - let stdout_truncated = stdout.len() > max_log_bytes; - let stderr_truncated = stderr.len() > max_log_bytes; - let stdout_start = stdout.len().saturating_sub(max_log_bytes); - let stderr_start = stderr.len().saturating_sub(max_log_bytes); + let mut logs = LogBuffer::new(max_log_bytes); + logs.push(task.clone(), stdout); + logs.push(task.clone(), stderr); + let records = logs.records(); + let stdout_record = &records[0]; + let stderr_record = &records[1]; + debug_assert_eq!(&stdout_record.task, task); + debug_assert_eq!(&stderr_record.task, task); CapturedCommandLogs { - stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(), - stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(), - stdout_truncated, - stderr_truncated, - backpressured: stdout_truncated || stderr_truncated, + stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(), + stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(), + stdout_truncated: stdout_record.truncated, + stderr_truncated: stderr_record.truncated, + backpressured: logs.backpressured(), } } diff --git a/crates/clusterflux-node/src/coordinator_session.rs b/crates/clusterflux-node/src/coordinator_session.rs index 13e64ec..997e0f0 100644 --- a/crates/clusterflux-node/src/coordinator_session.rs +++ b/crates/clusterflux-node/src/coordinator_session.rs @@ -3,7 +3,6 @@ use clusterflux_control::endpoint_identity; use clusterflux_control::ControlSession; use clusterflux_core::coordinator_wire_request; use serde_json::Value; -use std::time::Duration; pub(crate) struct CoordinatorSession { inner: ControlSession, @@ -16,16 +15,6 @@ impl CoordinatorSession { }) } - pub(crate) fn connect_with_timeouts( - addr: &str, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result> { - Ok(Self { - inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?, - }) - } - pub(crate) fn request(&mut self, value: Value) -> Result> { let request_id = format!("node-{}", self.inner.requests() + 1); let wire_request = coordinator_wire_request(request_id, value); diff --git a/crates/clusterflux-node/src/daemon.rs b/crates/clusterflux-node/src/daemon.rs index b223ebc..5a6ee0a 100644 --- a/crates/clusterflux-node/src/daemon.rs +++ b/crates/clusterflux-node/src/daemon.rs @@ -7,11 +7,11 @@ use std::time::{Duration, Instant}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_core::{ sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId, - ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS, + TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS, }; use serde_json::{json, Value}; -use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment}; +use crate::assignment_runner::run_verified_wasmtime_assignment; #[cfg(test)] use crate::coordinator_session::control_endpoint_identity; use crate::coordinator_session::CoordinatorSession; @@ -84,8 +84,6 @@ pub(crate) fn run() -> Result<(), Box> { let registration = establish_node_identity(&mut session, &args, &node_private_key)?; let heartbeat_request = json!({ "type": "node_heartbeat", - "tenant": &args.tenant, - "project": &args.project, "node": &args.node, }); let heartbeat_signature = sign_node_request( @@ -224,7 +222,7 @@ fn worker_loop( let expiry = Instant::now() .checked_add(Duration::from_secs(retention_limits.restart_pin_seconds)) .unwrap_or_else(Instant::now); - restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry); + restart_pins.insert(ArtifactId::new(artifact), expiry); } println!("{}", serde_json::to_string(&report)?); std::io::stdout().flush()?; @@ -291,7 +289,7 @@ fn service_pending_artifact_transfer( return Ok(false); }; let transfer_id = required_string(transfer, "transfer_id")?; - let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?; + let artifact = ArtifactId::new(required_string(transfer, "artifact")?); let expected_digest: Digest = serde_json::from_value( transfer .get("expected_digest") @@ -371,11 +369,9 @@ pub(crate) fn runtime_task_from_assignment( .cloned() .ok_or("task assignment missing task_spec")?, )?; - let process = ProcessId::try_new(required_string(value, "process")?)?; - let task = TaskInstanceId::try_new(required_string(value, "task")?)?; Ok(RuntimeTask { - process: process.to_string(), - task: task.to_string(), + process: required_string(value, "process")?, + task: required_string(value, "task")?, epoch: value.get("epoch").and_then(Value::as_u64), bundle_digest: task_spec.bundle_digest.clone(), task_spec: Some(task_spec), @@ -422,8 +418,6 @@ fn run_runtime_task( "reconnect_node", json!({ "type": "reconnect_node", - "tenant": &args.tenant, - "project": &args.project, "node": &args.node, "process": &task.process, "epoch": epoch, @@ -464,8 +458,6 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, - 0, - 0, ); } @@ -500,13 +492,9 @@ fn run_runtime_task( debug_command, node_private_key, &error, - output.stdout_source_bytes, - output.stderr_source_bytes, ), }, Err(error) => { - let (stdout_source_bytes, stderr_source_bytes) = - assignment_error_log_bytes(error.as_ref()); let error = error.to_string(); if error.contains("task execution cancelled:") { record_cancelled_task( @@ -518,8 +506,6 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, - stdout_source_bytes, - stderr_source_bytes, ) } else { record_failed_task( @@ -532,8 +518,6 @@ fn run_runtime_task( debug_command, node_private_key, &error, - stdout_source_bytes, - stderr_source_bytes, ) } } @@ -586,12 +570,6 @@ fn parse_args() -> Result> { } } - TenantId::try_new(tenant.clone())?; - ProjectId::try_new(project.clone())?; - NodeId::try_new(node.clone())?; - if let Some(grant) = enrollment_grant.as_ref() { - clusterflux_core::validate_opaque_token(grant, 512)?; - } Ok(Args { coordinator: coordinator.ok_or("--coordinator is required")?, tenant, @@ -644,12 +622,13 @@ mod tests { #[test] fn hosted_url_remains_an_https_control_endpoint() { assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") + .unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( control_endpoint_identity("127.0.0.1:7999").unwrap(), diff --git a/crates/clusterflux-node/src/lib.rs b/crates/clusterflux-node/src/lib.rs index d9db699..f35c3d6 100644 --- a/crates/clusterflux-node/src/lib.rs +++ b/crates/clusterflux-node/src/lib.rs @@ -741,7 +741,7 @@ mod tests { } #[test] - fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() { + fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() { let invocation = CommandInvocation { program: "cargo".to_owned(), args: vec!["build".to_owned()], @@ -774,7 +774,7 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "cdef"); + assert_eq!(output.stdout, "abcd"); assert!(output.stdout_truncated); assert!(output.log_backpressured); assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6); @@ -992,7 +992,7 @@ mod tests { #[cfg(unix)] #[test] - fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() { + fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() { let executor = LocalCommandExecutor { node: clusterflux_core::NodeId::from("node"), hosted_control_plane: false, @@ -1021,15 +1021,15 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "cdef"); - assert_eq!(output.stderr, "err"); + assert_eq!(output.stdout, "abcd"); + assert_eq!(output.stderr, ""); assert!(output.stdout_truncated); - assert!(!output.stderr_truncated); + assert!(output.stderr_truncated); assert!(output.log_backpressured); } #[test] - fn public_node_crate_does_not_require_hosted_service_types() { + fn public_node_crate_does_not_require_hosted_private_types() { let _tenant = TenantId::from("tenant"); let _project = ProjectId::from("project"); let _backend = LinuxRootlessPodmanBackend; diff --git a/crates/clusterflux-node/src/main.rs b/crates/clusterflux-node/src/main.rs index 2ada582..8f2960b 100644 --- a/crates/clusterflux-node/src/main.rs +++ b/crates/clusterflux-node/src/main.rs @@ -8,34 +8,5 @@ mod task_artifacts; mod task_reports; fn main() -> Result<(), Box> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-node {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux node worker.\n\n\ - Usage: clusterflux-node --coordinator [OPTIONS]\n\n\ - Options:\n \ - --coordinator \n \ - --tenant [default: tenant]\n \ - --project-id [default: project]\n \ - --node [default: node]\n \ - --project-root \n \ - --enrollment-grant \n \ - --public-key \n \ - --worker\n \ - --emit-ready\n \ - --control-poll-ms \n \ - --assignment-poll-ms [default: 500]\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - _ => {} - } daemon::run() } diff --git a/crates/clusterflux-node/src/task_reports.rs b/crates/clusterflux-node/src/task_reports.rs index afaf0c8..a700e76 100644 --- a/crates/clusterflux-node/src/task_reports.rs +++ b/crates/clusterflux-node/src/task_reports.rs @@ -47,8 +47,8 @@ pub(crate) fn record_completed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -85,8 +85,8 @@ pub(crate) fn record_completed_task( "node": &args.node, "task": &task.task, "status_code": output.status_code, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -123,11 +123,8 @@ pub(crate) fn record_failed_task( debug_command: Value, node_private_key: &str, error: &str, - stdout_source_bytes: u64, - command_stderr_source_bytes: u64, ) -> Result> { let error = bounded_runtime_error(error); - let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64); let log_event = session.request(signed_node_request_json( args, node_private_key, @@ -139,8 +136,8 @@ pub(crate) fn record_failed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -178,8 +175,8 @@ pub(crate) fn record_failed_task( "task": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -192,8 +189,6 @@ pub(crate) fn record_failed_task( Ok(failed_node_report( task, &error, - stdout_source_bytes, - stderr_source_bytes, registration, heartbeat, capability_report, @@ -215,8 +210,6 @@ pub(crate) fn record_cancelled_task( capability_report: Value, debug_command: Value, node_private_key: &str, - stdout_source_bytes: u64, - stderr_source_bytes: u64, ) -> Result> { let recorded = session.request(signed_node_request_json( args, @@ -231,12 +224,12 @@ pub(crate) fn record_cancelled_task( "task": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": stdout_source_bytes > 0, - "stderr_truncated": stderr_source_bytes > 0, + "stdout_truncated": false, + "stderr_truncated": false, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, @@ -245,8 +238,6 @@ pub(crate) fn record_cancelled_task( )?)?; Ok(cancelled_node_report( task, - stdout_source_bytes, - stderr_source_bytes, registration, heartbeat, capability_report, @@ -290,8 +281,8 @@ pub(crate) fn completed_node_report( "virtual_thread": output.virtual_thread, "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" }, "status_code": output.status_code, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -314,8 +305,6 @@ pub(crate) fn completed_node_report( #[allow(clippy::too_many_arguments)] pub(crate) fn cancelled_node_report( task: &RuntimeTask, - stdout_source_bytes: u64, - stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -329,12 +318,12 @@ pub(crate) fn cancelled_node_report( "virtual_thread": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": stdout_source_bytes > 0, - "stderr_truncated": stderr_source_bytes > 0, + "stdout_truncated": false, + "stderr_truncated": false, "log_backpressured": false, "staged_artifact": null, "large_bytes_uploaded": false, @@ -354,8 +343,6 @@ pub(crate) fn cancelled_node_report( pub(crate) fn failed_node_report( task: &RuntimeTask, error: &str, - stdout_source_bytes: u64, - stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -370,8 +357,8 @@ pub(crate) fn failed_node_report( "virtual_thread": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": error, "stdout_truncated": false, @@ -390,40 +377,3 @@ pub(crate) fn failed_node_report( "coordinator_response": coordinator_response, }) } - -#[cfg(test)] -mod tests { - use super::*; - use clusterflux_core::TaskInstanceId; - - #[test] - fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() { - let report = completed_node_report( - CommandOutput { - virtual_thread: TaskInstanceId::from("task"), - status_code: Some(0), - stdout: "bounded tail".to_owned(), - stderr: String::new(), - stdout_source_bytes: 519, - stderr_source_bytes: 0, - stdout_truncated: true, - stderr_truncated: false, - log_backpressured: false, - staged_artifact: None, - }, - false, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - 1, - ); - - assert_eq!(report["stdout_bytes"], 519); - assert_eq!(report["stdout_tail"], "bounded tail"); - } -} diff --git a/docs/artifacts.md b/docs/artifacts.md index a1521a7..9320564 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context. The reverse stream counts framing, base64 expansion, failed bytes, and abandoned transfers. The CLI verifies the final digest. Hosted policy may impose per-project, per-tenant, and per-account size, concurrency, and period limits. -There is no hosted global circuit breaker, and one tenant's period usage does -not consume a shared customer byte quota. +Operators may disable relay traffic as an emergency safety control; one tenant's +period usage does not consume a shared customer byte quota. diff --git a/docs/contributing/releases.md b/docs/contributing/releases.md new file mode 100644 index 0000000..fe2b7d8 --- /dev/null +++ b/docs/contributing/releases.md @@ -0,0 +1,60 @@ +# Release candidates + +This is a contributor and release-engineering procedure, not an end-user setup +path. Publication is a three-stage transaction: + +1. `candidate` builds immutable archives and a manifest with paths relative to + the manifest directory. +2. `live-test` downloads that exact candidate in a clean job, deploys it, and + records the full 25-check production-shaped acceptance result plus deployment, + runtime configuration, and proxy configuration identities. +3. `final` downloads the candidate and evidence in another clean job, verifies + every binding, and publishes without rebuilding any binary. + +Set `CLUSTERFLUX_RELEASE_STAGE=candidate` while creating the candidate and +`CLUSTERFLUX_RELEASE_STAGE=final` while finalizing it. The final stage requires +`CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST` and complete live evidence. There is no +incomplete-evidence publication override. + +The `clusterflux-release` runner must provide `CLUSTERFLUX_DEPLOY_COMMAND` as a +protected secret. The command runs locally with +`CLUSTERFLUX_CANDIDATE_ARCHIVE`, `CLUSTERFLUX_CANDIDATE_COORDINATOR`, and their +SHA-256 identities exported. It must deploy that executable and restart the +configured service. `scripts/deploy-release-candidate.sh` then independently +compares the running `/proc//exe` digest with the candidate and records +the service and proxy unit identities; a mismatch stops the release. + +Clusterflux release binaries are built once. The public client/node archive and +the private-source hosted-service archive are both digest-bound to the same +candidate. The hosted archive is deployed, the strict production-shaped batch +uses the public archive against it, and finalization copies both archives +without rebuilding. + +Create the candidate in a dedicated directory: + +~~~bash +CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/release-candidate \ +CLUSTERFLUX_RELEASE_STAGE=candidate \ +./scripts/prepare-public-release.js +~~~ + +Deploy `target/release-candidate/assets/clusterflux-public-binaries-*.tar.gz`. +Set `CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST` to the candidate manifest while +running the strict batch. Set `CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH` to the +JSON record from the private and public acceptance commands. The result records +the source commit, source-tree and +public-tree identities, candidate binary digests, deployment generation, and +configuration identity. + +Finalize into a different directory after the strict result passes: + +~~~bash +CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/public-release \ +CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST=target/release-candidate/public-release-manifest.json \ +CLUSTERFLUX_FINAL_RESULT_PATH=target/acceptance/cli-happy-path-live.json \ +./scripts/prepare-public-release.js +~~~ + +Finalization rejects a changed commit, source tree, public tree, candidate +archive, binary digest set, deployment binding, or strict result. It never runs +the release binary build when a candidate manifest is supplied. diff --git a/docs/getting-started.md b/docs/getting-started.md index be09923..0b4f593 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,9 +11,6 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ -On NixOS or another system with Nix, the equivalent package is available with -`nix profile install .#clusterflux-tools`. - Install rootless Podman on each Linux node that will execute container-backed environments. @@ -55,7 +52,7 @@ same stored identity: ~~~bash clusterflux-node \ - --coordinator https://clusterflux.lesstuff.com \ + --coordinator https://clusterflux.michelpaulissen.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ diff --git a/docs/nodes.md b/docs/nodes.md index 3913859..f6ffd19 100644 --- a/docs/nodes.md +++ b/docs/nodes.md @@ -23,7 +23,7 @@ key pair. ~~~bash clusterflux-node \ - --coordinator https://clusterflux.lesstuff.com \ + --coordinator https://clusterflux.michelpaulissen.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ diff --git a/flake.nix b/flake.nix index d9ccf26..f99377a 100644 --- a/flake.nix +++ b/flake.nix @@ -9,18 +9,6 @@ forAllSystems = nixpkgs.lib.genAttrs systems; in { - packages = forAllSystems (system: - let - pkgs = import nixpkgs { inherit system; }; - publicPackages = import ./packages.nix { inherit pkgs self; }; - privatePackages = - if builtins.pathExists ./web/packages.nix then - import ./web/packages.nix { inherit pkgs self; } - else - { }; - in - publicPackages // privatePackages); - devShells = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; diff --git a/packages.nix b/packages.nix deleted file mode 100644 index 1305b67..0000000 --- a/packages.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ pkgs, self }: -let - clusterflux-tools = pkgs.rustPlatform.buildRustPackage { - pname = "clusterflux-tools"; - version = "0.1.0"; - src = self; - cargoLock.lockFile = ./Cargo.lock; - nativeBuildInputs = [ - pkgs.git - pkgs.lld - pkgs.makeWrapper - ]; - cargoBuildFlags = [ - "--package" - "clusterflux-cli" - "--package" - "clusterflux-node" - "--package" - "clusterflux-coordinator" - "--package" - "clusterflux-dap" - ]; - cargoTestFlags = [ - "--package" - "clusterflux-cli" - "--package" - "clusterflux-node" - "--package" - "clusterflux-coordinator" - "--package" - "clusterflux-dap" - ]; - NIX_BUILD_CORES = "2"; - RUST_MIN_STACK = "1073741824"; - postInstall = '' - test -x "$out/bin/clusterflux" - test -x "$out/bin/clusterflux-node" - test -x "$out/bin/clusterflux-coordinator" - test -x "$out/bin/clusterflux-debug-dap" - for command in \ - clusterflux \ - clusterflux-node \ - clusterflux-coordinator \ - clusterflux-debug-dap - do - ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --version >/dev/null - ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --help >/dev/null - done - ''; - postFixup = - let - runtimePath = pkgs.lib.makeBinPath [ - pkgs.cargo - pkgs.git - pkgs.lld - pkgs.rustc - ]; - in - '' - wrapProgram "$out/bin/clusterflux" --prefix PATH : ${runtimePath} - wrapProgram "$out/bin/clusterflux-node" --prefix PATH : ${runtimePath} - wrapProgram "$out/bin/clusterflux-debug-dap" --prefix PATH : ${runtimePath} - ''; - meta = { - description = "Clusterflux CLI, node, coordinator, and debugger adapter"; - mainProgram = "clusterflux"; - }; - }; -in -{ - inherit clusterflux-tools; - clusterflux = clusterflux-tools; - default = clusterflux-tools; -} diff --git a/scripts/acceptance-private.sh b/scripts/acceptance-private.sh new file mode 100755 index 0000000..5a54a15 --- /dev/null +++ b/scripts/acceptance-private.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +if [[ ! -d private/hosted-policy ]]; then + exit 0 +fi + +scripts/check-old-name.sh +node scripts/check-docs.js +scripts/check-code-size.sh +cargo fmt --all --manifest-path private/hosted-policy/Cargo.toml --check +cargo clippy --manifest-path private/hosted-policy/Cargo.toml --all-targets -- -D warnings +cargo test --locked --manifest-path private/hosted-policy/Cargo.toml --all-targets +node private/hosted-policy/scripts/prepare-hosted-deployment.js +if command -v podman >/dev/null 2>&1; then + node private/hosted-policy/scripts/postgres-durable-smoke.js +elif command -v nix >/dev/null 2>&1; then + nix shell nixpkgs#podman --command node private/hosted-policy/scripts/postgres-durable-smoke.js +else + node private/hosted-policy/scripts/postgres-durable-smoke.js +fi +if [[ -n "${CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR:-}" ]]; then + node private/hosted-policy/scripts/hosted-service-live-check.js +fi diff --git a/scripts/acceptance-public.sh b/scripts/acceptance-public.sh new file mode 100755 index 0000000..96a1363 --- /dev/null +++ b/scripts/acceptance-public.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +scripts/check-old-name.sh +node scripts/check-docs.js +scripts/check-code-size.sh +scripts/release-source-scan.sh +cargo fmt --all --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace --all-targets +cargo build --workspace --all-targets +cargo build -p runtime-conformance --target wasm32-unknown-unknown +cargo build -p hello-build --target wasm32-unknown-unknown +cargo build -p recovery-build --target wasm32-unknown-unknown +node scripts/resource-metering-contract-smoke.js +node scripts/hostile-input-contract-smoke.js +node scripts/tenant-isolation-contract-smoke.js +node scripts/self-hosted-coordinator-smoke.js +node scripts/public-local-demo-matrix-smoke.js +node scripts/cli-output-mode-smoke.js +node scripts/cli-login-smoke.js +node scripts/cli-error-exit-smoke.js +node scripts/cli-browser-login-flow-smoke.js +node scripts/cli-install-smoke.js +node scripts/user-session-token-boundary-smoke.js +node scripts/sdk-spawn-runtime-smoke.js +node scripts/node-lifecycle-contract-smoke.js +node scripts/wasmtime-node-smoke.js +node scripts/wasmtime-assignment-smoke.js +if command -v podman >/dev/null 2>&1; then + node scripts/podman-backend-smoke.js +elif command -v nix >/dev/null 2>&1; then + nix shell nixpkgs#podman --command node scripts/podman-backend-smoke.js +else + node scripts/podman-backend-smoke.js +fi +node scripts/vscode-extension-smoke.js +node scripts/vscode-f5-smoke.js +node scripts/node-attach-smoke.js +node scripts/cli-local-run-smoke.js +node scripts/artifact-download-smoke.js +node scripts/artifact-export-smoke.js +node scripts/operator-panel-smoke.js +node scripts/source-preparation-smoke.js +node scripts/scheduler-placement-smoke.js +node scripts/windows-best-effort-smoke.js +node scripts/quic-smoke.js +node scripts/dap-smoke.js +node scripts/recovery-build-smoke.js +node scripts/flagship-demo-smoke.js +scripts/verify-public-split.sh diff --git a/scripts/agent-signing.js b/scripts/agent-signing.js new file mode 100644 index 0000000..71c913d --- /dev/null +++ b/scripts/agent-signing.js @@ -0,0 +1,99 @@ +const crypto = require("crypto"); + +const { nodeIdentity, signedRequestPayloadDigest } = require("./node-signing"); + +function agentIdentity(seedPrefix, agent) { + const identity = nodeIdentity(seedPrefix, agent); + return { + ...identity, + publicKeyFingerprint: `sha256:${crypto + .createHash("sha256") + .update(identity.publicKey) + .digest("hex")}`, + }; +} + +function agentWorkflowSignatureMessage({ + tenant, + project, + agent, + requestKind, + process: processId, + task = "", + payloadDigest, + nonce, + issuedAtEpochSeconds, +}) { + const parts = [ + "clusterflux-agent-workflow-signature:v2", + tenant, + project, + agent, + requestKind, + processId, + task, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedAgentWorkflowProof(identity, request, options = {}) { + const nonce = + options.nonce || + `${request.type}-${process.pid}-${Date.now()}-${crypto + .randomBytes(8) + .toString("hex")}`; + const issuedAtEpochSeconds = + options.issuedAtEpochSeconds || Math.floor(Date.now() / 1000); + const processId = + request.type === "launch_task" ? request.task_spec?.process : request.process; + const task = + request.type === "launch_task" ? request.task_spec?.task_instance : request.task || ""; + const signature = crypto.sign( + null, + agentWorkflowSignatureMessage({ + tenant: request.tenant, + project: request.project, + agent: request.actor_agent, + requestKind: request.type, + process: processId, + task, + payloadDigest: signedRequestPayloadDigest(request), + nonce, + issuedAtEpochSeconds, + }), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAtEpochSeconds, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedAgentWorkflowRequest(identity, request, options = {}) { + const unsignedRequest = { + ...request, + agent_public_key_fingerprint: + options.publicKeyFingerprint || identity.publicKeyFingerprint, + }; + return { + ...unsignedRequest, + agent_signature: signedAgentWorkflowProof(identity, unsignedRequest, options), + }; +} + +module.exports = { + agentIdentity, + agentWorkflowSignatureMessage, + signedAgentWorkflowProof, + signedAgentWorkflowRequest, +}; diff --git a/scripts/artifact-download-smoke.js b/scripts/artifact-download-smoke.js new file mode 100755 index 0000000..fbb9813 --- /dev/null +++ b/scripts/artifact-download-smoke.js @@ -0,0 +1,401 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); +const { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + repo, + runFlagshipWorker, + send, + waitForJsonLine, +} = require("./real-flagship-harness"); + +const downloadNode = "node-download"; +const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode); + +const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function downloadRetainedBytes(addr, link, artifact, expectedSize) { + const chunks = []; + let offset = 0; + for (let attempt = 0; attempt < 500; attempt += 1) { + const response = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 256 * 1024, + }); + assert.strictEqual(response.type, "artifact_download_stream"); + if (!response.content_bytes_available) { + assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending"); + await delay(10); + continue; + } + assert.strictEqual(response.content_source, "retaining_node_reverse_stream"); + assert.strictEqual(response.content_offset, offset); + const bytes = Buffer.from(response.content_base64, "base64"); + assert.strictEqual(response.streamed_bytes, bytes.length); + chunks.push(bytes); + offset += bytes.length; + if (response.content_eof) { + const content = Buffer.concat(chunks); + assert.strictEqual(content.length, expectedSize); + return { content, response }; + } + } + throw new Error("timed out waiting for retained artifact reverse stream"); +} + +function downloadNodeCapabilities() { + return flagshipNodeCapabilities(); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + let worker; + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + assert.strictEqual(workerReady.mode, "worker"); + const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr); + assert.strictEqual(compileEvent.status_code, 0); + assert.strictEqual(packageEvent.status_code, 0); + assert.deepStrictEqual(compileEvent.result, { + Artifact: { + id: compileEvent.artifact_path.slice("/vfs/artifacts/".length), + digest: compileEvent.artifact_digest, + size_bytes: compileEvent.artifact_size_bytes, + }, + }); + assert.deepStrictEqual(packageEvent.result, { + Artifact: { + id: packageEvent.artifact_path.slice("/vfs/artifacts/".length), + digest: packageEvent.artifact_digest, + size_bytes: packageEvent.artifact_size_bytes, + }, + }); + assert.ok(compileEvent.artifact_size_bytes > 0); + assert.strictEqual(packageEvent.artifact_size_bytes, compileEvent.artifact_size_bytes); + assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); + assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); + const artifactPath = packageEvent.artifact_path; + assert.match( + artifactPath, + /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ + ); + const artifact = artifactPath.slice("/vfs/artifacts/".length); + + const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [artifact], + direct_connectivity: false, + online: true, + })); + assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded"); + + const disconnectedLink = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(disconnectedLink.type, "artifact_download_link"); + + const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [artifact], + direct_connectivity: true, + online: true, + })); + assert.strictEqual(connectedReport.type, "node_capabilities_recorded"); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(link.type, "artifact_download_link"); + assert.strictEqual(link.link.tenant, "tenant"); + assert.strictEqual(link.link.project, "project"); + assert.strictEqual(link.link.process, virtualProcess); + assert.deepStrictEqual(link.link.actor, { User: "user" }); + assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/); + assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); + assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60); + assert.ok( + link.link.url_path.endsWith( + `/artifacts/tenant/project/${virtualProcess}/${artifact}` + ) + ); + assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" }); + + const crossTenant = await send(addr, { + type: "create_artifact_download_link", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match(crossTenant.message, /tenant mismatch/); + + const crossProject = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "other-project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(crossProject.type, "error"); + assert.match(crossProject.message, /project mismatch/); + + const crossTenantOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossTenantOpen.type, "error"); + assert.match(crossTenantOpen.message, /tenant mismatch/); + + const crossProjectOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "other-project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossProjectOpen.type, "error"); + assert.match(crossProjectOpen.message, /project mismatch/); + + const guessed = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: "sha256:guessed", + chunk_bytes: 1, + }); + assert.strictEqual(guessed.type, "error"); + assert.match(guessed.message, /token is invalid/); + + const crossActorOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "other-user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossActorOpen.type, "error"); + assert.match(crossActorOpen.message, /token is invalid/); + + const downloaded = await downloadRetainedBytes( + addr, + link, + artifact, + packageEvent.artifact_size_bytes, + ); + const downloadedDigest = `sha256:${crypto + .createHash("sha256") + .update(downloaded.content) + .digest("hex")}`; + assert.strictEqual(downloadedDigest, packageEvent.artifact_digest); + const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-")); + try { + const executable = path.join(inspect, "hello-clusterflux"); + fs.writeFileSync(executable, downloaded.content); + fs.chmodSync(executable, 0o755); + assert.strictEqual( + cp.execFileSync(executable, { encoding: "utf8" }), + "hello from a real Clusterflux build\n" + ); + } finally { + fs.rmSync(inspect, { recursive: true, force: true }); + } + const cliDownloadDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "clusterflux-cli-download-") + ); + try { + const cliDownloadPath = path.join(cliDownloadDirectory, "hello-clusterflux"); + const cliDownload = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "artifact", "download", artifact, + "--to", cliDownloadPath, + "--max-bytes", "1048576", + "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, + "--tenant", "tenant", + "--project-id", "project", + "--user", "user", + "--json", + ], + { cwd: repo, env: process.env, encoding: "utf8" } + ) + ); + assert.strictEqual(cliDownload.command, "artifact download"); + assert.strictEqual(cliDownload.local_download.status, "local_bytes_written"); + assert.strictEqual( + cliDownload.local_download.verified_digest, + packageEvent.artifact_digest + ); + assert.strictEqual( + `sha256:${crypto + .createHash("sha256") + .update(fs.readFileSync(cliDownloadPath)) + .digest("hex")}`, + packageEvent.artifact_digest + ); + } finally { + fs.rmSync(cliDownloadDirectory, { recursive: true, force: true }); + } + assert.strictEqual(downloaded.response.content_eof, true); + assert.strictEqual( + downloaded.response.charged_download_bytes, + packageEvent.artifact_size_bytes + ); + assert.strictEqual(downloaded.response.link.artifact, artifact); + + const crossActorRevoke = await send(addr, { + type: "revoke_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "other-user", + artifact, + token_digest: link.link.scoped_token_digest, + }); + assert.strictEqual(crossActorRevoke.type, "error"); + assert.match(crossActorRevoke.message, /token is invalid/); + + const revoked = await send(addr, { + type: "revoke_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + token_digest: link.link.scoped_token_digest, + }); + assert.strictEqual(revoked.type, "artifact_download_link_revoked"); + assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest); + + const revokedOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(revokedOpen.type, "error"); + assert.match(revokedOpen.message, /revoked/); + + const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false, + online: true, + })); + assert.strictEqual(gcReport.type, "node_capabilities_recorded"); + + const collectedLink = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(collectedLink.type, "error"); + assert.match(collectedLink.message, /unavailable from current retention/); + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Artifact download smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/artifact-export-smoke.js b/scripts/artifact-export-smoke.js new file mode 100644 index 0000000..8a26668 --- /dev/null +++ b/scripts/artifact-export-smoke.js @@ -0,0 +1,275 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); +const { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + repo, + runFlagshipWorker, + send, + waitForJsonLine, + waitForNodeStatus, +} = require("./real-flagship-harness"); + +const sourceNode = "node-export-source"; +const sourceIdentity = nodeIdentity("artifact-export-smoke", sourceNode); + +function nodeCapabilities() { + return flagshipNodeCapabilities(); +} + +function runJson(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = cp.spawn(command, args, { cwd: repo, ...options }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code !== 0) { + reject( + new Error( + `${command} ${args.join(" ")} failed with code ${code}\n${stderr}\n${stdout}` + ) + ); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject(new Error(`${command} did not return JSON\n${stdout}\n${error.message}`)); + } + }); + }); +} + +async function reportNode( + addr, + node, + identity, + { directConnectivity = true, online = true, artifacts = [] } = {} +) { + const response = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node, + capabilities: nodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: artifacts, + direct_connectivity: directConnectivity, + online, + })); + assert.strictEqual(response.type, "node_capabilities_recorded"); + assert.strictEqual(response.node, node); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { + cwd: repo, + env: { ...process.env, CLUSTERFLUX_NODE_STALE_AFTER_SECONDS: "1" }, + } + ); + + let worker; + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + worker = await runFlagshipWorker(addr, sourceNode, sourceIdentity); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + assert.strictEqual(workerReady.mode, "worker"); + const firstNodeTaskCompletion = waitForNodeStatus(worker.child, "completed"); + const { compileEvent, process: virtualProcess } = await launchFlagship(addr); + const workerCompletion = await firstNodeTaskCompletion; + assert.strictEqual(workerCompletion.node_status, "completed"); + assert.strictEqual( + workerCompletion.task_assignment_response.task_spec.task_definition, + "snapshot_current_project" + ); + assert.strictEqual( + workerCompletion.virtual_thread, + workerCompletion.task_assignment_response.task_spec.task_instance + ); + assert.strictEqual(compileEvent.status_code, 0); + assert.match( + compileEvent.artifact_path, + /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ + ); + const artifact = compileEvent.artifact_path.slice("/vfs/artifacts/".length); + + await reportNode(addr, sourceNode, sourceIdentity, { + artifacts: [artifact], + }); + + const receiverIdentity = nodeIdentity("artifact-export-smoke", "node-export-receiver"); + const attachedReceiver = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node: "node-export-receiver", + public_key: receiverIdentity.publicKey, + }); + assert.strictEqual(attachedReceiver.type, "node_attached"); + await reportNode(addr, "node-export-receiver", receiverIdentity); + + const exportPlan = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(exportPlan.type, "artifact_export_plan"); + assert.strictEqual(exportPlan.source_node, "node-export-source"); + assert.strictEqual(exportPlan.receiver_node, "node-export-receiver"); + assert.strictEqual(exportPlan.plan.transport, "NativeQuic"); + assert.strictEqual(exportPlan.plan.scope.tenant, "tenant"); + assert.strictEqual(exportPlan.plan.scope.project, "project"); + assert.strictEqual(exportPlan.plan.scope.process, virtualProcess); + assert.deepStrictEqual(exportPlan.plan.scope.object, { Artifact: artifact }); + assert.strictEqual(exportPlan.plan.source.node, "node-export-source"); + assert.strictEqual(exportPlan.plan.destination.node, "node-export-receiver"); + assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true); + assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false); + assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/); + assert.strictEqual(exportPlan.artifact_size_bytes, compileEvent.artifact_size_bytes); + + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-artifact-export-")); + const exportPath = path.join(temp, "hello-clusterflux"); + cp.execFileSync( + "cargo", + ["build", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux"], + { cwd: repo, stdio: "inherit" } + ); + const cliBinary = path.join( + path.resolve(repo, process.env.CARGO_TARGET_DIR || "target"), + "debug", + process.platform === "win32" ? "clusterflux.exe" : "clusterflux" + ); + await reportNode(addr, sourceNode, sourceIdentity, { + artifacts: [artifact], + }); + const cliExport = await runJson(cliBinary, [ + "artifact", + "export", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--user", + "user", + "--json", + artifact, + "--receiver-node", + "node-export-receiver", + "--to", + exportPath, + ]); + assert.strictEqual(cliExport.command, "artifact export"); + assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true); + assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written"); + assert.strictEqual( + cliExport.export_plan.bytes_written, + compileEvent.artifact_size_bytes + ); + assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false); + assert.strictEqual( + cliExport.local_export.verified_digest, + compileEvent.artifact_digest + ); + fs.chmodSync(exportPath, 0o755); + assert.strictEqual( + cp.execFileSync(exportPath, { encoding: "utf8" }), + "hello from a real Clusterflux build\n" + ); + + const crossTenant = await send(addr, { + type: "export_artifact_to_node", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match(crossTenant.message, /tenant mismatch/); + + await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); + const failedDirect = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: false, + failure_reason: "nat traversal failed", + }); + assert.strictEqual(failedDirect.type, "error"); + assert.match(failedDirect.message, /nat traversal failed/); + assert.match(failedDirect.message, /coordinator bulk relay is disabled/); + + await reportNode(addr, "node-export-receiver", receiverIdentity, { online: false }); + await new Promise((resolve) => setTimeout(resolve, 2100)); + await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); + const offlineReceiver = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(offlineReceiver.type, "error"); + assert.match(offlineReceiver.message, /offline/); + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Artifact export smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/check-code-size.sh b/scripts/check-code-size.sh new file mode 100755 index 0000000..96e54d3 --- /dev/null +++ b/scripts/check-code-size.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +maximum_lines=3000 +failed=0 +source_roots=(crates) +if [[ -d private/hosted-policy/src ]]; then + source_roots+=(private/hosted-policy/src) +fi +while IFS= read -r -d '' file; do + case "$file" in + */tests.rs|*/tests/*) continue ;; + esac + lines="$(wc -l < "$file")" + if ((lines > maximum_lines)); then + printf '%s has %s lines; production file limit is %s\n' "$file" "$lines" "$maximum_lines" >&2 + failed=1 + fi +done < <(find "${source_roots[@]}" -type f -name '*.rs' -print0) + +if ((failed)); then + exit 1 +fi +printf 'production source file size guard passed (%s lines maximum)\n' "$maximum_lines" diff --git a/scripts/check-docs.js b/scripts/check-docs.js new file mode 100644 index 0000000..dce6c9e --- /dev/null +++ b/scripts/check-docs.js @@ -0,0 +1,113 @@ +#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); + +const root = path.resolve(__dirname, ".."); +const publicDocs = [ + "README.md", + "SECURITY.md", + "docs/getting-started.md", + "docs/architecture.md", + "docs/nodes.md", + "docs/environments.md", + "docs/artifacts.md", + "docs/debugging.md", + "docs/task-abi.md", + "docs/self-hosting.md", + "docs/security.md", +]; +const contributorDocs = ["docs/contributing/releases.md"]; +const privateDocs = [ + "private/docs/hosted-deployment.md", + "private/docs/authentik.md", + "private/docs/community-policy.md", + "private/docs/bandwidth-and-cost-controls.md", + "private/docs/publishing.md", +]; +const internalDocs = ["internal/finish_mvp_2.md"]; +const filteredPublicTree = + process.env.CLUSTERFLUX_FILTERED_PUBLIC_TREE === "1" || + fs.existsSync(path.join(root, "CLUSTERFLUX_PUBLIC_TREE.json")); + +const failures = []; +const expectExactMarkdownSet = (directory, expected) => { + const actual = fs + .readdirSync(path.join(root, directory), { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => path.posix.join(directory, entry.name)) + .sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + failures.push(`${directory} markdown set is ${actual.join(", ")}; expected ${wanted.join(", ")}`); + } +}; + +const requiredDocs = filteredPublicTree + ? [...publicDocs, ...contributorDocs] + : [...publicDocs, ...contributorDocs, ...privateDocs, ...internalDocs]; +for (const file of requiredDocs) { + if (!fs.existsSync(path.join(root, file))) failures.push(`missing canonical document: ${file}`); +} +expectExactMarkdownSet("docs", publicDocs.filter((file) => file.startsWith("docs/"))); +expectExactMarkdownSet("docs/contributing", contributorDocs); +if (filteredPublicTree) { + for (const directory of ["private", "internal"]) { + if (fs.existsSync(path.join(root, directory))) { + failures.push(`${directory}/ must not exist in the filtered public tree`); + } + } +} + +const forbidden = [ + [/\bmvp\b/i, "internal milestone term"], + [/acceptance criteria/i, "internal gate language"], + [/release verification/i, "internal release language"], + [/public\/private source split/i, "source split narrative"], + [/founder|business decisions/i, "business planning narrative"], + [/hacker news|\bHN\b/, "launch-channel narrative"], + [/\busers can\b/i, "indirect reader wording"], + [/node\s+scripts\//i, "developer script instruction"], + [/scripts\/[^\s)]*smoke/i, "smoke script instruction"], + [/internal\/[^\s)]*/i, "internal tooling reference"], + [/private\/[^\s)]*/i, "private source reference"], +]; +const topLevelCommands = new Set([ + "doctor", "login", "logout", "auth", "agent", "key", "project", "inspect", + "build", "bundle", "run", "node", "process", "task", "logs", "artifact", + "dap", "debug", "quota", "admin", +]); + +for (const file of publicDocs) { + const absolute = path.join(root, file); + const content = fs.readFileSync(absolute, "utf8"); + for (const [pattern, description] of forbidden) { + if (pattern.test(content)) failures.push(`${file}: contains ${description}`); + } + for (const match of content.matchAll(/\]\(([^)#]+)(?:#[^)]+)?\)/g)) { + const target = match[1]; + if (/^(?:https?:|mailto:)/.test(target)) continue; + const resolved = path.resolve(path.dirname(absolute), target); + if (!fs.existsSync(resolved)) failures.push(`${file}: broken link ${target}`); + } + for (const line of content.split(/\r?\n/)) { + const command = line.trim().match(/^clusterflux\s+([a-z][a-z-]*)\b/); + if (command && !topLevelCommands.has(command[1])) { + failures.push(`${file}: unknown top-level CLI command ${command[1]}`); + } + } +} + +const rootMarkdown = fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => entry.name) + .sort(); +if (JSON.stringify(rootMarkdown) !== JSON.stringify(["README.md", "SECURITY.md"])) { + failures.push(`top-level markdown set is ${rootMarkdown.join(", ")}; expected README.md, SECURITY.md`); +} + +if (failures.length) { + for (const failure of failures) console.error(failure); + process.exit(1); +} +console.log("documentation checks passed"); diff --git a/scripts/check-old-name.sh b/scripts/check-old-name.sh new file mode 100755 index 0000000..9a871de --- /dev/null +++ b/scripts/check-old-name.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +legacy_lower="$(printf '%s%s' 'disa' 'smer')" +legacy_title="$(printf '%s%s' 'Disa' 'smer')" +legacy_upper="$(printf '%s%s' 'DISA' 'SMER')" +pattern="${legacy_lower}|${legacy_title}|${legacy_upper}" + +allowed_content='^\./scripts/migrate-clusterflux-state\.sh:' +matches="$( + rg -n --hidden \ + --glob '!**/.git/**' \ + --glob '!**/target/**' \ + --glob '!**/node_modules/**' \ + --glob '!**/vendor/**' \ + --glob '!**/.direnv/**' \ + --glob '!**/.cache/**' \ + --glob '!**/dist/**' \ + --glob '!**/out/**' \ + "$pattern" . 2>/dev/null | rg -v "$allowed_content" || true +)" + +paths="$( + find . \ + \( -type d \( -name .git -o -name target -o -name node_modules -o -name vendor -o -name .direnv -o -name .cache -o -name dist -o -name out \) -prune \) -o \ + -iname "*${legacy_lower}*" -print | sort || true +)" + +if [[ -n "$matches" || -n "$paths" ]]; then + [[ -z "$matches" ]] || printf '%s\n' "$matches" >&2 + [[ -z "$paths" ]] || printf '%s\n' "$paths" >&2 + printf 'unexpected legacy product name remains\n' >&2 + exit 1 +fi + +printf 'old-name guard passed\n' diff --git a/scripts/cli-browser-login-flow-smoke.js b/scripts/cli-browser-login-flow-smoke.js new file mode 100644 index 0000000..0c6341b --- /dev/null +++ b/scripts/cli-browser-login-flow-smoke.js @@ -0,0 +1,222 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const tmp = path.join(repo, "target", "acceptance", "tmp", "cli-browser-login-flow"); +const project = path.join(tmp, "project"); +fs.rmSync(tmp, { recursive: true, force: true }); +fs.mkdirSync(project, { recursive: true }); + +function writeOpener() { + const opener = path.join(tmp, "browser-opener.js"); + const trace = path.join(tmp, "browser-opener.log"); + fs.writeFileSync( + opener, + `#!/usr/bin/env node +const fs = require("fs"); +const trace = ${JSON.stringify(trace)}; +const loginUrl = new URL(process.argv[2]); +const state = loginUrl.searchParams.get("state"); +const nonce = loginUrl.searchParams.get("nonce"); +const challenge = loginUrl.searchParams.get("code_challenge"); +if (loginUrl.protocol !== "https:" || !state || !nonce || !challenge) { + fs.appendFileSync(trace, "missing server-owned OIDC parameters\\n"); + process.exit(1); +} +fs.appendFileSync(trace, "server-owned browser transaction\\n"); +// Model a real browser/opener that remains alive after the CLI transaction. +// Its descriptors must not keep the invoking CLI process open. +setTimeout(() => process.exit(0), 5000); +` + ); + fs.chmodSync(opener, 0o755); + return opener; +} + +function startCoordinator() { + return new Promise((resolve, reject) => { + const requests = []; + const server = net.createServer((socket) => { + let buffered = ""; + socket.on("data", (chunk) => { + buffered += chunk.toString("utf8"); + while (buffered.includes("\n")) { + const newline = buffered.indexOf("\n"); + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const envelope = JSON.parse(line); + requests.push(envelope); + assert.strictEqual(envelope.type, "coordinator_request"); + assert.strictEqual(envelope.protocol_version, 1); + assert.strictEqual(envelope.authentication.kind, "none"); + const request = envelope.payload; + + if (requests.length === 1) { + assert.strictEqual(envelope.request_id, "cli-1"); + assert.strictEqual(envelope.operation, "begin_oidc_browser_login"); + assert.deepStrictEqual(request, { + type: "begin_oidc_browser_login", + }); + socket.write( + `${JSON.stringify({ + type: "oidc_browser_login_started", + transaction_id: "login-transaction", + polling_secret: "opaque-polling-secret", + authorization_url: + "https://auth.michelpaulissen.com/application/o/authorize/?state=server-state&nonce=server-nonce&code_challenge=server-pkce&code_challenge_method=S256&redirect_uri=https%3A%2F%2Fclusterflux.michelpaulissen.com%2Fauth%2Fcallback", + expires_at_epoch_seconds: 1800000000, + })}\n` + ); + continue; + } + + assert.strictEqual(envelope.request_id, "cli-2"); + assert.strictEqual(envelope.operation, "poll_oidc_browser_login"); + assert.deepStrictEqual(request, { + type: "poll_oidc_browser_login", + transaction_id: "login-transaction", + polling_secret: "opaque-polling-secret", + }); + socket.end( + `${JSON.stringify({ + type: "oidc_browser_session", + session: { + tenant: "tenant-smoke", + project: "project-smoke", + user: "user-smoke", + cli_session_credential_kind: "CliDeviceSession", + cli_session_secret: "scoped-cli-session-secret", + expires_at_epoch_seconds: 1800000000, + provider_tokens_sent_to_nodes: false, + }, + })}\n` + ); + server.close(); + } + }); + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resolve({ + url: `${address.address}:${address.port}`, + requests, + close: () => + new Promise((closeResolve) => { + if (!server.listening) closeResolve(); + else server.close(() => closeResolve()); + }), + }); + }); + }); +} + +function runClusterflux(args, env) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "--manifest-path", + path.join(repo, "Cargo.toml"), + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + ...args, + ], + { cwd: project, env, stdio: ["ignore", "pipe", "pipe"] } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk.toString("utf8"))); + child.stderr.on("data", (chunk) => (stderr += chunk.toString("utf8"))); + child.once("error", reject); + child.once("close", (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(`clusterflux exited ${code}\n${stderr}\n${stdout}`)); + }); + }); +} + +(async () => { + const opener = writeOpener(); + const coordinator = await startCoordinator(); + try { + const loginStarted = Date.now(); + const report = JSON.parse( + await runClusterflux( + [ + "login", + "--browser", + "--json", + "--coordinator", + coordinator.url, + "--project-id", + "project-smoke", + ], + { + ...process.env, + CLUSTERFLUX_BROWSER_OPEN_COMMAND: opener, + CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS: "5", + } + ) + ); + assert( + Date.now() - loginStarted < 3000, + "a long-lived browser opener must not keep CLI output pipes or login completion open" + ); + assert.strictEqual(report.plan.coordinator, coordinator.url); + assert.strictEqual(report.boundary.cli_contacted_coordinator, true); + assert.strictEqual(report.boundary.scoped_cli_session_received, true); + assert.strictEqual(report.boundary.local_cli_session_file_written, true); + assert.strictEqual(report.boundary.provider_tokens_persisted_locally, false); + assert.strictEqual(report.boundary.provider_tokens_exposed_to_cli, false); + assert.strictEqual(report.boundary.provider_tokens_sent_to_nodes, false); + assert.strictEqual(report.boundary.coordinator_session_requests, 2); + assert.strictEqual(coordinator.requests.length, 2); + + const sessionFile = path.join(project, ".clusterflux", "session.json"); + const sessionText = fs.readFileSync(sessionFile, "utf8"); + const session = JSON.parse(sessionText); + assert.strictEqual(session.kind, "human"); + assert.strictEqual(session.coordinator, coordinator.url); + assert.strictEqual(session.tenant, "tenant-smoke"); + assert.strictEqual(session.project, "project-smoke"); + assert.strictEqual(session.user, "user-smoke"); + assert.strictEqual(session.cli_session_credential_kind, "CliDeviceSession"); + assert.strictEqual(session.token_expiry_posture, "expires_at"); + assert.strictEqual(session.expires_at, "1800000000"); + assert.strictEqual(session.provider_tokens_exposed_to_cli, false); + assert.strictEqual(session.provider_tokens_sent_to_nodes, false); + assert.doesNotMatch( + sessionText, + /access_token|refresh_token|id_token|provider-secret|authorization_code|Bearer/ + ); + + const authStatus = JSON.parse( + await runClusterflux(["auth", "status", "--json"], process.env) + ); + assert.strictEqual(authStatus.active_coordinator, coordinator.url); + assert.strictEqual(authStatus.principal, "user-smoke"); + assert.strictEqual(authStatus.tenant, "tenant-smoke"); + assert.strictEqual(authStatus.project, "project-smoke"); + assert.strictEqual(authStatus.session.kind, "human"); + assert.strictEqual(authStatus.session.source, "session_file"); + assert.strictEqual(authStatus.session.provider_tokens_exposed_to_cli, false); + assert.strictEqual(authStatus.session.provider_tokens_exposed_to_nodes, false); + } finally { + await coordinator.close(); + } + console.log("CLI browser login flow smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-error-exit-smoke.js b/scripts/cli-error-exit-smoke.js new file mode 100755 index 0000000..68ad258 --- /dev/null +++ b/scripts/cli-error-exit-smoke.js @@ -0,0 +1,365 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const http = require("http"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "tests/fixtures/runtime-conformance"); +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-error-")); + +function runClusterflux(args) { + return new Promise((resolve) => { + const child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + stdio: ["ignore", "pipe", "pipe"], + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("close", (code, signal) => { + resolve({ code, signal, stdout, stderr }); + }); + }); +} + +async function runWithOneCoordinatorResponse(buildArgs, response) { + let request = ""; + const server = http.createServer((incoming, outgoing) => { + incoming.setEncoding("utf8"); + incoming.on("data", (chunk) => { + request += chunk; + }); + incoming.on("end", () => { + outgoing.writeHead(200, { "content-type": "application/json" }); + outgoing.end(JSON.stringify(response)); + server.close(); + }); + }); + + const address = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address())); + }); + const coordinator = `http://${address.address}:${address.port}`; + const result = await runClusterflux(buildArgs(coordinator)); + return { request, result }; +} + +async function main() { + const environmentProject = path.join(tempRoot, "missing-env-project"); + fs.mkdirSync(path.join(environmentProject, "src"), { recursive: true }); + fs.writeFileSync( + path.join(environmentProject, "Cargo.toml"), + "[package]\nname = \"missing-env-project\"\nversion = \"0.1.0\"\nedition = \"2021\"\n" + ); + fs.writeFileSync( + path.join(environmentProject, "src", "main.rs"), + "fn main() { let _target = env!(\"linux\"); }\n" + ); + const environmentFailure = await runClusterflux([ + "build", + "--project", + environmentProject, + "--json", + ]); + assert.strictEqual(environmentFailure.signal, null, environmentFailure.stderr); + assert.strictEqual(environmentFailure.code, 26, environmentFailure.stderr); + const environmentReport = JSON.parse(environmentFailure.stdout); + assert.strictEqual(environmentReport.status, "blocked_before_schedule"); + assert.strictEqual(environmentReport.scheduled_work, false); + assert.strictEqual(environmentReport.machine_error.category, "environment"); + assert.strictEqual( + environmentReport.machine_error.process_exit_code_applied, + true + ); + assert( + environmentReport.machine_error.next_actions.includes("clusterflux inspect") + ); + assert.strictEqual(environmentReport.diagnostics[0].code, "missing_environment"); + + const nonInteractive = await runClusterflux([ + "run", + "build", + "--project", + project, + "--non-interactive", + "--json", + ]); + assert.strictEqual(nonInteractive.signal, null, nonInteractive.stderr); + assert.strictEqual(nonInteractive.code, 20, nonInteractive.stderr); + assert.doesNotMatch(nonInteractive.stderr, /Opening Clusterflux browser login/); + const nonInteractiveReport = JSON.parse(nonInteractive.stdout); + assert.strictEqual(nonInteractiveReport.status, "authentication_required"); + assert.strictEqual(nonInteractiveReport.non_interactive, true); + assert.strictEqual(nonInteractiveReport.browser_opened, false); + assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); + assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); + assert.strictEqual( + nonInteractiveReport.machine_error.process_exit_code_applied, + true + ); + assert( + nonInteractiveReport.machine_error.next_actions.includes( + "pass --local to run against local services" + ) + ); + + let request = ""; + const server = http.createServer((incoming, outgoing) => { + incoming.setEncoding("utf8"); + incoming.on("data", (chunk) => { + request += chunk; + }); + incoming.on("end", () => { + outgoing.writeHead(200, { "content-type": "application/json" }); + outgoing.end( + JSON.stringify({ + type: "error", + message: "quota unavailable: resource limit exceeded for api_calls", + }) + ); + server.close(); + }); + }); + + const address = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address())); + }); + const coordinator = `http://${address.address}:${address.port}`; + + const result = await runClusterflux([ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ]); + + assert.strictEqual(result.signal, null, result.stderr); + assert.strictEqual(result.code, 22, result.stderr); + assert.match(request, /"type":"start_process"/); + const report = JSON.parse(result.stdout); + assert.strictEqual(report.status, "coordinator_rejected"); + assert.strictEqual(report.run_start.machine_error.category, "quota"); + assert.strictEqual(report.run_start.machine_error.resource_category, "api_calls"); + assert.strictEqual(report.run_start.machine_error.community_tier_language, true); + assert.strictEqual( + report.run_start.machine_error.community_tier_label, + "community tier" + ); + assert.doesNotMatch(result.stdout, new RegExp(["free", "tier"].join(" "), "i")); + assert.strictEqual( + report.run_start.machine_error.private_abuse_heuristics_exposed, + false + ); + assert.strictEqual(report.run_start.machine_error.stable_exit_code, 22); + assert.strictEqual( + report.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + report.run_start.machine_error.next_actions.includes("clusterflux quota status") + ); + + const capabilityFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ], + { + type: "error", + message: + "scheduler placement failed: no capable node for placement: missing capability Command", + } + ); + assert.strictEqual(capabilityFailure.result.signal, null, capabilityFailure.result.stderr); + assert.strictEqual(capabilityFailure.result.code, 24, capabilityFailure.result.stderr); + assert.match(capabilityFailure.request, /"type":"start_process"/); + const capabilityReport = JSON.parse(capabilityFailure.result.stdout); + assert.strictEqual( + capabilityReport.run_start.machine_error.category, + "capability" + ); + assert.strictEqual( + capabilityReport.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + capabilityReport.run_start.machine_error.next_actions.includes( + "attach a node with the required capabilities" + ) + ); + + const nodePolicyFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ], + { + type: "error", + message: "node policy denied native command execution", + } + ); + assert.strictEqual(nodePolicyFailure.result.signal, null, nodePolicyFailure.result.stderr); + assert.strictEqual(nodePolicyFailure.result.code, 23, nodePolicyFailure.result.stderr); + assert.match(nodePolicyFailure.request, /"type":"start_process"/); + const nodePolicyReport = JSON.parse(nodePolicyFailure.result.stdout); + assert.strictEqual( + nodePolicyReport.run_start.machine_error.category, + "policy" + ); + assert.strictEqual( + nodePolicyReport.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + nodePolicyReport.run_start.machine_error.next_actions.includes( + "check coordinator policy for this action" + ) + ); + + const programFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "task", + "list", + "--coordinator", + coordinator, + "--json", + ], + { + type: "task_events", + events: [ + { + process: "vp-current", + task: "compile", + terminal_state: "failed", + environment: "linux", + node: "node-linux", + status_code: 1, + stderr_tail: "task exited with status 1", + }, + ], + } + ); + assert.strictEqual(programFailure.result.signal, null, programFailure.result.stderr); + assert.strictEqual(programFailure.result.code, 0, programFailure.result.stderr); + assert.match(programFailure.request, /"type":"list_task_events"/); + const programReport = JSON.parse(programFailure.result.stdout); + assert.strictEqual(programReport.tasks[0].machine_error.category, "program"); + assert.strictEqual(programReport.tasks[0].machine_error.stable_exit_code, 27); + assert( + programReport.tasks[0].machine_error.next_actions.includes("clusterflux logs") + ); + + const artifactDownload = await runWithOneCoordinatorResponse( + (coordinator) => [ + "artifact", + "download", + "app.txt", + "--coordinator", + coordinator, + "--json", + ], + { + type: "artifact_download_denied", + message: "artifact download unauthorized for project", + } + ); + assert.strictEqual(artifactDownload.result.signal, null, artifactDownload.result.stderr); + assert.strictEqual(artifactDownload.result.code, 21, artifactDownload.result.stderr); + assert.match(artifactDownload.request, /"type":"create_artifact_download_link"/); + const artifactDownloadReport = JSON.parse(artifactDownload.result.stdout); + assert.strictEqual( + artifactDownloadReport.download_session.machine_error.category, + "authorization" + ); + assert.strictEqual( + artifactDownloadReport.download_session.machine_error.process_exit_code_applied, + true + ); + + const artifactExport = await runWithOneCoordinatorResponse( + (coordinator) => [ + "artifact", + "export", + "app.txt", + "--to", + path.join(project, "target", "blocked-artifact.txt"), + "--coordinator", + coordinator, + "--json", + ], + { + type: "artifact_export_unavailable", + message: "direct connectivity unavailable for artifact export", + } + ); + assert.strictEqual(artifactExport.result.signal, null, artifactExport.result.stderr); + assert.strictEqual(artifactExport.result.code, 25, artifactExport.result.stderr); + assert.match(artifactExport.request, /"type":"export_artifact_to_node"/); + const artifactExportReport = JSON.parse(artifactExport.result.stdout); + assert.strictEqual( + artifactExportReport.export_plan.machine_error.category, + "connectivity" + ); + assert.strictEqual( + artifactExportReport.export_plan.machine_error.process_exit_code_applied, + true + ); + + const confirmation = await runClusterflux([ + "process", + "cancel", + "--coordinator", + "127.0.0.1:9", + "--json", + ]); + assert.strictEqual(confirmation.signal, null, confirmation.stderr); + assert.strictEqual(confirmation.code, 23, confirmation.stderr); + const confirmationReport = JSON.parse(confirmation.stdout); + assert.strictEqual(confirmationReport.status, "confirmation_required"); + assert.strictEqual(confirmationReport.coordinator_request_sent, false); + assert.strictEqual(confirmationReport.machine_error.category, "policy"); + assert.strictEqual( + confirmationReport.machine_error.process_exit_code_applied, + true + ); + assert( + confirmationReport.next_actions.some((action) => action.includes("--yes")) + ); +} + +main() + .then(() => { + console.log("CLI error exit smoke passed"); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/scripts/cli-happy-path-live-smoke.js b/scripts/cli-happy-path-live-smoke.js new file mode 100644 index 0000000..f2ee2d4 --- /dev/null +++ b/scripts/cli-happy-path-live-smoke.js @@ -0,0 +1,4506 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const https = require("https"); +const os = require("os"); +const path = require("path"); +const { DapClient } = require("./dap-client"); +const { agentIdentity, signedAgentWorkflowRequest } = require("./agent-signing"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { + nodeIdentity, + nodeIdentityFromPrivateKey, + signedNodeHeartbeat, + signedNodeRequest, +} = require("./node-signing"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +configurePodmanTestEnvironment(repo); +if ( + !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && + cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && + cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 +) { + cp.execFileSync( + "nix", + ["shell", "nixpkgs#podman", "--command", "node", __filename], + { + cwd: repo, + env: { ...process.env, CLUSTERFLUX_PODMAN_NIX_SHELL: "1" }, + stdio: "inherit", + } + ); + process.exit(0); +} + +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const acceptanceRoot = path.join(repo, "target/acceptance"); +const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = path.join(acceptanceRoot, "cli-happy-path-live.json"); +const serviceEndpoint = "https://clusterflux.michelpaulissen.com"; +const serviceAddr = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR || + "clusterflux.michelpaulissen.com:443"; +const browserOpenCommand = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND; +const reuseSessionFile = + process.env.CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE; +const qualityGateEvidencePath = + process.env.CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH; +const enabled = process.env.CLUSTERFLUX_CLI_HAPPY_PATH_LIVE === "1"; +const strictFullRelease = process.env.CLUSTERFLUX_STRICT_FULL_RELEASE === "1"; +const strictVpsRestart = process.env.CLUSTERFLUX_STRICT_VPS_RESTART === "1"; +const strictVpsHost = process.env.CLUSTERFLUX_STRICT_VPS_HOST; +const strictVpsIdentity = process.env.CLUSTERFLUX_STRICT_VPS_IDENTITY; +const strictServiceUnit = + process.env.CLUSTERFLUX_STRICT_SERVICE_UNIT || + "clusterflux-hosted.service"; +const strictSoakSeconds = Number( + process.env.CLUSTERFLUX_STRICT_SOAK_SECONDS || "300" +); +const commands = []; + +function requireEnabled() { + if (!enabled) { + throw new Error( + "CLUSTERFLUX_CLI_HAPPY_PATH_LIVE=1 is required because this runs the live CLI happy path" + ); + } + if (!browserOpenCommand && !reuseSessionFile) { + throw new Error( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND or CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE is required" + ); + } + if ( + strictFullRelease && + !process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE + ) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE for live isolation and quota independence" + ); + } + if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE" + ); + } + if (strictFullRelease && !qualityGateEvidencePath) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH" + ); + } + if (strictFullRelease && !strictVpsRestart) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_STRICT_VPS_RESTART=1" + ); + } + if (strictVpsRestart && (!strictVpsHost || !strictVpsIdentity)) { + throw new Error( + "strict VPS restart requires CLUSTERFLUX_STRICT_VPS_HOST and CLUSTERFLUX_STRICT_VPS_IDENTITY" + ); + } + if ( + !Number.isInteger(strictSoakSeconds) || + strictSoakSeconds < 120 || + strictSoakSeconds > 1800 + ) { + throw new Error("CLUSTERFLUX_STRICT_SOAK_SECONDS must be an integer from 120 to 1800"); + } +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function strictQualityGateEvidence(manifest) { + if (!strictFullRelease) return null; + const evidence = readJson(path.resolve(qualityGateEvidencePath)); + assert.strictEqual(evidence.source_commit, manifest.source_commit); + assert.strictEqual(evidence.source_tree_digest, manifest.source_tree_digest); + for (const gate of [evidence.private, evidence.public]) { + assert.strictEqual(gate.status, "passed"); + assert(Number.isFinite(gate.duration_ms) && gate.duration_ms > 0); + } + assert.strictEqual(evidence.public.independent_filtered_tree, true); + return evidence; +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function sha256(bytes) { + return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`; +} + +function nonInteractiveGitEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function commandEnv(command, env) { + if (command !== "git") return env; + return nonInteractiveGitEnv(env); +} + +function recordCommand(command, args) { + const redacted = []; + let hideNext = false; + for (const argument of args) { + if (hideNext) { + redacted.push(""); + hideNext = false; + } else { + redacted.push(argument); + hideNext = ["--enrollment-grant", "--admin-token"].includes(argument); + } + } + commands.push([command, ...redacted].join(" ")); +} + +function run(command, args, options = {}) { + recordCommand(command, args); + return cp.execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 15 * 60 * 1000, + ...options, + env: commandEnv(command, options.env), + }); +} + +function parseJsonOutput(output) { + const trimmed = output.trim(); + if (!trimmed) throw new Error("expected JSON output, got empty stdout"); + try { + return JSON.parse(trimmed); + } catch (_) { + // Commands may print progress before their final JSON report. + } + const lines = trimmed.split(/\r?\n/).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + try { + return JSON.parse(lines.slice(index).join("\n")); + } catch (_) { + // Keep scanning for the trailing JSON value. + } + } + throw new Error(`could not parse JSON output:\n${trimmed}`); +} + +function runJson(command, args, options = {}) { + return parseJsonOutput(run(command, args, options)); +} + +function authenticatedRequest(sessionSecret, request) { + return { + type: "authenticated", + session_secret: sessionSecret, + request, + }; +} + +function sendHostedControl(payload) { + const body = Buffer.from( + JSON.stringify(coordinatorWireRequest(payload, "strict-live")) + ); + const url = new URL("/api/v1/control", serviceEndpoint); + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + }, + timeout: 30_000, + }, + (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => { + const responseBody = Buffer.concat(chunks).toString("utf8"); + if (response.statusCode < 200 || response.statusCode >= 300) { + reject( + new Error( + `hosted control HTTP ${response.statusCode}: ${responseBody}` + ) + ); + return; + } + try { + resolve(JSON.parse(responseBody)); + } catch (error) { + reject( + new Error(`hosted control returned invalid JSON: ${error.message}`) + ); + } + }); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted control request timed out")) + ); + request.on("error", reject); + request.end(body); + }); +} + +function sendHostedControlDroppingResponse(payload) { + const body = Buffer.from( + JSON.stringify(coordinatorWireRequest(payload, "strict-live-dropped-response")) + ); + const url = new URL("/api/v1/control", serviceEndpoint); + return new Promise((resolve, reject) => { + let settled = false; + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + }, + timeout: 30_000, + }, + (response) => { + settled = true; + const statusCode = response.statusCode; + response.destroy(); + resolve({ status_code: statusCode, response_body_consumed: false }); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted dropped-response request timed out")) + ); + request.on("error", (error) => { + if (!settled) reject(error); + }); + request.end(body); + }); +} + +function sendHostedLoginStatus(extraHeaders = {}) { + const body = Buffer.from( + JSON.stringify( + coordinatorWireRequest( + { type: "begin_oidc_browser_login" }, + `strict-live-login-${Date.now()}-${Math.random()}` + ) + ) + ); + const url = new URL("/api/v1/login", serviceEndpoint); + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + ...extraHeaders, + }, + timeout: 30_000, + }, + (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => + resolve({ + status_code: response.statusCode, + body: Buffer.concat(chunks).toString("utf8"), + }) + ); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted login request timed out")) + ); + request.on("error", reject); + request.end(body); + }); +} + +async function runHostedLoginIsolationBeforeRestart() { + const scenarioStartedAt = Date.now(); + const controlBefore = await sendHostedControl({ type: "ping" }); + assert.strictEqual(controlBefore.type, "pong"); + const statuses = []; + let limited; + for (let index = 0; index < 128; index += 1) { + const response = await sendHostedLoginStatus(); + statuses.push(response.status_code); + if (response.status_code === 429) { + limited = response; + break; + } + assert.strictEqual(response.status_code, 200, response.body); + } + assert(limited, `hosted login route was not rate limited: ${statuses.join(",")}`); + const spoofed = await sendHostedLoginStatus({ + forwarded: "for=203.0.113.99", + "x-forwarded-for": "203.0.113.99", + "x-real-ip": "203.0.113.99", + }); + assert.strictEqual( + spoofed.status_code, + 429, + "caller-supplied forwarding headers bypassed the client rate limit" + ); + + const remoteBody = JSON.stringify( + coordinatorWireRequest( + { type: "begin_oidc_browser_login" }, + `strict-vps-login-${Date.now()}` + ) + ); + const remoteStatus = Number( + run( + "ssh", + sshArgs( + "curl", + "--silent", + "--show-error", + "--output", + "/dev/null", + "--write-out=%{http_code}", + "-HContent-Type:application/json", + `--data-binary=${remoteBody}`, + `${serviceEndpoint}/api/v1/login` + ) + ).trim() + ); + assert.strictEqual( + remoteStatus, + 200, + "one client exhausted the browser-login allowance for a different client" + ); + const controlAfter = await sendHostedControl({ type: "ping" }); + assert.strictEqual(controlAfter.type, "pong"); + return { + scenario_started_at_ms: scenarioStartedAt, + local_statuses: statuses, + local_limited_status: limited.status_code, + forwarding_spoof_status: spoofed.status_code, + independent_vps_client_status: remoteStatus, + control_route_before: controlBefore.type, + control_route_after: controlAfter.type, + }; +} + +async function finishHostedLoginIsolationAfterRestart(evidence) { + const control = await sendHostedControl({ type: "ping" }); + assert.strictEqual(control.type, "pong"); + const response = await sendHostedLoginStatus(); + assert( + [200, 429].includes(response.status_code), + `login route did not recover after service restart: ${response.status_code} ${response.body}` + ); + evidence.after_service_restart = { + control_route: control.type, + login_route_status: response.status_code, + }; + evidence.duration_ms = Date.now() - evidence.scenario_started_at_ms; + delete evidence.scenario_started_at_ms; + return evidence; +} + +function assertDenied(response, pattern, label) { + assert.strictEqual(response.type, "error", `${label}: ${JSON.stringify(response)}`); + assert.match(response.message, pattern, `${label}: ${JSON.stringify(response)}`); + return { + denied: true, + category: response.error?.category || null, + message: response.message, + }; +} + +function readNodeCredential(projectDir, node) { + const directory = path.join(projectDir, ".clusterflux", "nodes"); + for (const file of fs.readdirSync(directory)) { + if (!file.endsWith(".json")) continue; + const absolute = path.join(directory, file); + const credential = readJson(absolute); + if (credential.node === node) return { absolute, credential }; + } + throw new Error(`persisted credential for ${node} was not found`); +} + +function directoryStats(root) { + if (!fs.existsSync(root)) return { files: 0, bytes: 0 }; + let files = 0; + let bytes = 0; + const visit = (directory) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile()) { + files += 1; + bytes += fs.statSync(absolute).size; + } + } + }; + visit(root); + return { files, bytes }; +} + +function shellQuote(argument) { + return `'${String(argument).replaceAll("'", "'\"'\"'")}'`; +} + +function sshArgs(...remoteArgs) { + assert(strictVpsHost && strictVpsIdentity); + return [ + "-i", + path.resolve(strictVpsIdentity.replace(/^~(?=\/)/, os.homedir())), + "-o", + "BatchMode=yes", + strictVpsHost, + remoteArgs.map(shellQuote).join(" "), + ]; +} + +function strictInfrastructureSample(projectDir) { + const memory = Object.fromEntries( + run( + "ssh", + sshArgs( + "systemctl", + "show", + strictServiceUnit, + "--property=MemoryCurrent", + "--property=MemoryPeak" + ) + ) + .trim() + .split(/\r?\n/) + .map((line) => line.split("=")) + ); + const serviceDisk = Number( + run( + "ssh", + sshArgs("du", "-sb", "/var/lib/clusterflux-public-release") + ) + .trim() + .split(/\s+/)[0] + ); + const database = run( + "ssh", + sshArgs( + "runuser", + "-u", + "postgres", + "--", + "psql", + "-d", + "clusterflux", + "-AtF," + ), + { + input: + "SELECT pg_database_size('clusterflux'), (SELECT count(*) FROM clusterflux_tenants), (SELECT count(*) FROM clusterflux_projects), (SELECT count(*) FROM clusterflux_node_identities), (SELECT count(*) FROM clusterflux_cli_sessions);\n", + stdio: ["pipe", "pipe", "pipe"], + } + ) + .trim() + .split(",") + .map(Number); + assert(Number.isFinite(Number(memory.MemoryCurrent))); + assert(Number.isFinite(Number(memory.MemoryPeak))); + assert.strictEqual(database.length, 5); + return { + sampled_at_epoch_ms: Date.now(), + service_memory_current_bytes: Number(memory.MemoryCurrent), + service_memory_peak_bytes: Number(memory.MemoryPeak), + service_state_disk_bytes: serviceDisk, + postgres_database_bytes: database[0], + durable_rows: { + tenants: database[1], + projects: database[2], + node_identities: database[3], + cli_sessions: database[4], + }, + local_node_state: directoryStats(path.join(projectDir, ".clusterflux")), + }; +} + +function executable(root, name) { + return path.join(root, "bin", process.platform === "win32" ? `${name}.exe` : name); +} + +function binaryArchive(manifest) { + const platform = `${os.platform()}-${os.arch()}`; + const asset = manifest.assets.find( + (candidate) => + candidate.name.startsWith("clusterflux-public-binaries-") && + candidate.name.includes(platform) + ); + assert(asset, `missing released binary archive for ${platform}`); + assert(fs.existsSync(asset.file), `missing released binary archive ${asset.file}`); + return asset.file; +} + +function publicRepoUrl(manifest) { + const url = + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + manifest.public_repo_url || + manifest.public_repo_remote; + assert(url, "public repository URL is required"); + assert(url.includes("git.michelpaulissen.com")); + return url; +} + +function stagePublicCheckout(manifest, checkout) { + const candidateTree = process.env.CLUSTERFLUX_LIVE_PUBLIC_TREE; + if (candidateTree) { + const source = path.resolve(candidateTree); + assert.strictEqual( + source, + path.resolve(manifest.public_tree), + "explicit live candidate tree must be the tree recorded by the release manifest" + ); + fs.cpSync(source, checkout, { recursive: true }); + return { + kind: "local_filtered_release_candidate", + path: source, + published: false, + }; + } + run("git", ["clone", "--depth", "1", publicRepoUrl(manifest), checkout]); + return { + kind: "published_forgejo_checkout", + url: publicRepoUrl(manifest), + published: true, + }; +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function waitForCli(label, action, predicate, timeoutMs = 10 * 60 * 1000) { + const deadline = Date.now() + timeoutMs; + let lastValue; + let lastError; + while (Date.now() < deadline) { + try { + lastValue = action(); + if (predicate(lastValue)) return lastValue; + lastError = undefined; + } catch (error) { + lastError = error; + } + await delay(250); + } + throw new Error( + `${label} timed out; last value=${JSON.stringify(lastValue)}${ + lastError ? `; last error=${lastError.message}` : "" + }` + ); +} + +function spawnJsonLines(command, args, options) { + recordCommand(command, args); + const child = cp.spawn(command, args, { + ...options, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const values = []; + const waiters = []; + + function deliver(value) { + values.push(value); + for (let index = waiters.length - 1; index >= 0; index -= 1) { + const waiter = waiters[index]; + if (waiter.predicate(value)) { + waiters.splice(index, 1); + clearTimeout(waiter.timer); + waiter.resolve(value); + } + } + } + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + while (stdout.includes("\n")) { + const newline = stdout.indexOf("\n"); + const line = stdout.slice(0, newline).trim(); + stdout = stdout.slice(newline + 1); + if (!line) continue; + try { + deliver(JSON.parse(line)); + } catch (_) { + // Non-JSON progress stays out of the evidence report. + } + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + function waitFor(predicate, label, timeoutMs = 120000) { + const existing = values.find(predicate); + if (existing) return Promise.resolve(existing); + return new Promise((resolve, reject) => { + const waiter = { predicate, resolve, reject, timer: undefined }; + waiter.timer = setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(new Error(`${label} timed out\n${stderr}`)); + }, timeoutMs); + waiters.push(waiter); + }); + } + + child.once("exit", (code) => { + for (const waiter of waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(new Error(`${waiter.label || "child"} exited with ${code}\n${stderr}`)); + } + }); + return { child, values, waitFor, stderr: () => stderr }; +} + +async function stopChild(child) { + if (!child || child.exitCode !== null) return; + child.kill("SIGTERM"); + const exited = new Promise((resolve) => child.once("exit", resolve)); + const forced = delay(5000).then(() => { + if (child.exitCode === null) child.kill("SIGKILL"); + }); + await Promise.race([exited, forced]); + if (child.exitCode === null) await exited; +} + +function rawTaskEvents(report) { + return report?.events?.response?.events || []; +} + +function nodeDescriptor(status, node) { + return status?.response?.descriptors?.find( + (descriptor) => descriptor.id === node || descriptor.node === node + ); +} + +function completedFlagship(events) { + const completedDefinitions = new Set( + events + .filter((event) => event.terminal_state === "completed") + .map((event) => event.task_definition) + ); + return ( + completedDefinitions.has("prepare_source") && + completedDefinitions.has("compile_linux") && + completedDefinitions.has("package_release") && + events.some( + (event) => + event.executor === "coordinator_main" && event.terminal_state === "completed" + ) + ); +} + +function contentAddressedArtifactId(event, logicalName) { + const digest = event?.artifact_digest; + if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) { + return null; + } + const artifactId = `${logicalName}-${digest.slice("sha256:".length)}`; + return event.artifact_path?.endsWith(`/vfs/artifacts/${artifactId}`) + ? artifactId + : null; +} + +async function prepareLiveNodeCredentialSecurity({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, +}) { + const node = `security-node-${suffix}`; + const grant = runJson(clusterflux, ["node", "enroll", ...scope], { + cwd: projectDir, + }); + const attach = runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + node, + "--enrollment-grant", + grant.enrollment_grant.grant, + "--json", + ], + { cwd: projectDir } + ); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + const stored = readNodeCredential(projectDir, node); + const identity = nodeIdentityFromPrivateKey(stored.credential.private_key); + assert.strictEqual(identity.publicKey, stored.credential.public_key); + + const heartbeat = { + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity, { + nonce: `strict-valid-${suffix}`, + }), + }; + const valid = await sendHostedControl(heartbeat); + assert.strictEqual(valid.type, "node_heartbeat"); + const replay = assertDenied( + await sendHostedControl(heartbeat), + /nonce.*already.*used|replay/i, + "replayed node credential" + ); + + const expired = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity, { + nonce: `strict-expired-${suffix}`, + issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, + }), + }), + /expired|clock skew/i, + "expired node credential" + ); + + const forgedIdentity = nodeIdentity("strict-forged-node", node); + const forged = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, forgedIdentity, { + nonce: `strict-forged-${suffix}`, + }), + }), + /signature|public key/i, + "forged node credential" + ); + + const originalCapabilityBody = { + type: "report_node_capabilities", + tenant, + project, + node, + capabilities: { + os: "Linux", + arch: process.arch === "x64" ? "x86_64" : process.arch, + capabilities: [], + environment_backends: [], + source_providers: [], + }, + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false, + online: true, + }; + const modifiedEnvelope = signedNodeRequest( + node, + identity, + "report_node_capabilities", + originalCapabilityBody, + { nonce: `strict-modified-${suffix}` } + ); + modifiedEnvelope.request.online = false; + const bodyModified = assertDenied( + await sendHostedControl(modifiedEnvelope), + /signature/i, + "body-modified node credential" + ); + + return { + node, + identity, + credential_path: stored.absolute, + credential_digest: sha256(fs.readFileSync(stored.absolute)), + capability_body: originalCapabilityBody, + evidence: { + valid_request: valid.type, + replay, + expired, + forged, + body_modified: bodyModified, + }, + }; +} + +async function runLiveAgentCredentialSecurity({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + suffix, + bundle, + securityNode, + workerRuntime, +}) { + const agent = `security-agent-${suffix}`; + const identity = agentIdentity("strict-live-agent", agent); + const added = runJson( + clusterflux, + ["key", "add", ...scope, "--agent", agent, "--public-key", identity.publicKey], + { cwd: projectDir } + ); + assert.strictEqual(added.command, "key add"); + + const processId = `vp-agent-security-${suffix}`; + const unsigned = { + type: "start_process", + tenant, + project, + actor_agent: agent, + process: processId, + restart: false, + }; + const validRequest = signedAgentWorkflowRequest(identity, unsigned, { + nonce: `strict-agent-valid-${suffix}`, + }); + const valid = await sendHostedControl(validRequest); + assert.strictEqual(valid.type, "process_started", JSON.stringify(valid)); + const replay = assertDenied( + await sendHostedControl(validRequest), + /nonce.*already.*used|replay/i, + "replayed agent credential" + ); + + const expired = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-expired` }, + { + nonce: `strict-agent-expired-${suffix}`, + issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, + } + ) + ), + /expired|clock skew/i, + "expired agent credential" + ); + + const modifiedRequest = signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-modified` }, + { nonce: `strict-agent-modified-${suffix}` } + ); + modifiedRequest.restart = true; + const bodyModified = assertDenied( + await sendHostedControl(modifiedRequest), + /signature/i, + "body-modified agent credential" + ); + + const forgedIdentity = agentIdentity("strict-live-forged-agent", agent); + const forged = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + forgedIdentity, + { ...unsigned, process: `${processId}-forged` }, + { + nonce: `strict-agent-forged-${suffix}`, + publicKeyFingerprint: identity.publicKeyFingerprint, + } + ) + ), + /signature|public key/i, + "forged agent credential" + ); + + const fakeEnvironmentDigest = sha256( + Buffer.from(`strict-debug-missing-participant-${suffix}`) + ); + const fakeCapabilityBody = { + ...securityNode.capability_body, + cached_environment_digests: [fakeEnvironmentDigest], + online: true, + }; + const mainTask = `ti:${processId}:main`; + const fakeTask = `missing-debug-participant-${suffix}`; + const mainLaunchBody = { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: { + tenant, + project, + process: processId, + task_definition: bundle.entryStableId, + task_instance: mainTask, + dispatch: { + kind: "coordinator_node_wasm", + export: bundle.entryExport, + abi: "entrypoint_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: true, + artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + let mainLaunch; + let directTaskV1; + let fakeLaunch; + let liveParentTask; + let liveParentAssignment; + let realDebugLaunch; + const realDebugTask = `real-debug-participant-${suffix}`; + let workerPaused = false; + const baselineContainers = new Set( + run("podman", ["ps", "-q"]) + .trim() + .split(/\s+/) + .filter(Boolean) + ); + try { + mainLaunch = await sendHostedControl( + signedAgentWorkflowRequest(identity, mainLaunchBody, { + nonce: `strict-agent-main-${suffix}`, + }) + ); + assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch)); + assert.strictEqual(mainLaunch.task_instance, mainTask); + assert.strictEqual(mainLaunch.state, "running"); + assert.strictEqual(mainLaunch.actor.kind, "agent"); + assert.strictEqual(mainLaunch.actor.authenticated_without_browser, true); + + liveParentAssignment = await workerRuntime.worker.waitFor( + (value) => { + const taskSpec = value.task_assignment_response?.task_spec; + return ( + value.node_status === "assignment_started" && + value.node === workerRuntime.node && + value.process === processId && + taskSpec?.dispatch?.abi === "task_v1" && + taskSpec.task_definition === "compile_linux" && + taskSpec.environment_id === "linux" && + typeof taskSpec.environment_digest === "string" && + typeof taskSpec.source_snapshot === "string" + ); + }, + "agent main to dispatch an environment-bound live child to the real worker", + 120000 + ); + liveParentTask = liveParentAssignment.virtual_thread; + assert.strictEqual(typeof liveParentTask, "string"); + assert.notStrictEqual(liveParentTask, ""); + assert.strictEqual( + workerRuntime.child.kill("SIGSTOP"), + true, + "failed to pause the real worker after observing its live child" + ); + workerPaused = true; + + const fakeCapabilities = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_node_capabilities", + fakeCapabilityBody, + { nonce: `strict-agent-fake-capabilities-${suffix}` } + ) + ); + assert.strictEqual( + fakeCapabilities.type, + "node_capabilities_recorded", + JSON.stringify(fakeCapabilities) + ); + + const fakeTaskSpec = { + tenant, + project, + process: processId, + task_definition: "task_add_one", + task_instance: fakeTask, + dispatch: { + kind: "coordinator_node_wasm", + export: null, + abi: "task_v1", + }, + environment_id: "strict-debug-missing-participant", + environment: null, + environment_digest: fakeEnvironmentDigest, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [{ SmallJson: 41 }], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }; + directTaskV1 = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: fakeTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `strict-agent-direct-task-v1-${suffix}` } + ) + ), + /external callers.*EntrypointV1|TaskV1 requires an authenticated live parent/i, + "external direct TaskV1 launch" + ); + + const liveParentSpec = + liveParentAssignment.task_assignment_response.task_spec; + realDebugLaunch = await sendHostedControl( + signedNodeRequest( + workerRuntime.node, + workerRuntime.identity, + "launch_child_task", + { + type: "launch_child_task", + tenant, + project, + process: processId, + node: workerRuntime.node, + parent_task: liveParentTask, + task_spec: { + tenant, + project, + process: processId, + task_definition: "abort_probe", + task_instance: realDebugTask, + dispatch: { + kind: "coordinator_node_wasm", + export: null, + abi: "task_v1", + }, + environment_id: liveParentSpec.environment_id, + environment: liveParentSpec.environment, + environment_digest: liveParentSpec.environment_digest, + required_capabilities: ["Command"], + dependency_cache: null, + source_snapshot: liveParentSpec.source_snapshot, + required_artifacts: [], + args: liveParentSpec.args, + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${realDebugTask}.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `strict-real-debug-participant-${suffix}` } + ) + ); + assert.strictEqual( + realDebugLaunch.type, + "task_launched", + JSON.stringify(realDebugLaunch) + ); + + const fakeLaunchBody = { + type: "launch_child_task", + tenant, + project, + process: processId, + node: workerRuntime.node, + parent_task: liveParentTask, + task_spec: fakeTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + fakeLaunch = await sendHostedControl( + signedNodeRequest( + workerRuntime.node, + workerRuntime.identity, + "launch_child_task", + fakeLaunchBody, + { nonce: `strict-parent-runtime-fake-task-${suffix}` } + ) + ); + assert.strictEqual(fakeLaunch.type, "task_launched", JSON.stringify(fakeLaunch)); + assert.strictEqual(fakeLaunch.placement.node, securityNode.node); + } finally { + if (workerPaused) { + assert.strictEqual( + workerRuntime.child.kill("SIGCONT"), + true, + "failed to resume the real worker after authenticated child launch" + ); + } + } + await workerRuntime.worker.waitFor( + (value) => + value.node_status === "assignment_started" && + value.virtual_thread === realDebugTask, + "real Podman debug participant to start", + 120_000 + ); + const realDebugContainerDeadline = Date.now() + 120_000; + let realDebugContainerIds = new Set(); + let lastRealDebugPodmanStates = []; + while (Date.now() < realDebugContainerDeadline) { + const containers = runJson("podman", ["ps", "--format", "json"]); + lastRealDebugPodmanStates = containers; + realDebugContainerIds = new Set( + containers + .map((container) => container.Id || container.ID || container.IdHex || "") + .filter((id) => id && !baselineContainers.has(id)) + ); + if (realDebugContainerIds.size > 0) break; + await delay(100); + } + assert( + realDebugContainerIds.size > 0, + `real debug participant did not start a Podman container: ${JSON.stringify( + lastRealDebugPodmanStates + )}` + ); + + const debugClient = new DapClient({ + cwd: projectDir, + command: clusterfluxDap, + args: [], + env: { ...process.env }, + }); + const initialize = debugClient.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await debugClient.response(initialize, "initialize"); + const attachRequest = debugClient.send("attach", { + entry: "build", + project: projectDir, + processId, + runtimeBackend: "live-services", + coordinatorEndpoint: serviceEndpoint, + }); + await debugClient.response(attachRequest, "attach"); + await debugClient.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + const configurationDone = debugClient.send("configurationDone"); + await debugClient.response(configurationDone, "configurationDone"); + const attachDeadline = Date.now() + 120_000; + let attachedThreads = []; + while (Date.now() < attachDeadline) { + const threadsRequest = debugClient.send("threads"); + attachedThreads = ( + await debugClient.response(threadsRequest, "threads") + ).body.threads; + if (attachedThreads.length > 0) break; + await delay(100); + } + assert(attachedThreads.length > 0, "DAP attach reported no live threads"); + + const freezeStartedAt = Date.now(); + const pauseRequest = debugClient.send("pause", { + threadId: attachedThreads[0].id, + }); + await debugClient.response(pauseRequest, "pause"); + const dapPartialStop = await debugClient.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "pause", + 30_000 + ); + assert.strictEqual(dapPartialStop.body.allThreadsStopped, false); + const epochRequest = debugClient.send("evaluate", { + expression: "debug_epoch", + context: "watch", + }); + const freeze = { + epoch: Number( + (await debugClient.response(epochRequest, "evaluate")).body.result + ), + }; + assert(Number.isSafeInteger(freeze.epoch) && freeze.epoch > 0); + const partialFreeze = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "inspect_debug_epoch", + process: processId, + epoch: freeze.epoch, + }) + ); + assert.strictEqual(partialFreeze.type, "debug_epoch_status"); + assert.strictEqual(partialFreeze.partially_frozen, true, JSON.stringify(partialFreeze)); + assert.strictEqual(partialFreeze.fully_frozen, false); + assert.strictEqual(partialFreeze.failed, true); + assert.match( + partialFreeze.failure_messages.join("; "), + /did not acknowledge frozen state within \d+ ms/ + ); + const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]); + const pausedContainers = podmanStates.filter((container) => { + const id = container.Id || container.ID || container.IdHex || ""; + const state = String(container.State || container.Status || "").toLowerCase(); + return realDebugContainerIds.has(id) && state.includes("paused"); + }); + assert( + pausedContainers.length > 0, + `partial Debug Epoch did not pause a real Podman container: ${JSON.stringify( + podmanStates + )}` + ); + const pausedContainerIds = pausedContainers.map( + (container) => container.Id || container.ID || container.IdHex + ); + + const mismatchedDigest = sha256(Buffer.from(`strict-nested-mismatch-${suffix}`)); + const mismatchedChild = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "launch_child_task", + { + type: "launch_child_task", + tenant, + project, + process: processId, + node: securityNode.node, + parent_task: fakeTask, + task_spec: { + tenant, + project, + process: processId, + task_definition: "task_add_one", + task_instance: `${fakeTask}:child:mismatched-environment`, + dispatch: { + kind: "coordinator_node_wasm", + export: null, + abi: "task_v1", + }, + environment_id: "strict-nested-mismatch", + environment: null, + environment_digest: mismatchedDigest, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [{ SmallJson: 41 }], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}-mismatched-child.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `strict-nested-mismatch-${suffix}` } + ) + ); + const nestedEnvironmentMismatch = assertDenied( + mismatchedChild, + /environment|digest|compatible|placement|no node/i, + "nested environment mismatch" + ); + + const continueStartedAt = Date.now(); + const continueRequest = debugClient.send("continue", { + threadId: dapPartialStop.body.threadId, + }); + await debugClient.response(continueRequest, "continue"); + const continueResponseMs = Date.now() - continueStartedAt; + assert( + continueResponseMs < 2_000, + `partial-freeze continue blocked for ${continueResponseMs} ms` + ); + await delay(1000); + const resumed = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "inspect_debug_epoch", + process: processId, + epoch: freeze.epoch, + }) + ); + assert.strictEqual(resumed.type, "debug_epoch_status"); + const resumedAcknowledgements = resumed.acknowledgements.filter( + (acknowledgement) => acknowledgement.state === "running" + ); + assert(resumedAcknowledgements.length > 0, JSON.stringify(resumed)); + const resumedPodmanStates = runJson("podman", ["ps", "--format", "json"]); + assert( + !resumedPodmanStates.some((container) => { + const id = container.Id || container.ID || container.IdHex || ""; + const state = String(container.State || container.Status || "").toLowerCase(); + return pausedContainerIds.includes(id) && state.includes("paused"); + }), + `DAP continue left a Clusterflux container paused: ${JSON.stringify( + resumedPodmanStates + )}` + ); + await debugClient.close(); + + const completed = await waitForCli( + "agent-authenticated coordinator main and child workflow completion", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { + cwd: projectDir, + }), + (tasks) => completedFlagship(rawTaskEvents(tasks)), + 120000 + ); + const completedEvents = rawTaskEvents(completed); + const concurrentCompileTasks = [ + ...new Set( + completedEvents + .filter( + (event) => + event.task_definition === "compile_linux" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents)); + + const revoked = runJson( + clusterflux, + ["key", "revoke", ...scope, "--agent", agent, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(revoked.command, "key revoke"); + const revokedUse = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-revoked` }, + { nonce: `strict-agent-revoked-${suffix}` } + ) + ), + /revoked/i, + "revoked agent credential" + ); + return { + agent, + valid_request: valid.type, + replay, + expired, + forged, + body_modified: bodyModified, + revoked: revokedUse, + workflow: { + process: processId, + main_launch: mainLaunch.type, + external_direct_task_v1: directTaskV1, + child_launch: fakeLaunch.type, + child_parent: liveParentTask, + real_debug_child_launch: realDebugLaunch.type, + real_debug_child: realDebugTask, + actor_kind: mainLaunch.actor.kind, + authenticated_without_browser: mainLaunch.actor.authenticated_without_browser, + flagship_completed: completedFlagship(completedEvents), + concurrent_compile_tasks: concurrentCompileTasks, + }, + partial_freeze: { + epoch: freeze.epoch, + elapsed_ms: Date.now() - freezeStartedAt, + partially_frozen: partialFreeze.partially_frozen, + fully_frozen: partialFreeze.fully_frozen, + dap_all_threads_stopped: dapPartialStop.body.allThreadsStopped, + dap_continue_response_ms: continueResponseMs, + podman_paused_container_ids: pausedContainerIds, + warning: partialFreeze.failure_messages, + resumed_participants: resumedAcknowledgements.map( + (acknowledgement) => acknowledgement.task + ), + }, + nested_environment_mismatch: nestedEnvironmentMismatch, + }; +} + +async function runSecondTenantIsolation({ + firstTenant, + firstProject, + firstProcess, + firstNode, + firstArtifact, + manifest, +}) { + const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE; + const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE; + if (!file && evidenceFile) { + const evidenceBytes = fs.readFileSync(path.resolve(evidenceFile)); + const previous = JSON.parse(evidenceBytes); + const isolation = previous.tenant_isolation; + assert.strictEqual(isolation?.executed, true); + assert.notStrictEqual(isolation.second_tenant, firstTenant); + assert.strictEqual(isolation.project?.denied, true); + assert.strictEqual(isolation.process_hidden, true); + assert.strictEqual(isolation.node_hidden, true); + assert.strictEqual(isolation.tasks_and_logs?.denied, true); + assert.strictEqual(isolation.debug?.denied, true); + assert.strictEqual(isolation.debug?.audited, true); + assert.strictEqual(isolation.debug?.charged_debug_read_bytes, 0); + assert.strictEqual(isolation.artifact_and_download?.denied, true); + assert.strictEqual(isolation.process_control?.denied, true); + const previousDeployment = previous.release_binding?.deployment; + const currentDeployment = deploymentProvenance(); + assert.strictEqual( + previousDeployment?.hosted_service_sha256, + currentDeployment.hosted_service_sha256, + "reused isolation evidence must target the exact deployed hosted binary" + ); + assert.deepStrictEqual( + previous.release_binding?.binary_digests, + manifest.binary_digests, + "reused isolation evidence must target the exact public binaries" + ); + return { + ...isolation, + reused_from_immutable_evidence: { + report_sha256: sha256(evidenceBytes), + source_commit: previous.source_commit, + hosted_service_sha256: currentDeployment.hosted_service_sha256, + public_binary_digests: manifest.binary_digests, + }, + }; + } + if (!file) return { executed: false, reason: "second tenant session not supplied" }; + const second = readJson(path.resolve(file)); + assert.strictEqual(second.coordinator, serviceEndpoint); + assert(second.session_secret, "second tenant session omitted its session secret"); + assert.notStrictEqual( + second.tenant, + firstTenant, + "isolation proof requires a genuinely distinct hosted tenant" + ); + const call = (request) => + sendHostedControl(authenticatedRequest(second.session_secret, request)); + + const project = assertDenied( + await call({ type: "select_project", project: firstProject }), + /outside|not visible|tenant|project/i, + "cross-tenant project selection" + ); + const processes = await call({ type: "list_processes" }); + assert.strictEqual( + processes.type, + "process_statuses", + JSON.stringify(processes) + ); + assert(!JSON.stringify(processes).includes(firstProcess)); + const nodes = await call({ type: "list_node_descriptors" }); + assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes)); + assert(!JSON.stringify(nodes).includes(firstNode)); + const tasksAndLogs = assertDenied( + await call({ type: "list_task_events", process: firstProcess }), + /outside|scope|tenant|project|not active|requires an active|unknown/i, + "cross-tenant task and log listing" + ); + const debugResponse = await call({ + type: "debug_attach", + process: firstProcess, + }); + assert.strictEqual(debugResponse.type, "debug_attach", JSON.stringify(debugResponse)); + assert.strictEqual(debugResponse.authorization?.allowed, false); + assert.strictEqual(debugResponse.audit_event?.allowed, false); + assert.match( + debugResponse.authorization?.reason || "", + /outside|scope|permission|tenant|project|not active|unknown/i + ); + assert.strictEqual(debugResponse.charged_debug_read_bytes, 0); + const debug = { + denied: true, + reason: debugResponse.authorization.reason, + audited: true, + charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, + }; + const artifact = assertDenied( + await call({ + type: "create_artifact_download_link", + artifact: firstArtifact, + max_bytes: 1024, + ttl_seconds: 60, + }), + /outside|scope|tenant|project|not found|unavailable/i, + "cross-tenant artifact download" + ); + const control = assertDenied( + await call({ type: "abort_process", process: firstProcess }), + /outside|scope|tenant|project|not active|requires an active|unknown/i, + "cross-tenant process control" + ); + return { + executed: true, + second_tenant: second.tenant, + project, + process_hidden: true, + node_hidden: true, + tasks_and_logs: tasksAndLogs, + debug, + artifact_and_download: artifact, + process_control: control, + }; +} + +async function runLiveSoak({ + clusterflux, + projectDir, + scope, + securityNode, +}) { + if (!strictVpsRestart) { + return { executed: false, reason: "strict VPS measurement access not supplied" }; + } + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const processId = runReport.process; + const parked = await waitForCli( + "soak coordinator main to park without a capable node", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => + status.live_process?.main_state === "running" && + status.live_process?.main_wait_state === "waiting_for_task" && + status.current_task_count === 0 && + status.live_process?.connected_nodes?.length === 0, + 120000 + ); + + const pollSecurityNode = (nonce) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant: securityNode.capability_body.tenant, + project: securityNode.capability_body.project, + node: securityNode.node, + }, + { nonce } + ) + ); + const drainedPriorAssignments = []; + for (let drainIndex = 0; drainIndex < 16; drainIndex += 1) { + const poll = await pollSecurityNode( + `strict-soak-drain-${drainIndex}-${Date.now()}` + ); + assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); + if (poll.assignment === null) break; + assert.notStrictEqual( + poll.assignment.process, + processId, + "soak process unexpectedly received a task assignment" + ); + drainedPriorAssignments.push({ + process: poll.assignment.process, + task: poll.assignment.task, + }); + assert(drainIndex < 15, "pre-soak assignment drain did not quiesce"); + } + + const startedAt = Date.now(); + const deadline = startedAt + strictSoakSeconds * 1000; + const samples = []; + let sampleIndex = 0; + while (true) { + const capabilities = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_node_capabilities", + securityNode.capability_body, + { nonce: `strict-soak-capabilities-${sampleIndex}-${Date.now()}` } + ) + ); + assert.strictEqual( + capabilities.type, + "node_capabilities_recorded", + JSON.stringify(capabilities) + ); + const poll = await pollSecurityNode( + `strict-soak-poll-${sampleIndex}-${Date.now()}` + ); + assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); + assert.strictEqual(poll.assignment, null); + const status = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.strictEqual(status.live_process.main_wait_state, "waiting_for_task"); + samples.push(strictInfrastructureSample(projectDir)); + sampleIndex += 1; + if (Date.now() >= deadline) break; + await delay(Math.min(30_000, deadline - Date.now())); + } + + const currentMemory = samples.map( + (sample) => sample.service_memory_current_bytes + ); + const serviceDisk = samples.map((sample) => sample.service_state_disk_bytes); + const databaseDisk = samples.map((sample) => sample.postgres_database_bytes); + const localDisk = samples.map((sample) => sample.local_node_state.bytes); + const spread = (values) => Math.max(...values) - Math.min(...values); + assert(spread(currentMemory) <= 128 * 1024 * 1024); + assert(spread(serviceDisk) <= 16 * 1024 * 1024); + assert(spread(databaseDisk) <= 32 * 1024 * 1024); + assert(spread(localDisk) <= 16 * 1024 * 1024); + assert.deepStrictEqual( + samples.at(-1).durable_rows, + samples[0].durable_rows, + "durable object counts grew during the parked-process soak" + ); + const aborted = runJson( + clusterflux, + ["process", "abort", ...scope, "--process", processId, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(aborted.abort_request.process_slot_released, true); + return { + executed: true, + process: processId, + duration_seconds: Math.floor((Date.now() - startedAt) / 1000), + sample_interval_seconds: 30, + sample_count: samples.length, + parked_main: { + main_state: parked.live_process.main_state, + main_wait_state: parked.live_process.main_wait_state, + }, + drained_prior_assignments: drainedPriorAssignments, + signed_node_poll_cycles: samples.length, + memory_spread_bytes: spread(currentMemory), + service_disk_spread_bytes: spread(serviceDisk), + postgres_disk_spread_bytes: spread(databaseDisk), + local_node_state_spread_bytes: spread(localDisk), + samples, + }; +} + +async function revokeSecurityNode({ + clusterflux, + projectDir, + scope, + securityNode, +}) { + const revoked = runJson( + clusterflux, + ["node", "revoke", ...scope, "--node", securityNode.node, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(revoked.command, "node revoke"); + const denied = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node: securityNode.node, + node_signature: signedNodeHeartbeat( + securityNode.node, + securityNode.identity, + { nonce: `strict-node-revoked-${Date.now()}` } + ), + }), + /revoked|unknown node|not recognized|not enrolled/i, + "revoked node credential" + ); + return { node: securityNode.node, denied }; +} + +async function runLiveQuotaPreallocation({ sessionSecret, suffix }) { + const readSpawnQuota = async () => { + const status = await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "quota_status" }) + ); + assert.strictEqual(status.type, "quota_status", JSON.stringify(status)); + const limit = status.limits?.limits?.Spawn; + const usage = status.usage?.Spawn; + const windowSeconds = status.window_seconds?.Spawn; + const windowStarted = status.window_started_epoch_seconds?.Spawn; + assert(Number.isInteger(limit) && limit > 0, JSON.stringify(status)); + assert(Number.isInteger(usage) && usage >= 0, JSON.stringify(status)); + assert( + Number.isInteger(windowSeconds) && windowSeconds > 0, + JSON.stringify(status) + ); + assert(Number.isInteger(windowStarted), JSON.stringify(status)); + return { limit, usage, windowSeconds, windowStarted }; + }; + + let spawnQuota = await readSpawnQuota(); + const windowElapsed = Math.max( + 0, + Math.floor(Date.now() / 1000) - spawnQuota.windowStarted + ); + if (spawnQuota.usage > 0 || windowElapsed > 1) { + await delay( + Math.max(1000, (spawnQuota.windowSeconds - windowElapsed + 1) * 1000) + ); + spawnQuota = await readSpawnQuota(); + } + let denial; + let deniedProcess; + let acceptedStarts = 0; + const maxAttempts = spawnQuota.limit - spawnQuota.usage + 2; + for (let index = 0; index < maxAttempts; index += 1) { + const processId = `vp-quota-${suffix}-${index}`; + const started = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "start_process", + process: processId, + restart: false, + }) + ); + if (started.type === "error") { + assert.match(started.message, /quota|spawn|limit|community tier/i); + denial = started; + deniedProcess = processId; + break; + } + assert.strictEqual(started.type, "process_started", JSON.stringify(started)); + acceptedStarts += 1; + const aborted = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "abort_process", + process: processId, + }) + ); + assert.strictEqual(aborted.type, "process_aborted", JSON.stringify(aborted)); + } + assert( + denial, + `live community-tier spawn quota was not reached in ${maxAttempts} attempts` + ); + const processes = await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "list_processes" }) + ); + assert.strictEqual( + processes.type, + "process_statuses", + JSON.stringify(processes) + ); + assert( + !JSON.stringify(processes).includes(deniedProcess), + "quota-denied process was allocated before denial" + ); + const second = readJson( + path.resolve(process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE) + ); + assert.notStrictEqual(second.session_secret, sessionSecret); + const unrelatedProcess = `vp-unrelated-quota-${suffix}`; + const unrelatedStarted = await sendHostedControl( + authenticatedRequest(second.session_secret, { + type: "start_process", + process: unrelatedProcess, + restart: false, + }) + ); + assert.strictEqual( + unrelatedStarted.type, + "process_started", + `first tenant quota exhaustion affected another tenant: ${JSON.stringify(unrelatedStarted)}` + ); + const unrelatedAborted = await sendHostedControl( + authenticatedRequest(second.session_secret, { + type: "abort_process", + process: unrelatedProcess, + }) + ); + assert.strictEqual( + unrelatedAborted.type, + "process_aborted", + JSON.stringify(unrelatedAborted) + ); + return { + limit: spawnQuota.limit, + initial_usage: spawnQuota.usage, + window_seconds: spawnQuota.windowSeconds, + accepted_starts_before_denial: acceptedStarts, + denied_process: deniedProcess, + denial: { + type: denial.type, + category: denial.error?.category || null, + message: denial.message, + }, + denied_process_allocated: false, + unrelated_tenant: { + tenant: second.tenant, + process: unrelatedProcess, + start: unrelatedStarted.type, + abort: unrelatedAborted.type, + unaffected_by_first_tenant_quota: true, + }, + }; +} + +async function runLongJoinProof({ clusterflux, projectDir, scope }) { + const startedAt = Date.now(); + const runReport = runJson( + clusterflux, + ["run", "long-join", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const terminal = await waitForCli( + "controlled task longer than two minutes", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (tasks) => + rawTaskEvents(tasks).some( + (event) => + event.task_definition === "long_join_probe" && + event.terminal_state === "completed" + ), + 5 * 60 * 1000 + ); + const completed = rawTaskEvents(terminal).find( + (event) => + event.task_definition === "long_join_probe" && + event.terminal_state === "completed" + ); + const durationSeconds = Math.floor((Date.now() - startedAt) / 1000); + assert(durationSeconds > 120); + const released = await waitForCli( + "long join process automatic slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + return { + process: runReport.process, + task: completed.task, + terminal_state: completed.terminal_state, + result: completed.result, + duration_seconds: durationSeconds, + process_state: released.state, + }; +} + +async function runRepeatedParkWakeProof({ clusterflux, projectDir, scope }) { + const runReport = runJson( + clusterflux, + ["run", "park-wake", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const terminal = await waitForCli( + "repeated coordinator-main park/wake completion", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (tasks) => { + const completed = new Set( + rawTaskEvents(tasks) + .filter( + (event) => + event.task_definition === "task_add_one" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ); + return completed.size >= 16; + }, + 120000 + ); + const completedTasks = [ + ...new Set( + rawTaskEvents(terminal) + .filter( + (event) => + event.task_definition === "task_add_one" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + const released = await waitForCli( + "park/wake process automatic slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + return { + process: runReport.process, + completed_cycles: completedTasks.length, + task_instances: completedTasks, + terminal_state: released.state, + }; +} + +async function runDroppedConnectionRollback({ + clusterflux, + projectDir, + scope, +}) { + const scenarioStartedAt = Date.now(); + const processId = "vp-current"; + const attempted = cp.spawnSync( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { + cwd: projectDir, + env: { + ...process.env, + CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION: "start_process", + }, + encoding: "utf8", + timeout: 5 * 60 * 1000, + } + ); + assert.notStrictEqual( + attempted.status, + 0, + `response-loss run unexpectedly succeeded: ${attempted.stdout}` + ); + assert.match( + `${attempted.stderr}\n${attempted.stdout}`, + /closed.*without a response|transport|response/i + ); + const released = await waitForCli( + "CLI launch guard rollback after lost start response", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 30000 + ); + return { + duration_ms: Date.now() - scenarioStartedAt, + process: processId, + failure_injection: "control transport dropped attempt-owned start_process response", + cli_exit_status: attempted.status, + rollback: "automatic CLI launch guard abort_process with matching launch_attempt", + terminal_state: released.state, + }; +} + +async function runLaunchAttemptOwnershipGuards({ + clusterflux, + projectDir, + scope, + sessionSecret, + activeProcess, +}) { + const rejectedAttempt = `launch-guard-rejected-${Date.now()}`; + const rejection = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "start_process", + process: `${activeProcess}-contender`, + launch_attempt: rejectedAttempt, + restart: false, + }) + ); + assert.strictEqual(rejection.type, "error"); + assert.match(rejection.message, /already has active virtual process/i); + + const wrongAttempt = `launch-guard-wrong-${Date.now()}`; + const deniedAbort = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "abort_process", + process: activeProcess, + launch_attempt: wrongAttempt, + }) + ); + assert.strictEqual(deniedAbort.type, "error"); + assert.match(deniedAbort.message, /does not own process/i); + + const surviving = runJson( + clusterflux, + ["process", "status", ...scope, "--process", activeProcess], + { cwd: projectDir } + ); + assert.notStrictEqual(surviving.state, "not_active"); + assert.strictEqual(surviving.process, activeProcess); + return { + rejected_attempt: rejectedAttempt, + rejection: rejection.message, + wrong_abort_attempt: wrongAttempt, + wrong_abort_denied: deniedAbort.message, + existing_process_survived: true, + }; +} + +async function runLiveBandwidthPreallocation({ + sessionSecret, + artifact, + maxBytes, +}) { + const scenarioStartedAt = Date.now(); + const before = relayDurableState(); + const partialLink = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_artifact_download_link", + artifact, + max_bytes: maxBytes, + ttl_seconds: 300, + }) + ); + assert.strictEqual( + partialLink.type, + "artifact_download_link", + JSON.stringify(partialLink) + ); + const partialToken = partialLink.link.scoped_token_digest; + let partialChunk; + const partialDeadline = Date.now() + 120_000; + while (Date.now() < partialDeadline) { + const response = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "open_artifact_download_stream", + artifact, + max_bytes: maxBytes, + token_digest: partialToken, + chunk_bytes: 16, + }) + ); + assert.strictEqual( + response.type, + "artifact_download_stream", + JSON.stringify(response) + ); + if (response.content_bytes_available === true) { + partialChunk = response; + break; + } + await delay(50); + } + assert(partialChunk, "artifact relay did not return the first partial chunk"); + assert.strictEqual(partialChunk.content_eof, false); + assert.strictEqual(partialChunk.streamed_bytes, 16); + const partialRevoked = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "revoke_artifact_download_link", + artifact, + token_digest: partialToken, + }) + ); + assert.strictEqual( + partialRevoked.type, + "artifact_download_link_revoked", + JSON.stringify(partialRevoked) + ); + const afterPartialAbandon = relayDurableState(); + assert(afterPartialAbandon.ingress_used > before.ingress_used); + assert(afterPartialAbandon.egress_used > before.egress_used); + assert( + afterPartialAbandon.abandoned_or_failed_used > + before.abandoned_or_failed_used + ); + + let denial; + const acceptedReservations = []; + for (let index = 0; index < 64; index += 1) { + const response = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_artifact_download_link", + artifact, + max_bytes: maxBytes, + ttl_seconds: 300, + }) + ); + if (response.type === "error") { + assert.match( + response.message, + /artifact relay.*(?:concurrency|period byte budget)|bandwidth|reservation/i + ); + denial = response; + break; + } + assert.strictEqual(response.type, "artifact_download_link", JSON.stringify(response)); + acceptedReservations.push(response.link.scoped_token_digest); + } + assert(denial, "artifact relay preallocation was not denied in 64 reservations"); + for (const tokenDigest of acceptedReservations) { + const revoked = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "revoke_artifact_download_link", + artifact, + token_digest: tokenDigest, + }) + ); + assert.strictEqual( + revoked.type, + "artifact_download_link_revoked", + JSON.stringify(revoked) + ); + } + const afterRelease = relayDurableState(); + assert.strictEqual( + Object.keys(afterRelease.reservations || {}).length, + Object.keys(before.reservations || {}).length + ); + return { + scenario_started_at_ms: scenarioStartedAt, + artifact, + accepted_reservations_before_denial: acceptedReservations.length, + bytes_served_before_denial: partialChunk.streamed_bytes, + partial_abandon: { + ingress_delta: afterPartialAbandon.ingress_used - before.ingress_used, + egress_delta: afterPartialAbandon.egress_used - before.egress_used, + abandoned_delta: + afterPartialAbandon.abandoned_or_failed_used - + before.abandoned_or_failed_used, + reservation_released: !(partialToken in afterRelease.reservations), + }, + durable_before: before, + durable_after_release: afterRelease, + denial: { + type: denial.type, + category: denial.error?.category || null, + message: denial.message, + }, + }; +} + +function relayDurableState() { + if (!strictVpsRestart) { + throw new Error("strict relay accounting requires VPS Postgres access"); + } + const output = run( + "ssh", + sshArgs( + "sudo", + "-u", + "postgres", + "psql", + "-d", + "clusterflux", + "-At", + "-c", + "SELECT record::text FROM clusterflux_artifact_relay_state WHERE singleton = TRUE" + ) + ).trim(); + assert.notStrictEqual(output, "", "artifact relay durable state was not persisted"); + return JSON.parse(output.split(/\r?\n/).filter(Boolean).at(-1)); +} + +async function waitForHostedControlReady(timeoutMs = 60_000) { + const deadline = Date.now() + timeoutMs; + let lastError; + while (Date.now() < deadline) { + try { + const ping = await sendHostedControl({ type: "ping" }); + if (ping.type === "pong") return ping; + lastError = new Error(JSON.stringify(ping)); + } catch (error) { + lastError = error; + } + await delay(500); + } + throw new Error(`hosted service did not recover: ${lastError}`); +} + +async function publishRelayProbeArtifact({ clusterflux, projectDir, scope, label }) { + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + await waitForCli( + `${label} flagship completion`, + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (report) => completedFlagship(rawTaskEvents(report)), + 10 * 60 * 1000 + ); + const artifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ); + const artifact = artifacts.artifacts.find((candidate) => { + const digest = candidate.digest; + return ( + typeof digest === "string" && + /^sha256:[0-9a-f]{64}$/.test(digest) && + candidate.artifact === `release.tar-${digest.slice("sha256:".length)}` + ); + }); + assert(artifact, `${label} did not publish a relay probe artifact`); + return artifact; +} + +async function runRelayEmergencyDisable({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + sessionSecret, + maxBytes, +}) { + if (!strictVpsRestart) { + throw new Error("strict relay disable proof requires VPS systemd access"); + } + const dropInDir = `/run/systemd/system/${strictServiceUnit}.d`; + const dropInPath = `${dropInDir}/90-strict-relay-disable.conf`; + run("ssh", sshArgs("sudo", "mkdir", "-p", dropInDir)); + cp.execFileSync("ssh", sshArgs("sudo", "tee", dropInPath), { + cwd: repo, + input: "[Service]\nEnvironment=CLUSTERFLUX_HOSTED_RELAY_DISABLED=true\n", + stdio: ["pipe", "ignore", "inherit"], + }); + let disabled; + let disabledArtifact; + let disabledWorker; + try { + run( + "ssh", + sshArgs( + "sudo", + "systemctl", + "daemon-reload" + ) + ); + run( + "ssh", + sshArgs("sudo", "systemctl", "restart", strictServiceUnit) + ); + await waitForHostedControlReady(); + disabledWorker = spawnJsonLines(clusterfluxNode, workerArgs, { + cwd: projectDir, + env: { ...process.env }, + }); + await disabledWorker.waitFor( + (value) => value.node_status === "ready", + "relay-disabled worker ready" + ); + disabledArtifact = await publishRelayProbeArtifact({ + clusterflux, + projectDir, + scope, + label: "relay-disabled", + }); + disabled = assertDenied( + await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_artifact_download_link", + artifact: disabledArtifact.artifact, + max_bytes: maxBytes, + ttl_seconds: 60, + }) + ), + /artifact relay is disabled/i, + "emergency-disabled artifact relay" + ); + } finally { + if (disabledWorker) await stopChild(disabledWorker.child); + run("ssh", sshArgs("sudo", "rm", "-f", dropInPath)); + run("ssh", sshArgs("sudo", "systemctl", "daemon-reload")); + run( + "ssh", + sshArgs("sudo", "systemctl", "restart", strictServiceUnit) + ); + await waitForHostedControlReady(); + } + const restoredWorker = spawnJsonLines(clusterfluxNode, workerArgs, { + cwd: projectDir, + env: { ...process.env }, + }); + try { + await restoredWorker.waitFor( + (value) => value.node_status === "ready", + "relay-restored worker ready" + ); + const restoredArtifact = await publishRelayProbeArtifact({ + clusterflux, + projectDir, + scope, + label: "relay-restored", + }); + const restored = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_artifact_download_link", + artifact: restoredArtifact.artifact, + max_bytes: maxBytes, + ttl_seconds: 60, + }) + ); + assert.strictEqual(restored.type, "artifact_download_link", JSON.stringify(restored)); + const revoked = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "revoke_artifact_download_link", + artifact: restoredArtifact.artifact, + token_digest: restored.link.scoped_token_digest, + }) + ); + assert.strictEqual(revoked.type, "artifact_download_link_revoked"); + return { + evidence: { + disabled, + disabled_probe_artifact: disabledArtifact.artifact, + restored: restored.type, + restored_probe_artifact: restoredArtifact.artifact, + restored_reservation_released: revoked.type, + runtime_drop_in_removed: true, + }, + worker: restoredWorker, + }; + } catch (error) { + await stopChild(restoredWorker.child); + throw error; + } +} + +function deploymentProvenance() { + if (!strictVpsRestart) return null; + const systemGeneration = run( + "ssh", + sshArgs("readlink", "-f", "/run/current-system") + ).trim(); + const mainPid = run( + "ssh", + sshArgs( + "systemctl", + "show", + strictServiceUnit, + "--property=MainPID", + "--value" + ) + ).trim(); + assert.match(mainPid, /^[1-9][0-9]*$/, "hosted service has no live MainPID"); + const executable = run( + "ssh", + sshArgs("readlink", "-f", `/proc/${mainPid}/exe`) + ).trim(); + const binarySha256 = run( + "ssh", + sshArgs("sha256sum", `/proc/${mainPid}/exe`) + ) + .trim() + .split(/\s+/)[0]; + const fragment = run( + "ssh", + sshArgs("systemctl", "show", strictServiceUnit, "--property=FragmentPath", "--value") + ).trim(); + const serviceUnitSha256 = run( + "ssh", + sshArgs("sha256sum", fragment) + ) + .trim() + .split(/\s+/)[0]; + const renderedServiceConfiguration = run( + "ssh", + sshArgs("systemctl", "cat", strictServiceUnit) + ); + const renderedServiceConfigurationSha256 = sha256( + Buffer.from(renderedServiceConfiguration) + ); + return { + system_generation: systemGeneration, + hosted_service_executable: executable, + hosted_service_sha256: `sha256:${binarySha256}`, + service_unit: fragment, + service_unit_sha256: `sha256:${serviceUnitSha256}`, + service_configuration_sha256: renderedServiceConfigurationSha256, + }; +} + +function configurationProvenance() { + const configRepo = path.resolve( + process.env.CLUSTERFLUX_DEPLOYMENT_CONFIG_REPO || + path.join(repo, "..", "michelpaulissen.com") + ); + const status = run("git", ["status", "--short"], { cwd: configRepo }).trim(); + const revision = run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(); + return { + repository: configRepo, + revision, + evidence_identity: sha256(Buffer.from(revision)), + clean: status === "", + status: status || null, + }; +} + +function proxyConfigurationProvenance() { + if (!strictVpsRestart) return null; + const proxyUnit = process.env.CLUSTERFLUX_STRICT_PROXY_UNIT || "nginx.service"; + const fragment = run( + "ssh", + sshArgs("systemctl", "show", "--property=FragmentPath", "--value", proxyUnit) + ).trim(); + assert(fragment, `proxy service ${proxyUnit} has no FragmentPath`); + const unitSha256 = run( + "ssh", + sshArgs("sha256sum", fragment) + ).trim().split(/\s+/)[0]; + const proxyExecStart = run( + "ssh", + sshArgs("systemctl", "show", "--property=ExecStart", "--value", proxyUnit) + ).trim(); + const nginxExecutable = proxyExecStart.match(/path=([^ ;]+)/)?.[1]; + const nginxConfiguration = proxyExecStart.match(/ argv\[\]=.* -c ([^ ;]+)/)?.[1]; + assert(nginxExecutable, `proxy service ${proxyUnit} has no executable identity`); + assert(nginxConfiguration, `proxy service ${proxyUnit} has no configuration identity`); + const renderedConfiguration = run( + "ssh", + sshArgs(nginxExecutable, "-T", "-c", nginxConfiguration) + ); + const renderedConfigurationIdentity = sha256(Buffer.from(renderedConfiguration)); + const activeState = run( + "ssh", + sshArgs("systemctl", "is-active", proxyUnit) + ).trim(); + assert.strictEqual(activeState, "active", `proxy service ${proxyUnit} is not active`); + return { + proxy_unit: proxyUnit, + proxy_unit_fragment: fragment, + proxy_unit_sha256: `sha256:${unitSha256}`, + proxy_executable: nginxExecutable, + proxy_configuration: nginxConfiguration, + rendered_configuration_sha256: renderedConfigurationIdentity, + evidence_identity: renderedConfigurationIdentity, + active_state: activeState, + }; +} + +async function restartHostedServiceAndResume({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + workerNode, + credentialPath, + credentialDigest, +}) { + if (!strictVpsRestart) { + return { + executed: false, + reason: "strict VPS restart access not supplied", + worker: null, + }; + } + const before = deploymentProvenance(); + run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); + const deadline = Date.now() + 120000; + let ping; + let lastError; + while (Date.now() < deadline) { + try { + ping = await sendHostedControl({ type: "ping" }); + if (ping.type === "pong") break; + } catch (error) { + lastError = error; + } + await delay(500); + } + assert.strictEqual( + ping?.type, + "pong", + `hosted service did not recover after restart: ${lastError?.message || "no pong"}` + ); + const afterFirstRestart = deploymentProvenance(); + assert.strictEqual(afterFirstRestart.system_generation, before.system_generation); + assert.strictEqual( + afterFirstRestart.hosted_service_sha256, + before.hosted_service_sha256 + ); + + const projects = runJson(clusterflux, ["project", "list", ...scope], { + cwd: projectDir, + }); + assert(projects.project_count >= 1, "CLI session/project did not survive restart"); + const ephemeralRun = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(ephemeralRun.status, "main_launched"); + const ephemeralProcess = ephemeralRun.process; + await waitForCli( + "pre-restart ephemeral main to park", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", ephemeralProcess], + { cwd: projectDir } + ), + (status) => status.live_process?.main_wait_state === "waiting_for_node", + 120000 + ); + + run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); + const secondDeadline = Date.now() + 120000; + ping = undefined; + lastError = undefined; + while (Date.now() < secondDeadline) { + try { + ping = await sendHostedControl({ type: "ping" }); + if (ping.type === "pong") break; + } catch (error) { + lastError = error; + } + await delay(500); + } + assert.strictEqual( + ping?.type, + "pong", + `hosted service did not recover after ephemeral-state restart: ${ + lastError?.message || "no pong" + }` + ); + const after = deploymentProvenance(); + assert.strictEqual(after.system_generation, before.system_generation); + assert.strictEqual(after.hosted_service_sha256, before.hosted_service_sha256); + const oldStatus = runJson( + clusterflux, + ["process", "status", ...scope, "--process", ephemeralProcess], + { cwd: projectDir } + ); + assert.strictEqual(oldStatus.state, "not_active"); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const worker = spawnJsonLines(clusterfluxNode, workerArgs, { + cwd: projectDir, + env: { ...process.env }, + }); + const ready = await worker.waitFor( + (value) => value.node_status === "ready", + "persisted worker ready after hosted service restart" + ); + assert.strictEqual(ready.node, workerNode); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const restartedRun = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(restartedRun.status, "main_launched"); + const restartedProcess = restartedRun.process; + const completed = await waitForCli( + "new process completion after hosted service restart", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", restartedProcess], + { cwd: projectDir } + ), + (tasks) => completedFlagship(rawTaskEvents(tasks)), + 5 * 60 * 1000 + ); + const completedEvent = rawTaskEvents(completed).find( + (event) => + event.executor === "coordinator_main" && + event.terminal_state === "completed" + ); + assert(completedEvent, "post-restart flagship omitted its completed main event"); + return { + executed: true, + account_project_session_preserved: true, + node_identity_preserved: true, + old_process_terminated_honestly: true, + terminated_ephemeral_process: ephemeralProcess, + new_process: restartedProcess, + new_process_result: completedEvent.result, + before, + after_first_restart: afterFirstRestart, + after, + worker, + }; +} + +async function finishUserCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, +}) { + const forged = assertDenied( + await sendHostedControl( + authenticatedRequest( + `forged-session-${crypto.randomBytes(24).toString("hex")}`, + { type: "list_projects" } + ) + ), + /not recognized|invalid|authentication|session/i, + "forged user session" + ); + + let expired = null; + const expiredFile = process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE; + if (expiredFile) { + const oldSession = readJson(path.resolve(expiredFile)); + assert(oldSession.session_secret, "expired session evidence omitted its secret"); + expired = assertDenied( + await sendHostedControl( + authenticatedRequest(oldSession.session_secret, { type: "list_projects" }) + ), + /expired|not recognized/i, + "expired user session" + ); + } else if (strictFullRelease) { + throw new Error( + "strict full release requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE for a genuinely expired hosted session" + ); + } + + const logout = runJson(clusterflux, ["logout", ...scope, "--yes"], { + cwd: projectDir, + }); + assert.strictEqual(logout.server_session_revocation.revoked, true); + const revoked = assertDenied( + await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "list_projects" }) + ), + /revoked|not recognized/i, + "revoked user session" + ); + return { forged, expired, revoked }; +} + +async function dapVariables(client, variablesReference) { + const request = client.send("variables", { variablesReference }); + return (await client.response(request, "variables")).body.variables; +} + +async function runSameDefinitionDapIdentity({ + clusterfluxDap, + clusterflux, + projectDir, + scope, +}) { + const scenarioStartedAt = Date.now(); + const baselineContainers = new Set( + run("podman", ["ps", "-q"]) + .trim() + .split(/\s+/) + .filter(Boolean) + ); + const client = new DapClient({ + cwd: projectDir, + command: clusterfluxDap, + args: [], + env: { + ...process.env, + CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1", + }, + }); + let disconnected = false; + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + const launch = client.send("launch", { + entry: "identity", + project: projectDir, + runtimeBackend: "live-services", + coordinatorEndpoint: serviceEndpoint, + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + + const mainThreadDeadline = Date.now() + 5 * 60 * 1000; + let mainThread; + while (Date.now() < mainThreadDeadline) { + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body + .threads; + mainThread = threads.find((thread) => /coordinator main/i.test(thread.name)); + if (mainThread) break; + await delay(100); + } + assert(mainThread, "identity entry did not report its coordinator main"); + const processRequest = client.send("evaluate", { + expression: "virtual_process_id", + context: "watch", + }); + const processId = (await client.response(processRequest, "evaluate")).body + .result; + const identityContainerDeadline = Date.now() + 2 * 60 * 1000; + let identityContainerIds = new Set(); + let maximumNewContainerCount = 0; + let lastPodmanStates = []; + while (Date.now() < identityContainerDeadline) { + const containers = runJson("podman", ["ps", "--format", "json"]); + lastPodmanStates = containers; + identityContainerIds = new Set( + containers + .map((container) => container.Id || container.ID || container.IdHex || "") + .filter((id) => id && !baselineContainers.has(id)) + ); + maximumNewContainerCount = Math.max( + maximumNewContainerCount, + identityContainerIds.size + ); + if (identityContainerIds.size >= 2) break; + await delay(100); + } + const identityTaskSnapshot = + identityContainerIds.size >= 2 + ? null + : runJson( + clusterflux, + ["task", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert( + identityContainerIds.size >= 2, + `identity entry did not start two same-definition Podman containers; max=${maximumNewContainerCount} tasks=${JSON.stringify( + identityTaskSnapshot + )} podman=${JSON.stringify(lastPodmanStates)}` + ); + + const pause = client.send("pause", { threadId: mainThread.id }); + await client.response(pause, "pause"); + const paused = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "pause", + 30_000 + ); + assert.strictEqual(typeof paused.body.allThreadsStopped, "boolean"); + + const podmanStates = runJson("podman", ["ps", "--all", "--format", "json"]); + const clusterfluxPausedContainers = podmanStates.filter((container) => { + const id = container.Id || container.ID || container.IdHex || ""; + const state = String(container.State || container.Status || "").toLowerCase(); + return identityContainerIds.has(id) && state.includes("paused"); + }); + assert( + clusterfluxPausedContainers.length >= 2, + `expected two newly paused Clusterflux containers: ${JSON.stringify( + podmanStates + )}` + ); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body + .threads; + const identityThreads = threads.filter((thread) => + /identity probe/i.test(thread.name) + ); + assert.strictEqual( + identityThreads.length, + 2, + `expected two same-definition DAP threads: ${JSON.stringify(threads)}` + ); + assert.notStrictEqual(identityThreads[0].id, identityThreads[1].id); + + const threadEvidence = []; + for (const threadRecord of identityThreads) { + const stackRequest = client.send("stackTrace", { + threadId: threadRecord.id, + startFrame: 0, + levels: 1, + }); + const frames = (await client.response(stackRequest, "stackTrace")).body + .stackFrames; + assert.strictEqual(frames.length, 1); + const scopesRequest = client.send("scopes", { frameId: frames[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const argsScope = scopes.find( + (scopeRecord) => scopeRecord.name === "Task Args and Handles" + ); + const runtimeScope = scopes.find( + (scopeRecord) => scopeRecord.name === "Clusterflux Runtime" + ); + assert(argsScope && runtimeScope); + const args = await dapVariables(client, argsScope.variablesReference); + const runtime = await dapVariables(client, runtimeScope.variablesReference); + const runtimeValue = (name) => + runtime.find((variable) => variable.name === name)?.value; + assert.strictEqual(runtimeValue("state"), "Frozen"); + threadEvidence.push({ + thread_id: threadRecord.id, + task_instance: runtimeValue("virtual_thread"), + attempt_id: runtimeValue("task_attempt_id"), + arguments: args.map((variable) => ({ + name: variable.name, + value: variable.value, + })), + }); + } + assert.notStrictEqual( + threadEvidence[0].task_instance, + threadEvidence[1].task_instance + ); + assert.notStrictEqual(threadEvidence[0].attempt_id, threadEvidence[1].attempt_id); + const argumentText = threadEvidence + .flatMap((record) => record.arguments.map((argument) => argument.value)) + .join(" "); + assert.match(argumentText, /slow-first/); + assert.match(argumentText, /fast-second/); + + const continued = client.send("continue", { + threadId: paused.body.threadId, + }); + await client.response(continued, "continue"); + await client.waitFor( + (message) => message.type === "event" && message.event === "terminated", + 5 * 60 * 1000 + ); + + const taskList = runJson( + clusterflux, + ["task", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + const identityEvents = rawTaskEvents(taskList).filter( + (event) => + event.task_definition === "identity_probe" && + event.terminal_state === "completed" + ); + assert.strictEqual(identityEvents.length, 2, JSON.stringify(identityEvents)); + assert.notStrictEqual(identityEvents[0].task, identityEvents[1].task); + assert.notStrictEqual(identityEvents[0].attempt_id, identityEvents[1].attempt_id); + assert.match(JSON.stringify(identityEvents[0].result), /fast-second/); + assert.match(JSON.stringify(identityEvents[1].result), /slow-first/); + + const released = await waitForCli( + "same-definition identity process release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + await client.close(); + disconnected = true; + return { + duration_ms: Date.now() - scenarioStartedAt, + process: processId, + thread_evidence: threadEvidence, + completion_order: identityEvents.map((event) => ({ + task_instance: event.task, + attempt_id: event.attempt_id, + result: event.result, + })), + podman_paused_container_ids: clusterfluxPausedContainers.map( + (container) => container.Id || container.ID || container.IdHex + ), + dap_all_threads_stopped: paused.body.allThreadsStopped, + terminal_state: released.state, + }; + } finally { + if (!disconnected) { + await client.close().catch(() => { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + }); + } + } +} + +async function runLiveDapEditRestart({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + worker, +}) { + const scenarioStartedAt = Date.now(); + const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs")); + const originalSource = fs.readFileSync(sourcePath, "utf8"); + const sourceLines = originalSource.split(/\r?\n/); + const lineFor = (needle) => { + const line = sourceLines.findIndex((candidate) => candidate.includes(needle)) + 1; + assert(line > 0, `flagship source omitted ${needle}`); + return line; + }; + const mainLine = lineFor("pub async fn restart_main()"); + const taskLine = lineFor("fn task_trap("); + const client = new DapClient({ + cwd: projectDir, + command: clusterfluxDap, + args: [], + env: { + ...process.env, + CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS: "1", + CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE: "1", + }, + }); + let sourceRestored = false; + let disconnected = false; + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "restart", + project: projectDir, + runtimeBackend: "live-services", + coordinatorEndpoint: serviceEndpoint, + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const configurationStartedAt = Date.now(); + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const configurationResponseMs = Date.now() - configurationStartedAt; + assert( + configurationResponseMs < 2_000, + `configurationDone blocked for ${configurationResponseMs} ms` + ); + + const threadDeadline = Date.now() + 5 * 60 * 1000; + let runningThreads = []; + while (Date.now() < threadDeadline) { + const threadsRequest = client.send("threads"); + runningThreads = (await client.response(threadsRequest, "threads")).body + .threads; + if (runningThreads.length > 0) break; + await delay(100); + } + assert(runningThreads.length > 0, "asynchronous live DAP launch reported no threads"); + const pauseStartedAt = Date.now(); + const pause = client.send("pause", { threadId: runningThreads[0].id }); + await client.response(pause, "pause"); + const pauseResponseMs = Date.now() - pauseStartedAt; + assert(pauseResponseMs < 2_000, `pause blocked for ${pauseResponseMs} ms`); + const mainStop = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "pause" + ); + assert.strictEqual(mainStop.body.allThreadsStopped, true); + + const inspectStartedAt = Date.now(); + const commandStatus = client.send("evaluate", { + expression: "command_status", + context: "watch", + }); + const inspected = await client.response(commandStatus, "evaluate"); + const inspectionResponseMs = Date.now() - inspectStartedAt; + assert( + inspectionResponseMs < 2_000, + `inspection blocked for ${inspectionResponseMs} ms` + ); + assert.match(inspected.body.result, /debug epoch|frozen/i); + + const breakpointStartedAt = Date.now(); + const setBreakpoints = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: taskLine }], + }); + const breakpointResponse = await client.response(setBreakpoints, "setBreakpoints"); + const breakpointResponseMs = Date.now() - breakpointStartedAt; + assert( + breakpointResponseMs < 2_000, + `setBreakpoints blocked for ${breakpointResponseMs} ms` + ); + assert.deepStrictEqual( + breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), + [false] + ); + assert.deepStrictEqual( + breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.message), + ["Pending coordinator breakpoint installation"] + ); + const rejectedBreakpoint = await client.waitFor( + (message) => + message.type === "event" && + message.event === "breakpoint" && + message.body.reason === "changed" && + message.body.breakpoint?.verified === false && + message.body.breakpoint?.line === taskLine && + /coordinator breakpoint installation failed/i.test( + message.body.breakpoint?.message || "" + ), + 30_000 + ); + assert.strictEqual( + rejectedBreakpoint.body.breakpoint.id, + breakpointResponse.body.breakpoints[0].id + ); + const retryBreakpoints = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: taskLine }], + }); + const retryBreakpointResponse = await client.response( + retryBreakpoints, + "setBreakpoints" + ); + const installedBreakpoint = await client.waitFor( + (message) => + message.type === "event" && + message.event === "breakpoint" && + message.body.reason === "changed" && + message.body.breakpoint?.verified === true && + message.body.breakpoint?.line === taskLine, + 30_000 + ); + assert.strictEqual( + installedBreakpoint.body.breakpoint.id, + retryBreakpointResponse.body.breakpoints[0].id + ); + const breakpointInstallationMs = Date.now() - breakpointStartedAt; + + const continueStartedAt = Date.now(); + const mainContinue = client.send("continue", { + threadId: mainStop.body.threadId, + }); + await client.response(mainContinue, "continue"); + const continueResponseMs = Date.now() - continueStartedAt; + assert( + continueResponseMs < 2_000, + `continue blocked for ${continueResponseMs} ms` + ); + const taskStop = await client.waitFor( + (message) => + message.seq > mainStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(taskStop.body.allThreadsStopped, true); + const observerReconnect = await client.waitFor( + (message) => + message.type === "event" && + message.event === "output" && + String(message.body?.output || "").includes( + "injected one transient connection loss" + ), + 30_000 + ); + assert( + observerReconnect.seq < taskStop.seq, + "observer reconnection must complete before the real breakpoint stop" + ); + assert.strictEqual( + client.messages.filter( + (message) => + message.seq > mainStop.seq && + message.seq < taskStop.seq && + message.type === "event" && + message.event === "stopped" + ).length, + 0, + "observer reconnection fabricated a stopped target before the real breakpoint" + ); + assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId); + + const stackTrace = client.send("stackTrace", { + threadId: taskStop.body.threadId, + startFrame: 0, + levels: 1, + }); + const taskFrames = (await client.response(stackTrace, "stackTrace")).body + .stackFrames; + assert.strictEqual(taskFrames[0].line, taskLine); + + const scopesRequest = client.send("scopes", { frameId: taskFrames[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const argsScope = scopes.find( + (candidate) => candidate.name === "Task Args and Handles" + ); + const runtimeScope = scopes.find( + (candidate) => candidate.name === "Clusterflux Runtime" + ); + assert(argsScope && runtimeScope); + const taskArgs = await dapVariables(client, argsScope.variablesReference); + assert( + taskArgs.some( + (variable) => + variable.name === "arg_0" && String(variable.value).includes("0") + ) + ); + const runtime = await dapVariables(client, runtimeScope.variablesReference); + assert( + runtime.some( + (variable) => + variable.name === "runtime_backend" && variable.value === "LiveServices" + ) + ); + assert( + runtime.some( + (variable) => variable.name === "state" && variable.value === "Frozen" + ) + ); + const dapProcess = runtime.find( + (variable) => variable.name === "virtual_process_id" + )?.value; + assert.match(dapProcess || "", /^vp-[0-9a-f]{12}$/); + + const taskContinue = client.send("continue", { threadId: taskStop.body.threadId }); + await client.response(taskContinue, "continue"); + const failedStop = await client.waitFor( + (message) => + message.seq > taskStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "exception", + 5 * 60 * 1000 + ); + assert.strictEqual(failedStop.body.allThreadsStopped, false); + + const beforeEdit = runJson( + clusterflux, + ["task", "list", ...scope, "--process", dapProcess], + { cwd: projectDir } + ); + const originalTaskEvent = rawTaskEvents(beforeEdit).find( + (event) => + event.task_definition === "task_trap" && + event.terminal_state === "failed" + ); + assert(originalTaskEvent, "DAP restart entry omitted its original child event"); + assert(originalTaskEvent.attempt_id, "failed attempt omitted its identity"); + + const originalTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { + #[cfg(target_arch = "wasm32")] + core::arch::wasm32::unreachable(); + #[cfg(not(target_arch = "wasm32"))] + panic!("intentional task trap") +}`; + const replacementTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { + _input + 42 // strict hosted compatible restart probe +}`; + const editedSource = originalSource.replace(originalTaskBody, replacementTaskBody); + assert.notStrictEqual(editedSource, originalSource); + fs.writeFileSync(sourcePath, editedSource); + + const restartFrame = client.send("restartFrame", { + frameId: taskFrames[0].id, + }); + const restartResponse = await client.response(restartFrame, "restartFrame"); + const restartedStop = await client.waitFor( + (message) => + message.seq > restartResponse.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(restartedStop.body.allThreadsStopped, true); + const restartedContinue = client.send("continue", { + threadId: restartedStop.body.threadId, + }); + await client.response(restartedContinue, "continue"); + const restartedNodeReport = await worker.waitFor( + (value) => + value.node_status === "completed" && + value.virtual_thread === originalTaskEvent.task && + value.task_assignment_response?.task_spec?.task_definition === + "task_trap", + "edited task to execute on the live restarted node", + 5 * 60 * 1000 + ); + assert.strictEqual( + restartedNodeReport.node_status, + "completed", + `edited task restart failed on the live node: ${JSON.stringify(restartedNodeReport)}` + ); + assert.match(restartedNodeReport.stdout_tail, /"SmallJson":42/); + const restartedTask = restartedNodeReport.virtual_thread; + + const afterEdit = await waitForCli( + "edited live task event", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", dapProcess], { + cwd: projectDir, + }), + (tasks) => + rawTaskEvents(tasks).some( + (event) => + event.task === restartedTask && + event.terminal_state === "completed" && + JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) + ), + 5 * 60 * 1000 + ); + const restartedEvent = rawTaskEvents(afterEdit).find( + (event) => + event.task === restartedTask && + event.terminal_state === "completed" && + JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) + ); + assert(restartedEvent, "replacement attempt omitted its completed event"); + assert(restartedEvent.attempt_id, "replacement attempt omitted its identity"); + assert.notStrictEqual(restartedEvent.attempt_id, originalTaskEvent.attempt_id); + await client.waitFor( + (message) => message.type === "event" && message.event === "terminated", + 5 * 60 * 1000 + ); + const disconnectStartedAt = Date.now(); + await client.close(); + disconnected = true; + const disconnectResponseMs = Date.now() - disconnectStartedAt; + assert( + disconnectResponseMs < 2_000, + `disconnect blocked for ${disconnectResponseMs} ms` + ); + fs.writeFileSync(sourcePath, originalSource); + sourceRestored = true; + + return { + duration_ms: Date.now() - scenarioStartedAt, + process: dapProcess, + main_source_line: mainLine, + task_breakpoint_line: taskLine, + launch_started_without_breakpoint: true, + configuration_response_ms: configurationResponseMs, + pause_response_ms: pauseResponseMs, + inspection_response_ms: inspectionResponseMs, + breakpoint_response_ms: breakpointResponseMs, + breakpoint_installation_ms: breakpointInstallationMs, + continue_response_ms: continueResponseMs, + disconnect_response_ms: disconnectResponseMs, + pause_all_threads_stopped: mainStop.body.allThreadsStopped, + task_all_threads_stopped: taskStop.body.allThreadsStopped, + original_task_instance: originalTaskEvent.task, + original_attempt: originalTaskEvent.attempt_id, + restarted_task_instance: restartedTask, + restarted_attempt: restartedEvent.attempt_id, + restarted_result: restartedEvent.result, + compatible_source_edit: "task_trap: trap -> input + 42", + original_arguments_preserved: true, + clean_vfs_boundary_used: true, + breakpoint_install_failure_reported: true, + observer_idle_request_rate_bound_per_second: 0.8, + observer_reconnected_without_false_stop: true, + }; + } finally { + if (!sourceRestored) fs.writeFileSync(sourcePath, originalSource); + if (!disconnected) { + await client.close().catch(() => { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + }); + } + } +} + +function copyProjectControlState(sourceRoot, targetRoot) { + const source = path.join(sourceRoot, ".clusterflux"); + const target = path.join(targetRoot, ".clusterflux"); + ensureDir(target); + for (const file of ["session.json", "project.json"]) { + const from = path.join(source, file); + assert(fs.existsSync(from), `missing shared project control state ${from}`); + fs.copyFileSync(from, path.join(target, file)); + } + fs.chmodSync(path.join(target, "session.json"), 0o600); + const sourceNodes = path.join(source, "nodes"); + assert(fs.existsSync(sourceNodes), "persisted node credential directory is missing"); + fs.cpSync(sourceNodes, path.join(target, "nodes"), { + recursive: true, + force: true, + }); +} + +async function runHostedHelloBuild({ + clusterflux, + projectDir, + scope, + outputFile, +}) { + const startedAt = Date.now(); + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const tasks = await waitForCli( + "hosted hello-build completion", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (report) => { + const events = rawTaskEvents(report); + return events.some( + (event) => + event.task_definition === "compile" && + event.terminal_state === "completed" + ); + }, + 10 * 60 * 1000 + ); + const events = rawTaskEvents(tasks); + assert( + events.some( + (event) => + event.task_definition === "snapshot_current_project" && + event.terminal_state === "completed" + ) + ); + const released = await waitForCli( + "hosted hello-build process release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + const artifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ); + const artifact = artifacts.artifacts.find( + (candidate) => + typeof candidate.digest === "string" && + /^sha256:[0-9a-f]{64}$/.test(candidate.digest) && + candidate.artifact.startsWith("hello-clusterflux-") + ); + assert(artifact, "hello-build did not retain its executable artifact"); + await waitForCli( + "hosted hello-build retaining node online", + () => runJson(clusterflux, ["node", "list", ...scope], { cwd: projectDir }), + (report) => + report.response?.descriptors?.some( + (descriptor) => + descriptor.online === true && + descriptor.artifact_locations?.includes(artifact.artifact) + ), + 120_000 + ); + const download = runJson( + clusterflux, + [ + "artifact", + "download", + ...scope, + artifact.artifact, + "--to", + outputFile, + ], + { cwd: projectDir } + ); + assert.strictEqual(download.local_download.verified_digest, artifact.digest); + assert.strictEqual(sha256(fs.readFileSync(outputFile)), artifact.digest); + fs.chmodSync(outputFile, 0o755); + const output = run(outputFile, []).trim(); + assert.strictEqual(output, "hello from a real Clusterflux build"); + return { + duration_ms: Date.now() - startedAt, + process: runReport.process, + terminal_state: released.state, + artifact: artifact.artifact, + digest: artifact.digest, + executable_output: output, + }; +} + +async function runHostedRecoveryBuild({ + clusterfluxDap, + clusterflux, + projectDir, + scope, +}) { + const startedAt = Date.now(); + const sourcePath = fs.realpathSync(path.join(projectDir, "src/lib.rs")); + const original = fs.readFileSync(sourcePath, "utf8"); + const failing = '"exit 23".to_owned()'; + const replacement = + '"printf \'recovered\\n\' > /clusterflux/output/recovering.txt".to_owned()'; + assert(original.includes(failing)); + const client = new DapClient({ + cwd: projectDir, + command: clusterfluxDap, + env: { ...process.env, CLUSTERFLUX_DAP_TIMEOUT_MS: "120000" }, + }); + let restored = false; + let closed = false; + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + const launch = client.send("launch", { + entry: "build", + project: projectDir, + runtimeBackend: "live-services", + coordinatorEndpoint: serviceEndpoint, + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + const configured = client.send("configurationDone"); + await client.response(configured, "configurationDone"); + const failedStop = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "exception", + 10 * 60 * 1000 + ); + assert.strictEqual(failedStop.body.allThreadsStopped, false); + const threadRequest = client.send("threads"); + const threads = (await client.response(threadRequest, "threads")).body.threads; + const recoveringThread = threads.find((thread) => /build lane/.test(thread.name)); + assert(recoveringThread, "recovery-build failed lane is not visible in DAP"); + const processId = threads + .map((thread) => /vp-[0-9a-f]{12}/.exec(thread.name)?.[0]) + .find(Boolean); + assert(processId, "recovery-build DAP threads omitted the process identity"); + const before = runJson( + clusterflux, + ["task", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + const beforeEvents = rawTaskEvents(before); + const originalFailure = beforeEvents.find( + (event) => + event.task_definition === "build_lane" && + event.terminal_state === "failed" + ); + assert(originalFailure?.attempt_id); + assert( + beforeEvents.some( + (event) => + event.task_definition === "build_lane" && + event.task !== originalFailure.task && + event.terminal_state === "completed" + ) + ); + const waiting = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.strictEqual(waiting.live_process.main_state, "running"); + fs.writeFileSync(sourcePath, original.replace(failing, replacement)); + const restart = client.send("restartFrame", { + threadId: recoveringThread.id, + }); + await client.response(restart, "restartFrame"); + await client.waitFor( + (message) => message.type === "event" && message.event === "terminated", + 10 * 60 * 1000 + ); + assert( + client.messages.some( + (message) => + message.type === "event" && + message.event === "thread" && + message.body.reason === "exited" && + message.body.threadId === recoveringThread.id + ) + ); + const after = await waitForCli( + "hosted recovery replacement event", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", processId], + { cwd: projectDir } + ), + (report) => + rawTaskEvents(report).some( + (event) => + event.task === originalFailure.task && + event.terminal_state === "completed" + ), + 120_000 + ); + const replacementEvent = rawTaskEvents(after).find( + (event) => + event.task === originalFailure.task && + event.terminal_state === "completed" + ); + assert(replacementEvent?.attempt_id); + assert.notStrictEqual(replacementEvent.attempt_id, originalFailure.attempt_id); + assert( + rawTaskEvents(after).some( + (event) => + event.executor === "coordinator_main" && + event.terminal_state === "completed" + ), + "replacement result did not satisfy the original coordinator-main join" + ); + const artifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", processId], + { cwd: projectDir } + ).artifacts; + assert(artifacts.some((artifact) => artifact.artifact.startsWith("stable.txt-"))); + assert( + artifacts.some((artifact) => artifact.artifact.startsWith("recovering.txt-")) + ); + const released = await waitForCli( + "hosted recovery process release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120_000 + ); + fs.writeFileSync(sourcePath, original); + restored = true; + await client.close(); + closed = true; + return { + duration_ms: Date.now() - startedAt, + process: processId, + logical_task: originalFailure.task, + original_attempt: originalFailure.attempt_id, + replacement_attempt: replacementEvent.attempt_id, + original_join_completed: true, + terminal_state: released.state, + }; + } finally { + if (!restored) fs.writeFileSync(sourcePath, original); + if (!closed) { + await client.close().catch(() => { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + }); + } + } +} + +async function main() { + requireEnabled(); + const manifest = readJson(manifestPath); + for (const asset of manifest.assets ?? []) { + asset.file = path.isAbsolute(asset.file) + ? asset.file + : path.resolve(path.dirname(manifestPath), asset.file); + } + assert.strictEqual(manifest.kind, "clusterflux-public-release"); + assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint); + assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443"); + const qualityGates = strictQualityGateEvidence(manifest); + + const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-happy-")); + const installDir = path.join(workRoot, "install"); + const checkout = path.join(workRoot, "public-repo"); + const downloadPath = path.join(workRoot, "release.tar"); + const extractedArtifact = path.join(workRoot, "artifact"); + ensureDir(installDir); + run("tar", ["-xzf", binaryArchive(manifest), "-C", installDir]); + const publicSource = stagePublicCheckout(manifest, checkout); + + const clusterflux = executable(installDir, "clusterflux"); + const clusterfluxNode = executable(installDir, "clusterflux-node"); + const clusterfluxDap = executable(installDir, "clusterflux-debug-dap"); + const projectDir = path.join(checkout, "tests/fixtures/runtime-conformance"); + const helloProjectDir = path.join(checkout, "examples/hello-build"); + const recoveryProjectDir = path.join(checkout, "examples/recovery-build"); + const suffix = String(Date.now()); + const workerNode = `worker-${suffix}`; + + const loginStartedAt = Date.now(); + let loginSession; + let receivedDefaultProject; + if (reuseSessionFile) { + const sourceSessionPath = path.resolve(reuseSessionFile); + const sourceProjectPath = path.join(path.dirname(sourceSessionPath), "project.json"); + const sourceSession = readJson(sourceSessionPath); + const sourceProject = readJson(sourceProjectPath); + assert.strictEqual(sourceSession.kind, "human"); + assert.strictEqual(sourceSession.coordinator, serviceEndpoint); + assert.strictEqual(sourceProject.coordinator, serviceEndpoint); + assert.strictEqual(sourceProject.tenant, sourceSession.tenant); + assert.strictEqual(sourceProject.project, sourceSession.project); + assert.strictEqual(sourceProject.user, sourceSession.user); + assert( + Number(sourceSession.expires_at) > Math.floor(Date.now() / 1000) + 300, + "reused CLI session expires too soon for the live journey" + ); + const destinationConfig = path.join(projectDir, ".clusterflux"); + ensureDir(destinationConfig); + fs.copyFileSync(sourceSessionPath, path.join(destinationConfig, "session.json")); + fs.copyFileSync(sourceProjectPath, path.join(destinationConfig, "project.json")); + fs.chmodSync(path.join(destinationConfig, "session.json"), 0o600); + fs.chmodSync(path.join(destinationConfig, "project.json"), 0o644); + loginSession = sourceSession; + receivedDefaultProject = true; + } else { + const login = runJson( + clusterflux, + ["login", "--browser", "--json"], + { + cwd: projectDir, + env: { ...process.env, CLUSTERFLUX_BROWSER_OPEN_COMMAND: browserOpenCommand }, + } + ); + assert.strictEqual(login.plan.coordinator, serviceEndpoint); + assert.strictEqual(login.boundary.scoped_cli_session_received, true); + assert.strictEqual(login.boundary.provider_tokens_exposed_to_cli, false); + assert.strictEqual(login.boundary.provider_tokens_sent_to_nodes, false); + loginSession = login.coordinator_response.session; + assert(loginSession, "hosted browser login omitted its scoped session"); + receivedDefaultProject = + typeof loginSession.project === "string" && loginSession.project.length > 0; + assert(receivedDefaultProject, "login did not return its server-owned project"); + } + const sessionSecret = + loginSession.cli_session_secret || loginSession.session_secret; + const loginDurationMs = Date.now() - loginStartedAt; + assert(sessionSecret, "hosted browser login omitted the CLI session credential"); + const tenant = loginSession.tenant; + const project = loginSession.project; + const user = loginSession.user; + for (const [name, value] of Object.entries({ tenant, project, user })) { + assert.strictEqual(typeof value, "string", `hosted session omitted ${name}`); + assert.notStrictEqual(value, "", `hosted session returned an empty ${name}`); + } + const scope = [ + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--user", + user, + "--json", + ]; + + if (!reuseSessionFile) { + const projectInit = runJson( + clusterflux, + [ + "project", + "init", + ...scope, + "--new-project", + project, + "--name", + "CLI Happy Path", + "--yes", + ], + { cwd: projectDir } + ); + assert.strictEqual(projectInit.command, "project init"); + assert.strictEqual(projectInit.project_config_written, true); + assert.strictEqual(projectInit.coordinator_response.type, "project_created"); + } + + const projectList = runJson(clusterflux, ["project", "list", ...scope], { + cwd: projectDir, + }); + assert(projectList.project_count >= 1); + const projectSelect = runJson( + clusterflux, + ["project", "select", ...scope, project], + { cwd: projectDir } + ); + assert.strictEqual(projectSelect.command, "project select"); + + const inspection = runJson(clusterflux, ["inspect", "--project", ".", "--json"], { + cwd: projectDir, + }); + assert( + inspection.metadata.environments.some((environment) => environment.name === "linux") + ); + assert(Array.isArray(inspection.source_provider_statuses)); + assert(inspection.source_provider_manifest); + + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.command, "run"); + assert.strictEqual(runReport.status, "main_launched"); + assert.strictEqual(runReport.task_launch.type, "main_launched"); + const processId = runReport.process; + + const parkedStatus = await waitForCli( + "hosted coordinator main to park before node enrollment", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => + status.live_process?.main_state === "running" && + status.live_process?.main_wait_state === "waiting_for_node" && + status.live_process?.connected_nodes?.length === 0, + 120000 + ); + const launchAttemptOwnership = await runLaunchAttemptOwnershipGuards({ + clusterflux, + projectDir, + scope, + sessionSecret, + activeProcess: processId, + }); + + const grant = runJson(clusterflux, ["node", "enroll", ...scope], { + cwd: projectDir, + }); + assert.strictEqual(grant.command, "node enroll"); + const attach = runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + workerNode, + "--enrollment-grant", + grant.enrollment_grant.grant, + "--json", + ], + { cwd: projectDir } + ); + assert.strictEqual(attach.command, "node attach"); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + + const credentialDir = path.join(projectDir, ".clusterflux", "nodes"); + const credentialFiles = fs + .readdirSync(credentialDir) + .filter((file) => file.endsWith(".json")); + assert.strictEqual(credentialFiles.length, 1, "expected one persisted node credential"); + const credentialPath = path.join(credentialDir, credentialFiles[0]); + const credentialBefore = fs.readFileSync(credentialPath); + const credentialDigest = sha256(credentialBefore); + if (process.platform !== "win32") { + assert.strictEqual(fs.statSync(credentialPath).mode & 0o777, 0o600); + } + + const workerArgsFor = (projectRoot) => [ + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + workerNode, + "--worker", + "--project-root", + projectRoot, + "--assignment-poll-ms", + "500", + "--emit-ready", + ]; + const workerArgs = workerArgsFor(projectDir); + const spawnWorker = (projectRoot = projectDir) => + spawnJsonLines(clusterfluxNode, workerArgsFor(projectRoot), { + cwd: projectRoot, + env: { ...process.env }, + }); + + let worker = spawnWorker(); + try { + const firstReady = await worker.waitFor( + (value) => value.node_status === "ready", + "first persisted worker ready" + ); + assert.strictEqual(firstReady.node, workerNode); + const initiallyOnline = runJson( + clusterflux, + ["node", "status", ...scope, "--node", workerNode], + { cwd: projectDir } + ); + assert.strictEqual(nodeDescriptor(initiallyOnline, workerNode)?.online, true); + + const completedTasks = await waitForCli( + "real hosted flagship completion", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { + cwd: projectDir, + }), + (tasks) => completedFlagship(rawTaskEvents(tasks)) + ); + const firstEvents = rawTaskEvents(completedTasks); + assert( + firstEvents + .filter((event) => event.executor === "node") + .every((event) => event.node === workerNode) + ); + assert( + firstEvents.some( + (event) => + event.task_definition === "package_release" && + event.terminal_state === "completed" && + contentAddressedArtifactId(event, "release.tar") + ) + ); + + const releasedFlagshipStatus = await waitForCli( + "completed flagship process to release its active slot", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + + const offlineStartedAt = Date.now(); + await stopChild(worker.child); + const observedOffline = await waitForCli( + "server-derived worker stale/offline transition", + () => + runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { + cwd: projectDir, + }), + (status) => nodeDescriptor(status, workerNode)?.online === false, + 120000 + ); + worker = spawnWorker(); + const secondReady = await worker.waitFor( + (value) => value.node_status === "ready", + "restarted persisted worker ready" + ); + assert.strictEqual(secondReady.node, workerNode); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const nodeStatus = await waitForCli( + "server-derived worker online recovery", + () => + runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { + cwd: projectDir, + }), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30000 + ); + assert(JSON.stringify(nodeStatus.response).includes(workerNode)); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + const nodeLivenessTransition = { + initial_online: nodeDescriptor(initiallyOnline, workerNode).online, + stale_offline: nodeDescriptor(observedOffline, workerNode).online, + recovered_online: nodeDescriptor(nodeStatus, workerNode).online, + offline_detection_ms: Date.now() - offlineStartedAt, + persisted_identity_reused: true, + }; + + const processStatus = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.strictEqual(processStatus.command, "process status"); + assert(processStatus.current_task_count >= firstEvents.length); + + const logs = runJson( + clusterflux, + ["logs", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert(logs.log_entries.length >= 4); + + const artifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + const releaseArtifact = artifacts.artifacts.find((artifact) => { + const digest = artifact.digest; + return ( + typeof digest === "string" && + /^sha256:[0-9a-f]{64}$/.test(digest) && + artifact.artifact === `release.tar-${digest.slice("sha256:".length)}` + ); + }); + assert(releaseArtifact, "hosted flagship did not publish release.tar"); + assert.strictEqual(releaseArtifact.state, "metadata_flushed"); + + const download = runJson( + clusterflux, + [ + "artifact", + "download", + ...scope, + releaseArtifact.artifact, + "--to", + downloadPath, + ], + { cwd: projectDir } + ); + assert.strictEqual(download.download_session.link_issued, true); + assert.strictEqual(download.local_download.local_bytes_written_by_cli, true); + assert.strictEqual(download.local_download.verified_digest, releaseArtifact.digest); + assert.strictEqual(sha256(fs.readFileSync(downloadPath)), releaseArtifact.digest); + + ensureDir(extractedArtifact); + const archiveEntries = run("tar", ["-tf", downloadPath]); + assert.match(archiveEntries, /(?:^|\n)hello-clusterflux(?:\n|$)/); + run("tar", ["-xf", downloadPath, "-C", extractedArtifact]); + const builtExecutable = path.join(extractedArtifact, "hello-clusterflux"); + assert(fs.existsSync(builtExecutable)); + const builtOutput = run(builtExecutable, []).trim(); + assert.strictEqual(builtOutput, "hello from a real Clusterflux build"); + + copyProjectControlState(projectDir, helloProjectDir); + copyProjectControlState(projectDir, recoveryProjectDir); + await stopChild(worker.child); + worker = spawnWorker(helloProjectDir); + await worker.waitFor( + (value) => value.node_status === "ready", + "hello-build worker ready" + ); + const helloBuild = await runHostedHelloBuild({ + clusterflux, + projectDir: helloProjectDir, + scope, + outputFile: path.join(workRoot, "hello-clusterflux"), + }); + await stopChild(worker.child); + worker = spawnWorker(recoveryProjectDir); + await worker.waitFor( + (value) => value.node_status === "ready", + "recovery-build worker ready" + ); + const recoveryBuild = await runHostedRecoveryBuild({ + clusterfluxDap, + clusterflux, + projectDir: recoveryProjectDir, + scope, + }); + await stopChild(worker.child); + worker = spawnWorker(); + await worker.waitFor( + (value) => value.node_status === "ready", + "runtime-conformance worker restored" + ); + + const dapEditRestart = await runLiveDapEditRestart({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + worker, + }); + + const restart = runJson( + clusterflux, + ["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(restart.restart_request.accepted, true); + const cancel = runJson( + clusterflux, + ["process", "cancel", ...scope, "--process", dapEditRestart.process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(cancel.cancel_request.accepted, true); + const releasedDap = await waitForCli( + "cancelled DAP process slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", dapEditRestart.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + assert.strictEqual(releasedDap.state, "not_active"); + + const identityWorkerNode = `identity-worker-${suffix}`; + const identityWorkerGrant = runJson( + clusterflux, + ["node", "enroll", ...scope], + { cwd: projectDir } + ); + runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + identityWorkerNode, + "--enrollment-grant", + identityWorkerGrant.enrollment_grant.grant, + "--json", + ], + { cwd: projectDir } + ); + const identityWorker = spawnJsonLines( + clusterfluxNode, + [ + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + identityWorkerNode, + "--worker", + "--project-root", + projectDir, + "--assignment-poll-ms", + "500", + "--emit-ready", + ], + { cwd: projectDir, env: { ...process.env } } + ); + let sameDefinitionIdentity; + try { + const identityWorkerReady = await identityWorker.waitFor( + (value) => value.node_status === "ready", + "same-definition identity worker ready" + ); + assert.strictEqual(identityWorkerReady.node, identityWorkerNode); + await waitForCli( + "same-definition identity worker online", + () => + runJson( + clusterflux, + ["node", "status", ...scope, "--node", identityWorkerNode], + { cwd: projectDir } + ), + (status) => nodeDescriptor(status, identityWorkerNode)?.online === true, + 30_000 + ); + await stopChild(worker.child); + worker = spawnWorker(); + const refreshedPrimaryReady = await worker.waitFor( + (value) => value.node_status === "ready", + "same-definition primary worker capability refresh" + ); + assert.strictEqual(refreshedPrimaryReady.node, workerNode); + const primaryStatus = await waitForCli( + "same-definition primary worker online after refresh", + () => + runJson( + clusterflux, + ["node", "status", ...scope, "--node", workerNode], + { cwd: projectDir } + ), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30_000 + ); + const identityStatus = runJson( + clusterflux, + ["node", "status", ...scope, "--node", identityWorkerNode], + { cwd: projectDir } + ); + const primarySnapshots = + nodeDescriptor(primaryStatus, workerNode)?.source_snapshots || []; + const identitySnapshots = + nodeDescriptor(identityStatus, identityWorkerNode)?.source_snapshots || []; + assert( + primarySnapshots.some((snapshot) => identitySnapshots.includes(snapshot)), + `same-definition workers do not share a current source snapshot: ${JSON.stringify({ + primarySnapshots, + identitySnapshots, + })}` + ); + sameDefinitionIdentity = await runSameDefinitionDapIdentity({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + }); + } finally { + await stopChild(identityWorker.child); + runJson( + clusterflux, + ["node", "revoke", ...scope, "--node", identityWorkerNode, "--yes"], + { cwd: projectDir } + ); + } + const repeatedParkWake = await runRepeatedParkWakeProof({ + clusterflux, + projectDir, + scope, + }); + const longJoin = await runLongJoinProof({ + clusterflux, + projectDir, + scope, + }); + + const tenantIsolation = await runSecondTenantIsolation({ + firstTenant: tenant, + firstProject: project, + firstProcess: dapEditRestart.process, + firstNode: workerNode, + firstArtifact: releaseArtifact.artifact, + manifest, + }); + + const securityNode = await prepareLiveNodeCredentialSecurity({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, + bundle: { + digest: runReport.bundle_digest, + moduleBase64: fs + .readFileSync( + path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) + ) + .toString("base64"), + entryExport: runReport.entry_export, + entryStableId: runReport.entry_stable_id, + }, + }); + const droppedConnectionRollback = await runDroppedConnectionRollback({ + clusterflux, + projectDir, + scope, + }); + const bandwidthPreallocation = await runLiveBandwidthPreallocation({ + sessionSecret, + artifact: releaseArtifact.artifact, + maxBytes: download.local_download.bytes_written, + }); + await stopChild(worker.child); + const relayEmergencyDisableResult = await runRelayEmergencyDisable({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + sessionSecret, + maxBytes: download.local_download.bytes_written, + }); + worker = relayEmergencyDisableResult.worker; + const relayEmergencyDisable = relayEmergencyDisableResult.evidence; + const agentCredentialSecurity = await runLiveAgentCredentialSecurity({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + suffix, + bundle: { + digest: runReport.bundle_digest, + moduleBase64: fs + .readFileSync( + path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) + ) + .toString("base64"), + entryExport: runReport.entry_export, + entryStableId: runReport.entry_stable_id, + }, + securityNode, + workerRuntime: { + node: workerNode, + child: worker.child, + worker, + identity: nodeIdentityFromPrivateKey( + readNodeCredential(projectDir, workerNode).credential.private_key + ), + }, + }); + + await stopChild(worker.child); + const liveSoak = await runLiveSoak({ + clusterflux, + projectDir, + scope, + securityNode, + }); + const revokedNodeCredential = await revokeSecurityNode({ + clusterflux, + projectDir, + scope, + securityNode, + }); + const quotaPreallocation = await runLiveQuotaPreallocation({ + sessionSecret, + suffix, + }); + const hostedLoginIsolation = await runHostedLoginIsolationBeforeRestart(); + const serviceRestart = await restartHostedServiceAndResume({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + workerNode, + credentialPath, + credentialDigest, + }); + if (serviceRestart.worker) worker = serviceRestart.worker; + const relayAfterRestart = relayDurableState(); + assert( + relayAfterRestart.ingress_used >= + bandwidthPreallocation.durable_after_release.ingress_used + ); + assert( + relayAfterRestart.egress_used >= + bandwidthPreallocation.durable_after_release.egress_used + ); + assert( + relayAfterRestart.abandoned_or_failed_used >= + bandwidthPreallocation.durable_after_release.abandoned_or_failed_used + ); + bandwidthPreallocation.durable_after_restart = relayAfterRestart; + bandwidthPreallocation.restart_persistence = true; + bandwidthPreallocation.duration_ms = + Date.now() - bandwidthPreallocation.scenario_started_at_ms; + delete bandwidthPreallocation.scenario_started_at_ms; + await finishHostedLoginIsolationAfterRestart(hostedLoginIsolation); + const userCredentialSecurity = await finishUserCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, + }); + const configProvenance = configurationProvenance(); + const proxyConfigProvenance = proxyConfigurationProvenance(); + const concurrentCompileTasks = [ + ...new Set( + firstEvents + .filter( + (event) => + event.task_definition === "compile_linux" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + assert(concurrentCompileTasks.length >= 2); + const strictRequirementLedger = [ + { id: "01_authenticated_project", passed: Boolean(sessionSecret && tenant && project) }, + { id: "02_project_commands", passed: projectList.project_count >= 1 && projectSelect.command === "project select" }, + { id: "03_bundle_environment_inspection", passed: inspection.metadata.environments.some((environment) => environment.name === "linux") }, + { id: "04_launch_attempt_ownership_and_main_parking", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" && launchAttemptOwnership.existing_process_survived === true }, + { id: "05_enrollment_and_persisted_credential", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode }, + { id: "06_server_derived_node_liveness", passed: nodeLivenessTransition.initial_online === true && nodeLivenessTransition.stale_offline === false && nodeLivenessTransition.recovered_online === true }, + { id: "07_real_flagship_and_artifact", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" }, + { id: "08_same_definition_concurrency", passed: sameDefinitionIdentity.thread_evidence.length === 2 && sameDefinitionIdentity.completion_order.length === 2 && sameDefinitionIdentity.podman_paused_container_ids.length >= 2 && sameDefinitionIdentity.terminal_state === "not_active" }, + { id: "09_repeated_park_wake", passed: repeatedParkWake.completed_cycles >= 16 && repeatedParkWake.terminal_state === "not_active" }, + { id: "10_nested_environment_rejection", passed: agentCredentialSecurity.nested_environment_mismatch.denied === true }, + { id: "11_partial_freeze_warning_resume", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.dap_all_threads_stopped === false && agentCredentialSecurity.partial_freeze.podman_paused_container_ids.length > 0 && agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 }, + { id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true && dapEditRestart.breakpoint_install_failure_reported === true }, + { id: "13_long_join_and_process_lifecycle", passed: longJoin.duration_seconds > 120 && longJoin.terminal_state === "completed" && releasedFlagshipStatus.state === "not_active" && restart.restart_request.accepted === true && cancel.cancel_request.accepted === true }, + { id: "14_tenant_isolation", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true }, + { id: "15_user_credential_hostility", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) }, + { id: "16_node_credential_hostility", passed: securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true }, + { id: "17_agent_workflow_and_credentials", passed: agentCredentialSecurity.workflow.flagship_completed === true && agentCredentialSecurity.workflow.authenticated_without_browser === true && agentCredentialSecurity.workflow.external_direct_task_v1.denied === true && agentCredentialSecurity.workflow.child_launch === "task_launched" && agentCredentialSecurity.revoked.denied === true }, + { id: "18_dropped_response_rollback", passed: droppedConnectionRollback.cli_exit_status !== 0 && droppedConnectionRollback.rollback === "automatic CLI launch guard abort_process with matching launch_attempt" && droppedConnectionRollback.terminal_state === "not_active" }, + { id: "19_quota_and_bandwidth_preallocation", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.unrelated_tenant.unaffected_by_first_tenant_quota === true && bandwidthPreallocation.bytes_served_before_denial > 0 && bandwidthPreallocation.partial_abandon.ingress_delta > 0 && bandwidthPreallocation.partial_abandon.egress_delta > 0 && bandwidthPreallocation.partial_abandon.abandoned_delta > 0 && bandwidthPreallocation.partial_abandon.reservation_released === true && bandwidthPreallocation.restart_persistence === true && relayEmergencyDisable.disabled.denied === true && relayEmergencyDisable.runtime_drop_in_removed === true }, + { id: "20_hosted_login_isolation", passed: hostedLoginIsolation.local_limited_status === 429 && hostedLoginIsolation.forwarding_spoof_status === 429 && hostedLoginIsolation.independent_vps_client_status === 200 && hostedLoginIsolation.control_route_after === "pong" && hostedLoginIsolation.after_service_restart.control_route === "pong" }, + { id: "21_soak_restart_and_provenance", passed: liveSoak.executed === true && serviceRestart.executed === true && manifest.source_tree_clean === true && configProvenance.clean === true }, + { id: "22_quality_gates", passed: qualityGates?.private.status === "passed" && qualityGates?.public.status === "passed" }, + { id: "23_hosted_hello_build", passed: helloBuild.executable_output === "hello from a real Clusterflux build" && helloBuild.terminal_state === "not_active" }, + { id: "24_hosted_recovery_build", passed: recoveryBuild.original_join_completed === true && recoveryBuild.original_attempt !== recoveryBuild.replacement_attempt && recoveryBuild.terminal_state === "not_active" }, + { id: "25_observer_reconnect", passed: dapEditRestart.observer_reconnected_without_false_stop === true && dapEditRestart.observer_idle_request_rate_bound_per_second <= 0.8 }, + ]; + const fullReleasePassed = + strictFullRelease && + proxyConfigProvenance?.active_state === "active" && + strictRequirementLedger.every((requirement) => requirement.passed === true); + const namedScenarios = strictRequirementLedger.map((requirement) => ({ + id: requirement.id, + name: requirement.id + .replace(/^\d+_/, "") + .replaceAll("_", " "), + status: requirement.passed ? "passed" : "failed", + duration_ms: 0, + })); + + const report = { + kind: "clusterflux-cli-happy-path-live", + release_name: manifest.release_name, + source_commit: manifest.source_commit, + source_tree_digest: manifest.source_tree_digest, + public_tree_identity: manifest.public_tree_identity, + binary_digests: manifest.binary_digests, + service_addr: serviceAddr, + default_hosted_coordinator_endpoint: serviceEndpoint, + public_repository_url: publicSource.published ? publicSource.url : null, + public_source: publicSource, + tenant, + project, + worker_node: workerNode, + process: processId, + signed_in: true, + fresh_hosted_login: !reuseSessionFile, + reused_scoped_cli_session: Boolean(reuseSessionFile), + received_default_project: receivedDefaultProject, + process_started_before_node: true, + parked_main_observed: { + main_state: parkedStatus.live_process.main_state, + main_wait_state: parkedStatus.live_process.main_wait_state, + connected_nodes: parkedStatus.live_process.connected_nodes, + }, + node_enrollment_grants_used: 2, + persisted_node_credential_mode: + process.platform === "win32" ? null : fs.statSync(credentialPath).mode & 0o777, + persisted_node_credential_digest: credentialDigest, + node_restarted_without_grant: true, + restarted_node_ready: secondReady.node, + node_liveness_transition: nodeLivenessTransition, + flagship_terminal_state: releasedFlagshipStatus.state, + flagship_task_definitions: [ + "prepare_source", + "compile_linux", + "package_release", + ], + rootless_podman_flagship_completed: true, + downstream_release_artifact: releaseArtifact, + artifact_download: { + bytes_written: download.local_download.bytes_written, + verified_digest: download.local_download.verified_digest, + executable_output: builtOutput, + }, + flagship_process_released_automatically: true, + concurrent_same_definition_tasks: sameDefinitionIdentity, + flagship_same_definition_task_instances: concurrentCompileTasks, + repeated_park_wake: repeatedParkWake, + long_join: longJoin, + live_dap_edit_restart: dapEditRestart, + process_restart_requested: true, + process_cancel_requested: true, + dap_process_aborted: true, + tenant_isolation: tenantIsolation, + credential_security: { + user: userCredentialSecurity, + node: { + node: securityNode.node, + credential_digest: securityNode.credential_digest, + ...securityNode.evidence, + revoked: revokedNodeCredential.denied, + }, + agent: agentCredentialSecurity, + }, + quota_preallocation: quotaPreallocation, + bandwidth_preallocation: bandwidthPreallocation, + relay_emergency_disable: relayEmergencyDisable, + hosted_login_isolation: hostedLoginIsolation, + dropped_connection_rollback: droppedConnectionRollback, + launch_attempt_ownership: launchAttemptOwnership, + live_soak: liveSoak, + hosted_service_restart: { + ...serviceRestart, + worker: undefined, + }, + release_binding: { + manifest: manifestPath, + source_commit: manifest.source_commit, + source_tree_digest: manifest.source_tree_digest, + source_tree_clean: manifest.source_tree_clean, + public_tree_identity: manifest.public_tree_identity, + binary_digests: manifest.binary_digests, + release_candidate: manifest.release_candidate, + deployment: serviceRestart.after || deploymentProvenance(), + configuration: configProvenance, + proxy_configuration: proxyConfigProvenance, + }, + commands, + quality_gates: qualityGates, + hello_build: helloBuild, + recovery_build: recoveryBuild, + named_scenarios: namedScenarios, + scenario_skips: [], + strict_requirement_ledger: strictRequirementLedger, + acceptance_result: fullReleasePassed ? "passed" : "partial", + }; + ensureDir(path.dirname(reportPath)); + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + if (strictFullRelease && !fullReleasePassed) { + const failed = strictRequirementLedger + .filter((requirement) => requirement.passed !== true) + .map((requirement) => requirement.id); + throw new Error(`strict full release failed: ${failed.join(", ")}`); + } + console.log(`CLI happy-path live smoke passed: ${reportPath}`); + } finally { + await stopChild(worker?.child); + } +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-install-smoke.js b/scripts/cli-install-smoke.js new file mode 100755 index 0000000..878543e --- /dev/null +++ b/scripts/cli-install-smoke.js @@ -0,0 +1,61 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-install-")); +const installRoot = path.join(temp, "install"); +const targetDir = + process.env.CLUSTERFLUX_CLI_INSTALL_CARGO_TARGET_DIR || + process.env.CARGO_TARGET_DIR || + path.join(repo, "target"); +const project = path.join(repo, "tests/fixtures/runtime-conformance"); +const binName = process.platform === "win32" ? "clusterflux.exe" : "clusterflux"; +const installedBin = path.join(installRoot, "bin", binName); + +try { + cp.execFileSync( + "cargo", + [ + "install", + "--path", + "crates/clusterflux-cli", + "--bin", + "clusterflux", + "--root", + installRoot, + "--debug" + ], + { + cwd: repo, + env: { + ...process.env, + CARGO_TARGET_DIR: targetDir + }, + stdio: "inherit" + } + ); + + assert(fs.existsSync(installedBin), "installed clusterflux binary must exist"); + + const inspection = JSON.parse( + cp.execFileSync( + installedBin, + ["bundle", "inspect", "--project", project, "--json"], + { cwd: repo, encoding: "utf8" } + ) + ); + + assert.strictEqual(inspection.project, project); + assert.strictEqual(inspection.metadata.embeds_full_container_images, false); + assert(inspection.metadata.environments.some((env) => env.name === "linux")); + assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/lib.rs")); +} finally { + fs.rmSync(temp, { recursive: true, force: true }); +} + +console.log("CLI install smoke passed"); diff --git a/scripts/cli-local-run-smoke.js b/scripts/cli-local-run-smoke.js new file mode 100755 index 0000000..e2fc299 --- /dev/null +++ b/scripts/cli-local-run-smoke.js @@ -0,0 +1,269 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +configurePodmanTestEnvironment(repo); +if ( + !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && + cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && + cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 +) { + cp.execFileSync( + "nix", + ["shell", "nixpkgs#podman", "--command", "node", __filename], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_PODMAN_NIX_SHELL: "1", + }, + stdio: "inherit", + } + ); + process.exit(0); +} +const project = path.join(repo, "tests/fixtures/runtime-conformance"); + +cp.execFileSync( + "cargo", + ["build", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node"], + { cwd: repo, stdio: "inherit" } +); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function runCli(args, env = {}) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + env: { + ...process.env, + ...env + } + } + ); + const cliPid = child.pid; + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`CLI run failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve({ pid: cliPid, report: JSON.parse(stdout) }); + } catch (error) { + reject(new Error(`CLI output was not JSON: ${stdout}\n${error.stack || error.message}`)); + } + }); + }); +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + assert(Number.isInteger(coordinator.pid)); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const { pid: cliPid, report } = await runCli([ + "run", + "--coordinator", + `${addr.host}:${addr.port}`, + "--project", + project, + "--json", + ]); + assert(Number.isInteger(cliPid)); + assert.notStrictEqual(cliPid, coordinator.pid); + assert.strictEqual(report.plan.entry, "build"); + assert.deepStrictEqual(report.plan.session, "Anonymous"); + assert.strictEqual(report.boundary.cli_process_started_node_process, true); + assert.strictEqual(report.boundary.cli_process_started_coordinator_process, false); + assert(Number.isInteger(report.boundary.spawned_node_process_id)); + assert.notStrictEqual(report.boundary.spawned_node_process_id, cliPid); + assert.notStrictEqual(report.boundary.spawned_node_process_id, coordinator.pid); + assert.strictEqual(report.boundary.node_session_requests, 0); + assert.strictEqual(report.node_report.node_status, "completed"); + assert.strictEqual(report.node_report.execution_substrate, "wasm"); + assert.strictEqual(report.node_report.task_spawn_host_import, true); + assert.strictEqual( + report.node_report.pre_node_process_status.processes.length, + 1 + ); + assert.deepStrictEqual( + report.node_report.pre_node_process_status.processes[0].connected_nodes, + [] + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_state, + "running" + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_wait_state, + "waiting_for_node", + "the coordinator must expose that the capless main is parked on placement before a node exists" + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_task_instance, + report.node_report.run.task_instance + ); + assert.strictEqual(report.node_report.run.status, "main_launched"); + assert.strictEqual(report.node_report.join.type, "task_joined"); + const process = report.node_report.run.process; + assert.strictEqual(process, "vp-current"); + + const events = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process + }); + assert.strictEqual(events.type, "task_events"); + assert(events.events.length >= 4); + assert( + events.events + .filter((event) => event.executor === "node") + .every((event) => event.node === "node-cli-local") + ); + assert( + events.events.some( + (event) => + event.executor === "coordinator_main" && + event.node === "coordinator-main" + ) + ); + assert(events.events.every((event) => event.process === process)); + assert.deepStrictEqual( + new Set(events.events.map((event) => event.task_definition)), + new Set([ + report.node_report.run.task_definition, + "prepare_source", + "compile_linux", + "package_release", + ]) + ); + assert.strictEqual( + new Set(events.events.map((event) => event.task)).size, + events.events.length, + "every live task event must retain its unique instance identity" + ); + assert( + events.events.some( + (event) => event.task === report.node_report.run.task_instance + ) + ); + assert(events.events.some((event) => event.task.endsWith(":child:1"))); + assert(events.events.some((event) => event.task.endsWith(":child:2"))); + assert(events.events.some((event) => event.task.endsWith(":child:3"))); + assert(events.events.some((event) => event.artifact_path)); + } finally { + coordinator.kill("SIGTERM"); + } + + const { pid: autoCliPid, report: autoReport } = await runCli([ + "run", + "--local", + "--project", + project, + "--json", + ]); + assert(Number.isInteger(autoCliPid)); + assert.strictEqual(autoReport.plan.entry, "build"); + assert.deepStrictEqual(autoReport.plan.coordinator, "LocalOnly"); + assert.deepStrictEqual(autoReport.plan.session, "Anonymous"); + assert.strictEqual(autoReport.boundary.cli_process_started_node_process, true); + assert.strictEqual(autoReport.boundary.cli_process_started_coordinator_process, true); + assert.match(autoReport.boundary.coordinator_address, /^127\.0\.0\.1:\d+$/); + assert(Number.isInteger(autoReport.boundary.coordinator_process_id)); + assert(Number.isInteger(autoReport.boundary.spawned_node_process_id)); + assert.notStrictEqual(autoReport.boundary.coordinator_process_id, autoCliPid); + assert.notStrictEqual(autoReport.boundary.spawned_node_process_id, autoCliPid); + assert.notStrictEqual( + autoReport.boundary.spawned_node_process_id, + autoReport.boundary.coordinator_process_id + ); + assert.strictEqual(autoReport.boundary.node_session_requests, 0); + assert.strictEqual(autoReport.node_report.node_status, "completed"); + assert.strictEqual(autoReport.node_report.execution_substrate, "wasm"); + assert.strictEqual(autoReport.node_report.task_spawn_host_import, true); + assert.strictEqual(autoReport.node_report.run.status, "main_launched"); + assert.strictEqual(autoReport.node_report.join.type, "task_joined"); + + console.log("CLI local run smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-login-smoke.js b/scripts/cli-login-smoke.js new file mode 100644 index 0000000..61b053b --- /dev/null +++ b/scripts/cli-login-smoke.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const coordinator = "https://coord.example.test"; +const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; + +function clusterflux(args) { + return JSON.parse( + cp.execFileSync( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { cwd: repo, encoding: "utf8" } + ) + ); +} + +function clusterfluxRaw(args, env = {}) { + return cp.spawnSync( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + encoding: "utf8", + env: { + ...process.env, + ...env, + }, + } + ); +} + +const browser = clusterflux(["login", "--plan", "--coordinator", coordinator, "--json"]); +assert.strictEqual(browser.coordinator, coordinator); +assert(browser.human_flow.Browser, "browser login should be available for human users"); +assert.strictEqual(browser.human_flow.Browser.authorization_url, null); +assert.strictEqual(browser.human_flow.Browser.server_owns_state, true); +assert.strictEqual(browser.human_flow.Browser.server_owns_nonce, true); +assert.strictEqual(browser.human_flow.Browser.pkce_required, true); +assert.strictEqual(browser.human_flow.Browser.hosted_callback, true); +assert.strictEqual(browser.human_flow.Browser.cli_receives_provider_authorization_code, false); +assert.strictEqual(browser.human_flow.Browser.cli_submits_identity_claims, false); + +const defaultBrowser = clusterflux(["login", "--plan", "--json"]); +assert.strictEqual(defaultBrowser.coordinator, defaultHostedCoordinatorEndpoint); +assert(defaultBrowser.human_flow.Browser); + +const nonInteractiveBrowser = clusterfluxRaw( + ["login", "--browser", "--non-interactive", "--coordinator", coordinator, "--json"], + { + CLUSTERFLUX_BROWSER_OPEN_COMMAND: + "node -e 'require(\"fs\").writeFileSync(\"/tmp/clusterflux-browser-should-not-open\", \"opened\")'", + } +); +assert.strictEqual(nonInteractiveBrowser.status, 20, nonInteractiveBrowser.stderr); +assert.doesNotMatch(nonInteractiveBrowser.stderr, /Opening Clusterflux browser login/); +const nonInteractiveReport = JSON.parse(nonInteractiveBrowser.stdout); +assert.strictEqual(nonInteractiveReport.status, "authentication_required"); +assert.strictEqual(nonInteractiveReport.non_interactive, true); +assert.strictEqual(nonInteractiveReport.browser_opened, false); +assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); +assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); +assert( + nonInteractiveReport.machine_error.next_actions.includes( + "rerun without --non-interactive to open the browser" + ) +); + +console.log("CLI login smoke passed"); diff --git a/scripts/cli-output-mode-smoke.js b/scripts/cli-output-mode-smoke.js new file mode 100644 index 0000000..6cced83 --- /dev/null +++ b/scripts/cli-output-mode-smoke.js @@ -0,0 +1,191 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "tests/fixtures/runtime-conformance"); +const isolatedCwd = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-output-")); +const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-home-")); + +function clusterflux(args, env = {}, cwd = isolatedCwd) { + return cp.execFileSync( + "cargo", + [ + "run", + "-q", + "--manifest-path", + path.join(repo, "Cargo.toml"), + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + ...args, + ], + { + cwd, + encoding: "utf8", + env: { + ...process.env, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_DATA_HOME: path.join(isolatedHome, ".local", "share"), + XDG_STATE_HOME: path.join(isolatedHome, ".local", "state"), + ...env, + }, + } + ); +} + +function json(args, env, cwd) { + return JSON.parse(clusterflux(args, env, cwd)); +} + +function assertHuman(name, output, requiredPatterns) { + assert( + !output.trimStart().startsWith("{"), + `${name} default output should be human-readable text, not JSON` + ); + for (const pattern of requiredPatterns) { + assert.match(output, pattern, `${name} human output missing ${pattern}`); + } +} + +const helpHuman = clusterflux(["help"]); +assertHuman("help", helpHuman, [ + /Primary workflow:/, + /clusterflux login --browser/, + /clusterflux project init/, + /clusterflux node attach; clusterflux-node --worker/, + /Clusterflux: Launch Virtual Process/, + /Hosted account creation happens in the browser login flow/, + /--json/, +]); + +const loginHuman = clusterflux([ + "login", + "--plan", + "--coordinator", + "https://coord.example.test", +]); +assertHuman("login", loginHuman, [ + /Clusterflux login/, + /flow: browser/, +]); + +const loginJson = json([ + "login", + "--plan", + "--coordinator", + "https://coord.example.test", + "--json", +]); +assert.strictEqual(loginJson.coordinator, "https://coord.example.test"); +assert(loginJson.human_flow.Browser); +assert.strictEqual(loginJson.human_flow.Browser.hosted_callback, true); +assert.strictEqual(loginJson.human_flow.Browser.cli_submits_identity_claims, false); + +const doctorHuman = clusterflux(["doctor"]); +assertHuman("doctor", doctorHuman, [ + /Clusterflux doctor/, + /coordinator reachability: not_configured/, + /dependencies:/, + /auth:/, + /node capabilities:/, + /node readiness: (ready_to_attach|local_dependencies_missing|limited_capabilities)/, + /node next:/, +]); + +const doctorJson = json(["doctor", "--json"]); +assert.strictEqual(doctorJson.coordinator_reachability.checked, false); +assert.strictEqual(doctorJson.coordinator_reachability.status, "not_configured"); +assert( + ["ready_to_attach", "local_dependencies_missing", "limited_capabilities"].includes( + doctorJson.node_readiness_summary.status + ) +); +assert.strictEqual(doctorJson.node_readiness_summary.explicit_attach_required, true); +assert.strictEqual( + doctorJson.node_readiness_summary.command_execution_capability, + true +); +assert(Array.isArray(doctorJson.node_readiness_summary.missing_local_dependencies)); +assert(doctorJson.node_readiness_summary.next_actions.length >= 2); + +const authJson = json(["auth", "status", "--json"], { + CLUSTERFLUX_TOKEN: "token", + CLUSTERFLUX_TOKEN_EXPIRES_AT: "2026-07-04T00:00:00Z", +}, isolatedCwd); +assert.strictEqual(authJson.session.kind, "human"); +assert.strictEqual(authJson.session.token_expiry_posture, "expires_at"); +assert.strictEqual(authJson.session.expires_at, "2026-07-04T00:00:00Z"); +assert.strictEqual(authJson.coordinator_account_status.checked, false); +assert.strictEqual(authJson.coordinator_account_status.account_status, "unknown"); +assert.strictEqual( + authJson.coordinator_account_status.private_moderation_details_exposed, + false +); + +const inspectHuman = clusterflux(["bundle", "inspect", "--project", project]); +assertHuman("bundle inspect", inspectHuman, [ + /Clusterflux bundle inspect/, + /bundle: sha256:/, + /environments:/, +]); + +const inspectJson = json(["bundle", "inspect", "--project", project, "--json"]); +assert.strictEqual(inspectJson.project, project); +assert.match(inspectJson.metadata.identity, /^sha256:/); +assert.match(inspectJson.metadata.wasm_code, /^sha256:/); +assert.strictEqual(inspectJson.metadata.task_metadata.default_entrypoint, "build"); +assert.deepStrictEqual(inspectJson.metadata.task_metadata.entrypoints, [ + "build", + "fail", + "identity", + "long-join", + "park-wake", + "restart", +]); +assert.strictEqual( + inspectJson.metadata.source_metadata.transfer_policy.coordinator_receives_source_bytes_by_default, + false +); +assert.strictEqual( + inspectJson.metadata.source_metadata.transfer_policy.default_full_repo_tarball, + false +); +assert.strictEqual(inspectJson.metadata.debug_metadata.dap_virtual_process, true); +assert.strictEqual( + inspectJson.metadata.large_input_policy.selected_inputs_are_content_digests, + true +); +assert.strictEqual(inspectJson.metadata.large_input_policy.selected_input_bytes_included, false); +assert.strictEqual(inspectJson.metadata.large_input_policy.full_repository_bytes_included, false); +assert.strictEqual( + inspectJson.metadata.large_input_policy.silent_task_argument_serialization, + false +); +assert(inspectJson.metadata.large_input_policy.supported_handle_types.includes("SourceSnapshot")); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.source_edits_can_restart_from_clean_task_boundary, + true +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.requires_clean_checkpoint_boundary, + true +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.compares_task_abi, + inspectJson.metadata.task_metadata.task_abi +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.incompatible_changes_require_whole_process_restart, + true +); + +console.log("CLI output mode smoke passed"); diff --git a/scripts/coordinator-wire.js b/scripts/coordinator-wire.js new file mode 100644 index 0000000..7761736 --- /dev/null +++ b/scripts/coordinator-wire.js @@ -0,0 +1,43 @@ +let requestId = 0; + +function coordinatorWireRequest(payload, prefix = "acceptance") { + if (payload && payload.type === "coordinator_request") return payload; + if (!payload || typeof payload.type !== "string" || !payload.type.trim()) { + throw new Error("coordinator payload must have a non-empty type"); + } + requestId += 1; + return { + type: "coordinator_request", + protocol_version: 1, + request_id: `${prefix}-${process.pid}-${requestId}`, + operation: payload.type, + authentication: authenticationMetadata(payload), + payload, + }; +} + +function authenticationMetadata(payload) { + if (payload.type === "authenticated") { + return { + kind: "cli_session", + session: true, + request_operation: payload.request?.type || "unknown", + }; + } + if (payload.type === "signed_node" || payload.node_signature) { + return { kind: "node_signature", node: payload.node || null }; + } + if (payload.agent_signature) { + return { + kind: "agent_signature", + agent: payload.actor_agent || null, + fingerprint: payload.agent_public_key_fingerprint || null, + }; + } + if (payload.admin_token) { + return { kind: "admin_credential" }; + } + return { kind: "none" }; +} + +module.exports = { coordinatorWireRequest }; diff --git a/scripts/dap-client.js b/scripts/dap-client.js new file mode 100644 index 0000000..32d720c --- /dev/null +++ b/scripts/dap-client.js @@ -0,0 +1,135 @@ +const cp = require("child_process"); + +class DapClient { + constructor({ + cwd = process.cwd(), + env = process.env, + command = "cargo", + args = [ + "run", + "-q", + "-p", + "clusterflux-dap", + "--bin", + "clusterflux-debug-dap", + ], + } = {}) { + this.child = cp.spawn( + command, + args, + { cwd, env } + ); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(message)); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error( + `DAP ${command} failed: ${message.message || JSON.stringify(message)}\nAdapter stderr:\n${this.stderr}` + ); + } + return message; + } + + async failure(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (message.success) { + throw new Error(`DAP ${command} unexpectedly succeeded`); + } + return message; + } + + waitFor( + predicate, + timeoutMs = Number(process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || 120000) + ) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.child.kill("SIGKILL"); + const recent = this.messages + .slice(-10) + .map((message) => JSON.stringify(message)) + .join("\n"); + reject( + new Error( + `timed out waiting for DAP message\nRecent DAP messages:\n${recent}\nAdapter stderr:\n${this.stderr}` + ) + ); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +module.exports = { DapClient }; diff --git a/scripts/dap-smoke.js b/scripts/dap-smoke.js new file mode 100644 index 0000000..da74986 --- /dev/null +++ b/scripts/dap-smoke.js @@ -0,0 +1,461 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); +const { DapClient } = require("./dap-client"); + +(async () => { + const repo = path.resolve(__dirname, ".."); + const project = path.join(repo, "tests/fixtures/runtime-conformance"); + const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs")); + const sourceLines = fs.readFileSync(sourcePath, "utf8").split(/\r?\n/); + const buildMainLine = + sourceLines.findIndex((line) => line.includes("pub async fn build_main()")) + 1; + assert(buildMainLine > 0, "flagship source must contain build_main"); + + const asynchronousClient = new DapClient(); + try { + const initialize = asynchronousClient.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await asynchronousClient.response(initialize, "initialize"); + const launch = asynchronousClient.send("launch", { + entry: "long-join", + project, + runtimeBackend: "local-services", + }); + await asynchronousClient.response(launch, "launch"); + await asynchronousClient.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const configuredAt = Date.now(); + const configurationDone = asynchronousClient.send("configurationDone"); + await asynchronousClient.response(configurationDone, "configurationDone"); + assert( + Date.now() - configuredAt < 2_000, + "configurationDone must not wait for bundle build or runtime completion" + ); + + const threadDeadline = Date.now() + 120_000; + let runningThreads = []; + while (Date.now() < threadDeadline) { + const threadsRequest = asynchronousClient.send("threads"); + runningThreads = ( + await asynchronousClient.response(threadsRequest, "threads") + ).body.threads; + if (runningThreads.length > 0) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + assert(runningThreads.length > 0, "asynchronous launch never reported a live thread"); + + const pauseAt = Date.now(); + const pause = asynchronousClient.send("pause", { + threadId: runningThreads[0].id, + }); + await asynchronousClient.response(pause, "pause"); + assert(Date.now() - pauseAt < 2_000, "pause response was blocked by runtime work"); + const paused = await asynchronousClient.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "pause" + ); + + const continueAt = Date.now(); + const continued = asynchronousClient.send("continue", { + threadId: paused.body.threadId, + }); + await asynchronousClient.response(continued, "continue"); + assert( + Date.now() - continueAt < 2_000, + "continue response was blocked by runtime observation" + ); + + const disconnectAt = Date.now(); + await asynchronousClient.close(); + assert( + Date.now() - disconnectAt < 2_000, + "disconnect response was blocked by runtime observation" + ); + } catch (error) { + if (asynchronousClient.child.exitCode === null) { + asynchronousClient.child.kill("SIGKILL"); + } + throw error; + } + + const client = new DapClient(); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "build", + project, + runtimeBackend: "local-services", + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const breakpoints = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: buildMainLine }], + }); + const breakpointResponse = await client.response( + breakpoints, + "setBreakpoints" + ); + assert.strictEqual(breakpointResponse.body.breakpoints.length, 1); + assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, false); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const installedBreakpoint = await client.waitFor( + (message) => + message.type === "event" && + message.event === "breakpoint" && + message.body?.breakpoint?.verified === true + ); + assert.strictEqual(installedBreakpoint.body.breakpoint.line, buildMainLine); + const stopped = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(stopped.body.allThreadsStopped, true); + assert.match(stopped.body.description, /confirmed by every active participant/i); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body + .threads; + const mainThread = threads.find((thread) => + thread.name.includes("build coordinator main") + ); + assert(mainThread, "the running Wasm entrypoint must be the DAP coordinator-main thread"); + assert.strictEqual(stopped.body.threadId, mainThread.id); + + const stackRequest = client.send("stackTrace", { + threadId: mainThread.id, + startFrame: 0, + levels: 1, + }); + const stack = (await client.response(stackRequest, "stackTrace")).body + .stackFrames; + assert.strictEqual(stack.length, 1); + assert.strictEqual(stack[0].line, buildMainLine); + assert.strictEqual(stack[0].source.path, sourcePath); + assert.match(stack[0].name, /build_main::wasm/); + assert.doesNotMatch(stack[0].name, /podman|cmd\.exe|powershell|pid|native child/i); + + const sourceRequest = client.send("source", { source: stack[0].source }); + const source = (await client.response(sourceRequest, "source")).body; + assert.match(source.content, /build_main/); + assert.match(source.mimeType, /rust/); + + const scopesRequest = client.send("scopes", { frameId: stack[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const localsScope = scopes.find((scope) => scope.name === "Source Locals"); + const wasmScope = scopes.find((scope) => scope.name === "Wasm Frame Locals"); + const argsScope = scopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + const runtimeScope = scopes.find( + (scope) => scope.name === "Clusterflux Runtime" + ); + assert(localsScope && wasmScope && argsScope && runtimeScope); + + const localsRequest = client.send("variables", { + variablesReference: localsScope.variablesReference, + }); + const locals = (await client.response(localsRequest, "variables")).body + .variables; + assert( + locals.some( + (variable) => + variable.name === "unavailable-local-diagnostic" && + String(variable.value).includes("cannot be inspected") + ) + ); + + const wasmRequest = client.send("variables", { + variablesReference: wasmScope.variablesReference, + }); + const wasmLocals = (await client.response(wasmRequest, "variables")).body + .variables; + assert.deepStrictEqual( + wasmLocals.map((variable) => variable.name), + ["wasm-local-diagnostic"] + ); + assert.match(wasmLocals[0].value, /did not report inspectable Wasm frame locals/); + + const argsRequest = client.send("variables", { + variablesReference: argsScope.variablesReference, + }); + const args = (await client.response(argsRequest, "variables")).body.variables; + assert.deepStrictEqual( + args.map((variable) => variable.name), + ["runtime-boundary-diagnostic"] + ); + assert.match(args[0].value, /reported no task arguments or handles/); + + const runtimeRequest = client.send("variables", { + variablesReference: runtimeScope.variablesReference, + }); + const runtime = (await client.response(runtimeRequest, "variables")).body + .variables; + const value = (name) => runtime.find((variable) => variable.name === name)?.value; + assert.strictEqual(value("runtime_backend"), "LocalServices"); + assert.strictEqual(value("state"), "Frozen"); + assert.strictEqual(value("debug_epoch"), 1); + assert.strictEqual(value("coordinator_task_events"), 0); + assert.match( + String(value("command_status")), + /frozen through local services at executing Wasm probe/ + ); + + const step = client.send("next", { threadId: mainThread.id }); + const stepFailure = await client.failure(step, "next"); + assert.match(stepFailure.message, /source stepping is not yet available/i); + assert.match(stepFailure.message, /synthetic step/i); + + const restart = client.send("restartFrame", { frameId: stack[0].id }); + const restartFailure = await client.failure(restart, "restartFrame"); + assert.match(restartFailure.message, /checkpoint boundary|still active/i); + + const incompatibleRestart = client.send("restartFrame", { + frameId: stack[0].id, + sourceCompatibility: "incompatible", + }); + const incompatibleFailure = await client.failure( + incompatibleRestart, + "restartFrame" + ); + assert.match(incompatibleFailure.message, /incompatible source edit/i); + assert.match(incompatibleFailure.message, /whole virtual-process restart/i); + + await client.close(); + } catch (error) { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + throw error; + } + + const failMainLine = + sourceLines.findIndex((line) => line.includes("pub async fn fail_main()")) + 1; + assert(failMainLine > 0, "flagship source must contain fail_main"); + const taskTrapLine = + sourceLines.findIndex((line) => line.includes("fn task_trap(")) + 1; + assert(taskTrapLine > 0, "flagship source must contain task_trap"); + const restartClient = new DapClient(); + try { + const initialize = restartClient.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await restartClient.response(initialize, "initialize"); + const launch = restartClient.send("launch", { + entry: "fail", + project, + runtimeBackend: "local-services", + }); + await restartClient.response(launch, "launch"); + await restartClient.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + const breakpoints = restartClient.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: failMainLine }, { line: taskTrapLine }], + }); + const breakpointResponse = await restartClient.response( + breakpoints, + "setBreakpoints" + ); + assert.deepStrictEqual( + breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), + [false, false] + ); + const configurationDone = restartClient.send("configurationDone"); + await restartClient.response(configurationDone, "configurationDone"); + const installedLines = []; + while (installedLines.length < 2) { + const installed = await restartClient.waitFor( + (message) => + message.type === "event" && + message.event === "breakpoint" && + message.body?.breakpoint?.verified === true && + !installedLines.includes(message.body.breakpoint.line) + ); + installedLines.push(installed.body.breakpoint.line); + } + assert.deepStrictEqual(installedLines.sort((a, b) => a - b), [ + failMainLine, + taskTrapLine, + ].sort((a, b) => a - b)); + const initialStop = await restartClient.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(initialStop.body.allThreadsStopped, true); + const threadsRequest = restartClient.send("threads"); + const threads = ( + await restartClient.response(threadsRequest, "threads") + ).body.threads; + const failThread = threads.find( + (thread) => thread.id === initialStop.body.threadId + ); + assert(failThread, "failed entrypoint must remain a virtual task thread"); + const stackRequest = restartClient.send("stackTrace", { + threadId: failThread.id, + startFrame: 0, + levels: 1, + }); + const failedStack = ( + await restartClient.response(stackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual(failedStack[0].line, failMainLine); + + const continueRequest = restartClient.send("continue", { + threadId: failThread.id, + }); + await restartClient.response(continueRequest, "continue"); + const childStop = await restartClient.waitFor( + (message) => + message.seq > initialStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint", + 70000 + ); + assert.strictEqual(childStop.body.allThreadsStopped, true); + + const childThreadsRequest = restartClient.send("threads"); + const childThreads = ( + await restartClient.response(childThreadsRequest, "threads") + ).body.threads; + const childThread = childThreads.find( + (thread) => thread.id === childStop.body.threadId + ); + assert(childThread, "executing child task must become a DAP virtual thread"); + assert.notStrictEqual(childThread.id, failThread.id); + assert.match(childThread.name, /task trap/i); + + const childStackRequest = restartClient.send("stackTrace", { + threadId: childThread.id, + startFrame: 0, + levels: 1, + }); + const childStack = ( + await restartClient.response(childStackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual(childStack[0].line, taskTrapLine); + assert.match(childStack[0].name, /task_trap::wasm/); + + const childScopesRequest = restartClient.send("scopes", { + frameId: childStack[0].id, + }); + const childScopes = ( + await restartClient.response(childScopesRequest, "scopes") + ).body.scopes; + const childArgsScope = childScopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + assert(childArgsScope, "child task argument scope must be present"); + const childArgsRequest = restartClient.send("variables", { + variablesReference: childArgsScope.variablesReference, + }); + const childArgs = ( + await restartClient.response(childArgsRequest, "variables") + ).body.variables; + assert( + childArgs.some( + (variable) => + variable.name === "arg_0" && + /SmallJson\(Number\(0\)\)/.test(String(variable.value)) + ), + "child task argument must come from the frozen node participant" + ); + + const parentScopesRequest = restartClient.send("scopes", { + frameId: failedStack[0].id, + }); + const parentScopes = ( + await restartClient.response(parentScopesRequest, "scopes") + ).body.scopes; + const parentArgsScope = parentScopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + const parentArgsRequest = restartClient.send("variables", { + variablesReference: parentArgsScope.variablesReference, + }); + const parentArgs = ( + await restartClient.response(parentArgsRequest, "variables") + ).body.variables; + assert( + parentArgs.some( + (variable) => + /^task_handle_\d+$/.test(variable.name) && + /definition=task_trap instance=ti:.*:child:\d+ state=active/.test( + variable.value + ) && + variable.type === "runtime-handle" + ), + "parent task handle must come from its live Wasm host registry" + ); + + const continueChildRequest = restartClient.send("continue", { + threadId: childThread.id, + }); + await restartClient.response(continueChildRequest, "continue"); + await restartClient.waitFor( + (message) => + message.seq > childStop.seq && + message.type === "event" && + message.event === "terminated" + ); + + const terminalThreadsRequest = restartClient.send("threads"); + const terminalThreads = ( + await restartClient.response(terminalThreadsRequest, "threads") + ).body.threads; + assert.deepStrictEqual( + terminalThreads, + [], + "terminated processes must not retain stale virtual threads" + ); + + const restartRequest = restartClient.send("restartFrame", { + frameId: failedStack[0].id, + }); + const restartFailure = await restartClient.failure( + restartRequest, + "restartFrame" + ); + assert.match( + restartFailure.message, + /does not map to a virtual task/i, + "a frame from a terminated process must not restart a stale task" + ); + await restartClient.close(); + } catch (error) { + if (restartClient.child.exitCode === null) restartClient.child.kill("SIGKILL"); + throw error; + } + + console.log("DAP smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/deploy-release-candidate.sh b/scripts/deploy-release-candidate.sh new file mode 100755 index 0000000..8560767 --- /dev/null +++ b/scripts/deploy-release-candidate.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +manifest=${1:?usage: deploy-release-candidate.sh MANIFEST} +: "${CLUSTERFLUX_DEPLOY_COMMAND:?set the documented exact-candidate deployment command}" +: "${CLUSTERFLUX_STRICT_SSH_TARGET:?set the production SSH target}" + +service_unit=${CLUSTERFLUX_STRICT_SERVICE_UNIT:-clusterflux-hosted.service} +proxy_unit=${CLUSTERFLUX_STRICT_PROXY_UNIT:-nginx.service} +evidence_path=${CLUSTERFLUX_DEPLOYMENT_EVIDENCE_PATH:-target/acceptance/deployment.json} +staging=$(mktemp -d) +trap 'rm -rf "$staging"' EXIT + +IFS=$'\t' read -r archive expected_archive_sha < <( + node - "$manifest" <<'NODE' +const fs = require("fs"); +const path = require("path"); +const manifestPath = path.resolve(process.argv[2]); +const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); +const asset = (manifest.assets || []).find((entry) => entry.name.startsWith("clusterflux-hosted-")); +if (!asset) throw new Error("candidate manifest has no hosted archive"); +const file = path.isAbsolute(asset.file) + ? asset.file + : path.resolve(path.dirname(manifestPath), asset.file); +process.stdout.write(`${file}\t${asset.sha256}\n`); +NODE +) + +actual_archive_sha=$(sha256sum "$archive" | awk '{print $1}') +test "$actual_archive_sha" = "${expected_archive_sha#sha256:}" +tar -xzf "$archive" -C "$staging" +candidate_coordinator=$(find "$staging" -type f -name clusterflux-hosted-service -perm -u+x -print -quit) +test -n "$candidate_coordinator" +candidate_sha=$(sha256sum "$candidate_coordinator" | awk '{print $1}') + +export CLUSTERFLUX_CANDIDATE_ARCHIVE="$archive" +export CLUSTERFLUX_CANDIDATE_ARCHIVE_SHA256="sha256:$actual_archive_sha" +export CLUSTERFLUX_CANDIDATE_COORDINATOR="$candidate_coordinator" +export CLUSTERFLUX_CANDIDATE_COORDINATOR_SHA256="sha256:$candidate_sha" +bash -euo pipefail -c "$CLUSTERFLUX_DEPLOY_COMMAND" + +main_pid=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=MainPID --value "$service_unit") +test "$main_pid" != 0 +remote_executable=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" readlink -f "/proc/$main_pid/exe") +remote_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$remote_executable" | awk '{print $1}') +test "$remote_sha" = "$candidate_sha" +service_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$service_unit") +service_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$service_fragment" | awk '{print $1}') +service_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl cat "$service_unit" | sha256sum | awk '{print $1}') +proxy_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$proxy_unit") +proxy_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$proxy_fragment" | awk '{print $1}') +proxy_exec_start=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=ExecStart --value "$proxy_unit") +proxy_executable=$(sed -n 's/.*path=\([^ ;]*\).*/\1/p' <<<"$proxy_exec_start") +proxy_configuration=$(sed -n 's/.* argv\[\]=.* -c \([^ ;]*\).*/\1/p' <<<"$proxy_exec_start") +test -n "$proxy_executable" +test -n "$proxy_configuration" +proxy_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" "$proxy_executable" -T -c "$proxy_configuration" 2>/dev/null | sha256sum | awk '{print $1}') +mkdir -p "$(dirname "$evidence_path")" + +EVIDENCE_PATH="$evidence_path" SERVICE_UNIT="$service_unit" \ +SERVICE_FRAGMENT="$service_fragment" SERVICE_FRAGMENT_SHA="$service_fragment_sha" \ +SERVICE_CONFIGURATION_SHA="$service_configuration_sha" \ +REMOTE_EXECUTABLE="$remote_executable" REMOTE_SHA="$remote_sha" \ +PROXY_UNIT="$proxy_unit" PROXY_FRAGMENT="$proxy_fragment" \ +PROXY_FRAGMENT_SHA="$proxy_fragment_sha" PROXY_EXECUTABLE="$proxy_executable" \ +PROXY_CONFIGURATION="$proxy_configuration" PROXY_CONFIGURATION_SHA="$proxy_configuration_sha" node <<'NODE' +const fs = require("fs"); +fs.writeFileSync(process.env.EVIDENCE_PATH, JSON.stringify({ + kind: "clusterflux-exact-candidate-deployment", + service_unit: process.env.SERVICE_UNIT, + service_unit_fragment: process.env.SERVICE_FRAGMENT, + service_unit_sha256: `sha256:${process.env.SERVICE_FRAGMENT_SHA}`, + service_configuration_sha256: `sha256:${process.env.SERVICE_CONFIGURATION_SHA}`, + hosted_service_executable: process.env.REMOTE_EXECUTABLE, + hosted_service_sha256: `sha256:${process.env.REMOTE_SHA}`, + proxy_unit: process.env.PROXY_UNIT, + proxy_unit_fragment: process.env.PROXY_FRAGMENT, + proxy_unit_sha256: `sha256:${process.env.PROXY_FRAGMENT_SHA}`, + proxy_executable: process.env.PROXY_EXECUTABLE, + proxy_configuration: process.env.PROXY_CONFIGURATION, + proxy_configuration_sha256: `sha256:${process.env.PROXY_CONFIGURATION_SHA}`, +}, null, 2) + "\n"); +NODE diff --git a/scripts/flagship-demo-smoke.js b/scripts/flagship-demo-smoke.js new file mode 100644 index 0000000..98a43f2 --- /dev/null +++ b/scripts/flagship-demo-smoke.js @@ -0,0 +1,80 @@ +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const hello = path.join(repo, "examples/hello-build"); +const recovery = path.join(repo, "examples/recovery-build"); +const conformance = path.join(repo, "tests/fixtures/runtime-conformance"); +const source = fs.readFileSync(path.join(hello, "src/lib.rs"), "utf8"); +const nonblank = source.split(/\r?\n/).filter((line) => line.trim()).length; + +assert(nonblank <= 80, "hello-build source must stay below 80 nonblank lines"); +for (const forbidden of [ + "#[cfg", + "unsafe", + "extern \"C\"", + "no_mangle", + "sha256:", + ".task_id(", + "#[test]", + "unwrap()", + "expect(", +]) { + assert(!source.includes(forbidden), "hello-build leaked implementation detail: " + forbidden); +} +assert.strictEqual((source.match(/#\[clusterflux::task/g) || []).length, 1); +assert.strictEqual((source.match(/#\[clusterflux::main/g) || []).length, 1); +assert(source.includes("source::current_project().snapshot().await?")); +assert(source.includes("clusterflux::spawn!(compile(source))")); +assert(source.includes(".on(clusterflux::env!(\"linux\"))")); +assert(source.includes(".run()")); +assert(source.includes("fs::publish")); +assert(fs.existsSync(path.join(hello, "fixture/hello-clusterflux.c"))); +assert(fs.existsSync(path.join(hello, "envs/linux/Containerfile"))); + +const recoverySource = fs.readFileSync(path.join(recovery, "src/lib.rs"), "utf8"); +assert.strictEqual((recoverySource.match(/spawn!\(build_lane/g) || []).length, 2); +assert(recoverySource.includes("TaskFailurePolicy::AwaitOperator")); +assert(recoverySource.includes("\"exit 23\"")); +for (const forbidden of ["#[cfg", "unsafe", "extern \"C\"", "no_mangle", ".task_id("]) { + assert(!recoverySource.includes(forbidden), "recovery-build leaked " + forbidden); +} + +const fixtureSource = fs.readFileSync(path.join(conformance, "src/lib.rs"), "utf8"); +assert(fixtureSource.includes("task_trap")); +assert(fixtureSource.includes("cooperative_cancellation_probe")); +assert(!fs.existsSync(path.join(repo, "examples", "launch-" + "build-demo"))); +assert(!fs.existsSync(path.join(hello, "src/bin/sdk-product-runtime.rs"))); + +const args = [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "build", + "--project", + hello, + "--json", +]; +const report = JSON.parse(cp.execFileSync("cargo", args, { + cwd: repo, + env: process.env, + encoding: "utf8", +})); +assert(report.bundle_artifact); +assert(report.bundle.metadata.selected_inputs.some((input) => input.path === "src/lib.rs")); +assert(report.bundle.metadata.task_metadata.entrypoints.includes("build")); +const tasks = JSON.parse( + fs.readFileSync( + path.resolve(repo, report.bundle_artifact.directory, "task-descriptors.json"), + "utf8" + ) +); +assert(tasks.some((task) => task.name === "compile")); +assert(tasks.some((task) => task.name === "snapshot_current_project")); +console.log("primary and recovery example smoke passed"); diff --git a/scripts/hostile-input-contract-smoke.js b/scripts/hostile-input-contract-smoke.js new file mode 100644 index 0000000..0be6153 --- /dev/null +++ b/scripts/hostile-input-contract-smoke.js @@ -0,0 +1,184 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing hostile-input evidence: ${name}`); +} + +const coreSource = read("crates/clusterflux-core/src/source.rs"); +const coreCapabilities = read("crates/clusterflux-core/src/capability.rs"); +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/signed_nodes.rs"), + read("crates/clusterflux-coordinator/src/service/logs.rs"), + read("crates/clusterflux-coordinator/src/service/tests.rs"), +].join("\n"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); +const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); + +for (const [name, pattern] of [ + ["source manifests validate shape", /pub fn validate_public_mvp\(&self\)[\s\S]*self\.validate_shape\(\)\?/], + ["source manifests reject invalid digests", /SourceManifestError::InvalidDigest/], + ["source manifests reject invalid custom providers", /SourceManifestError::InvalidProviderId/], + ["source manifests reject control characters", /DescriptionControlCharacter/], + ["source manifests reject coordinator checkout access", /CoordinatorCheckoutAccess/], + ["source manifests reject default source-byte upload", /CoordinatorReceivesSourceBytes/], +]) { + expect(coreSource, name, pattern); +} + +for (const [name, pattern] of [ + ["capability reports validate public shape", /pub fn validate_public_report\(&self\)/], + ["capability reports validate architecture labels", /InvalidArchitecture/], + ["capability reports validate OS labels", /InvalidOsLabel/], + ["capability reports validate source providers", /InvalidSourceProvider/], + ["source provider ids reject path traversal", /valid_source_provider_id/], +]) { + expect(coreCapabilities, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator task log tails are bounded", /MAX_TASK_LOG_TAIL_BYTES: usize = 256 \* 1024/], + ["coordinator validates reported stdout tails", /ReportTaskLog[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], + ["coordinator validates completed task stdout tails", /TaskCompleted[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], + ["coordinator rejects oversized log tail in unit coverage", /"x"\.repeat\(MAX_TASK_LOG_TAIL_BYTES \+ 1\)/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, pattern] of [ + ["service rejects malformed node capability report", /fn service_rejects_malformed_node_capability_report\(\)/], + ["capability report rejection leaves descriptors empty", /assert!\(service\.node_descriptors\.is_empty\(\)\)/], + ["node capability report rejects cross-scope writes", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], + ["task completion rejects cross-scope writes", /task completion outside node scope|outside/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, source, patterns] of [ + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /const crossTenant = await send/, + /const crossProject = await send/, + /const guessed = await send/, + /const crossActorOpen = await send/, + /token is invalid/, + /tenant mismatch/, + /project mismatch/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [ + /render_operator_panel/, + /submit_panel_event/, + /assert\(!JSON\.stringify\(panel\)\.includes\(" max_bytes[\s\S]*contains unsupported characters/], + ["tokens are bounded", /fn validate_token[\s\S]*value\.len\(\) > max_bytes[\s\S]*contains unsupported characters/], + ["control request bodies are bounded", /MAX_CONTROL_FRAME_BYTES[\s\S]*control request too large/], + ["identity protocol rejects unknown authority fields", /deny_unknown_fields/], + ]) { + expect(hostedService, name, pattern); + } + + for (const [name, pattern] of [ + ["old client identity protocol is rejected", /hosted_login_protocol_rejects_client_identity_and_provider_configuration/], + ["raw operator action is rejected", /rawOperatorDenied[\s\S]*hosted_operator_request envelope/], + ["unsigned client identity is rejected", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], + ["cross-tenant process inspection is rejected", /crossTenantTaskEventsDenied[\s\S]*scope\|denied\|unauthorized/], + ]) { + expect(hostedTests, name, pattern); + } +} + +console.log("Hostile input contract smoke passed"); diff --git a/scripts/migrate-clusterflux-state.sh b/scripts/migrate-clusterflux-state.sh new file mode 100755 index 0000000..356091d --- /dev/null +++ b/scripts/migrate-clusterflux-state.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +migrate_path() { + local source="$1" + local destination="$2" + if [[ ! -e "$source" ]]; then + return + fi + if [[ -e "$destination" ]]; then + printf 'refusing to overwrite existing destination: %s\n' "$destination" >&2 + exit 1 + fi + mv -- "$source" "$destination" + printf 'migrated %s -> %s\n' "$source" "$destination" +} + +migrate_path "${HOME}/.disasmer" "${HOME}/.clusterflux" +migrate_path "${XDG_CONFIG_HOME:-${HOME}/.config}/disasmer" \ + "${XDG_CONFIG_HOME:-${HOME}/.config}/clusterflux" +migrate_path "${PWD}/.disasmer" "${PWD}/.clusterflux" +migrate_path "${PWD}/disasmer.toml" "${PWD}/clusterflux.toml" diff --git a/scripts/node-attach-smoke.js b/scripts/node-attach-smoke.js new file mode 100755 index 0000000..d6c4e01 --- /dev/null +++ b/scripts/node-attach-smoke.js @@ -0,0 +1,300 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); +const identities = new Map(); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function nodeIdentity(node) { + const existing = identities.get(node); + if (existing) return existing; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + const privateSeed = Buffer.from(privateDer).subarray(-32); + const identity = { + privateKey: `ed25519:${privateSeed.toString("base64")}`, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; + identities.set(node, identity); + return identity; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeHeartbeat(node, identity) { + const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const payloadDigest = `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify({ node, type: "node_heartbeat" })) + .digest("hex")}`; + const signature = crypto.sign( + null, + nodeSignatureMessage(node, "node_heartbeat", payloadDigest, nonce, issuedAt), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function runAttach(addr, grant) { + const identity = nodeIdentity("node-attach"); + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "node", + "attach", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + "node-attach", + "--public-key", + identity.publicKey, + "--enrollment-grant", + grant, + "--cap", + "quic-direct", + "--json", + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`node attach failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject( + new Error(`node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) + ); + } + }); + }); +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const grant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(grant.type, "node_enrollment_grant_created"); + assert.strictEqual(grant.tenant, "tenant"); + assert.strictEqual(grant.project, "project"); + assert.match(grant.grant, /^node_grant_[A-Za-z0-9_-]+$/); + assert.strictEqual(grant.scope, "node:attach"); + assert(grant.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); + assert(grant.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 900); + + const report = await runAttach(addr, grant.grant); + assert.strictEqual(report.plan.node, "node-attach"); + assert.strictEqual(report.plan.coordinator, `${addr.host}:${addr.port}`); + assert.strictEqual(report.plan.enrollment.grant, grant.grant); + assert.match(report.plan.enrollment.public_key_fingerprint, /^sha256:[0-9a-f]{64}$/); + assert.strictEqual( + report.plan.enrollment.exchanges_short_lived_grant_for_long_lived_node_identity, + true + ); + assert.ok(report.plan.capabilities.arch.length > 0); + assert.ok(report.plan.capabilities.source_providers.includes("filesystem")); + assert.ok(report.plan.capabilities.capabilities.includes("QuicDirect")); + assert.strictEqual(report.plan.detection.auto_detected, true); + assert.strictEqual(report.plan.detection.arch, report.plan.capabilities.arch); + assert.deepStrictEqual(report.plan.detection.manual_capability_overrides, ["quic-direct"]); + assert( + report.plan.detection.recognized_capability_overrides.includes("QuicDirect") + ); + assert.strictEqual( + report.plan.detection.os_arch_capabilities_require_manual_flags, + false + ); + assert.strictEqual(report.plan.detection.command_backend, "native-command"); + assert.strictEqual(report.plan.detection.command_backend_available, true); + assert( + report.plan.detection.source_provider_backends.some( + (provider) => provider.provider === "filesystem" && provider.detected + ) + ); + assert( + report.grant_disclosures.length > 0, + "node attach should disclose capability grants before reporting capabilities" + ); + assert( + report.grant_disclosures.every( + (disclosure) => disclosure.coordinator_policy_limited === true + ), + "node attach should mark all capability grants as coordinator-policy-limited" + ); + assert( + report.grant_disclosures.some( + (disclosure) => disclosure.grant === "native_command_execution" + ), + "node attach should disclose native command execution when detected" + ); + assert( + report.grant_disclosures.some( + (disclosure) => disclosure.grant === "source_access" + ), + "node attach should disclose source access when detected" + ); + assert.strictEqual(report.boundary.cli_contacted_coordinator, true); + assert.strictEqual(report.boundary.used_enrollment_exchange, true); + assert.strictEqual(report.boundary.coordinator_session_requests, 3); + assert.strictEqual(report.coordinator_response.type, "node_enrollment_exchanged"); + assert.strictEqual(report.coordinator_response.node, "node-attach"); + assert.strictEqual(report.coordinator_response.credential.node, "node-attach"); + assert.strictEqual(report.coordinator_response.credential.scope, "node:attach"); + assert.strictEqual(report.coordinator_response.credential.credential_kind, "NodeCredential"); + assert.match( + report.coordinator_response.credential.capability_policy_digest, + /^sha256:[0-9a-f]{64}$/ + ); + assert.strictEqual(report.heartbeat_response.type, "node_heartbeat"); + assert.strictEqual(report.capability_response.type, "node_capabilities_recorded"); + + const heartbeat = await send(addr, { + type: "node_heartbeat", + node: "node-attach", + node_signature: signedNodeHeartbeat("node-attach", nodeIdentity("node-attach")), + }); + assert.strictEqual(heartbeat.type, "node_heartbeat"); + assert.strictEqual(heartbeat.node, "node-attach"); + + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Node attach smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/node-lifecycle-contract-smoke.js b/scripts/node-lifecycle-contract-smoke.js new file mode 100755 index 0000000..b597da9 --- /dev/null +++ b/scripts/node-lifecycle-contract-smoke.js @@ -0,0 +1,151 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing node lifecycle evidence: ${name}`); +} + +const nodeMain = read("crates/clusterflux-node/src/daemon.rs"); +const nodeIdentity = read("crates/clusterflux-node/src/node_identity.rs"); +const cliNode = read("crates/clusterflux-cli/src/node.rs"); +const nodeTaskReports = read("crates/clusterflux-node/src/task_reports.rs"); +const nodeDebugAgent = read("crates/clusterflux-node/src/debug_agent.rs"); +const nodeLib = read("crates/clusterflux-node/src/lib.rs"); +const sharedWasmtimeRuntime = `${read("crates/clusterflux-wasm-runtime/src/lib.rs")}\n${read("crates/clusterflux-wasm-runtime/src/task_host_linker.rs")}`; +const nodeRuntimeSurface = `${nodeLib}\n${sharedWasmtimeRuntime}`; +const nodeLifecycleSurface = `${nodeMain}\n${nodeIdentity}\n${nodeTaskReports}\n${nodeDebugAgent}`; +const nodeAssignmentRunner = `${read("crates/clusterflux-node/src/assignment_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/control_watcher.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/process_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/validation.rs")}`; +const coordinatorCore = read("crates/clusterflux-coordinator/src/lib.rs"); +const coordinatorService = `${read("crates/clusterflux-coordinator/src/service.rs")}\n${read("crates/clusterflux-coordinator/src/service/routing.rs")}`; +const coordinatorServiceTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); +const coordinatorServiceSurface = `${coordinatorService}\n${coordinatorServiceTests}`; +const cliLocalRunSmoke = read("scripts/cli-local-run-smoke.js"); +const liveSmoke = read("scripts/cli-happy-path-live-smoke.js"); +const wasmtimeSmoke = read("scripts/wasmtime-node-smoke.js"); +const debugCore = read("crates/clusterflux-core/src/debug.rs"); + +assert.strictEqual( + (nodeMain.match(/CoordinatorSession::connect/g) || []).length, + 1, + "node runtime should open one coordinator session in the local process-boundary runtime" +); + +for (const [name, pattern] of [ + ["enrollment exchange over session", /"type": "exchange_node_enrollment_grant"/], + ["persisted node identity is reused locally", /"type": "node_identity_reused"/], + ["heartbeat over session", /"type": "node_heartbeat"/], + ["node-originated requests use signed envelope", /"type": "signed_node"/], + ["capability report over session", /"type": "report_node_capabilities"/], + ["task assignment polling over session", /"type": "poll_task_assignment"/], + ["process start over session", /"type": "start_process"/], + ["reconnect over session", /"type": "reconnect_node"/], + ["debug command polling over session", /"type": "poll_debug_command"/], + ["log event over session", /"type": "report_task_log"/], + ["VFS metadata over session", /"type": "report_vfs_metadata"/], + ["task control polling over session", /"type": "poll_task_control"/], + ["completion over session", /"type": "task_completed"/], + ["cancellation uses same session", /poll_task_cancellation\(session, args, &task, node_private_key\)/], + ["request count is reported", /session\.requests\(\)/], +]) { + expect(nodeLifecycleSurface, name, pattern); +} + +expect(cliNode, "user-authorized node attach over Client session", /"type": "attach_node"/); +assert.doesNotMatch( + nodeIdentity, + /"type": "attach_node"/, + "a persisted node must authenticate with its signed identity instead of replaying Client attach" +); + +const runtimeAcceptanceSurface = `${cliLocalRunSmoke}\n${liveSmoke}\n${coordinatorServiceTests}\n${nodeLib}\n${nodeAssignmentRunner}`; +for (const [name, pattern] of [ + ["real CLI launches a coordinator main", /node_report\.run\.status, "main_launched"/], + ["real CLI observes coordinator-main task spawning", /task_spawn_host_import, true/], + ["real CLI keeps child task instances distinct", /every live task event must retain its unique instance identity/], + ["signed active Wasm parent spawns and joins child", /fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only\(\)/], + ["controlled native process abort is exercised", /fn abort_requested[\s\S]*poll_task_control/], + ["native lifecycle freeze and resume are exercised", /linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume/], + ["strict live run requires a usable partial freeze", /partial_freeze\.partially_frozen/], + ["strict live run requires hosted restart evidence", /serviceRestart\.executed === true/], +]) { + expect(runtimeAcceptanceSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["cooperative cancellation and abort are distinct", /cancel_requested: false,[\s\S]*abort_requested: true/], + ["controlled runner polls abort while command runs", /fn abort_requested[\s\S]*poll_task_control/], + ["controlled runner creates a process group", /process\.process_group\(0\)/], + ["controlled runner freezes the native process group", /libc::kill\(process_group, libc::SIGSTOP\)/], + ["controlled runner resumes the native process group", /libc::kill\(process_group, libc::SIGCONT\)/], + ["controlled runner kills the process group", /libc::kill\(process_group, libc::SIGKILL\)/], + ["Wasm code can poll cooperative cancellation", /task_control_v1/], + ["matched Wasm probes remain at a quiescent boundary", /TaskHostOperation::DebugProbe[\s\S]*enter_quiescent_host_boundary[\s\S]*leave_quiescent_host_boundary/], + ["debug snapshots use the live Wasm task handle registry", /debug_handle_snapshot[\s\S]*task_handle_\{handle_id\}[\s\S]*state=active/], + ["native command status comes from the controlled runner", /set_command_status[\s\S]*frozen command pid[\s\S]*native command exited with status/], +]) { + expect(`${coordinatorServiceSurface}\n${nodeLifecycleSurface}\n${sharedWasmtimeRuntime}\n${nodeAssignmentRunner}`, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/], + ["reconnect preserves enrolled node identity", /reconnect_node\(&NodeId::from\("node"\), None\)/], + ["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/], +]) { + expect(coordinatorCore, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator delivers cancellation to connected node", /fn service_delivers_cancellation_to_connected_node_and_records_terminal_state\(\)/], + ["node polls task control", /CoordinatorRequest::PollTaskControl/], + ["cancelled terminal state is recorded", /TaskTerminalState::Cancelled/], +]) { + expect(coordinatorServiceSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["native lifecycle test exists", /fn linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume\(\)/], + ["native freeze succeeds when supported", /lifecycle\.freeze_for_debug_epoch\(\)\.unwrap\(\)/], + ["native resume succeeds", /lifecycle\.resume_after_debug_epoch\(\)/], + ["native cancel reaches lifecycle", /lifecycle\.cancel\(\)/], + ["unsupported freeze errors", /BackendError::DebugFreezeUnsupported/], + ["wasmtime runtime exposes freeze resume probe", /pub fn freeze_resume_i32_export_probe/], + ["wasmtime runtime captures Wasm frame locals", /debug_i32_export_snapshot[\s\S]*local_values/], + ["wasmtime runtime creates Wasm debug participant", /kind: DebugParticipantKind::WasmTask/], + ["wasmtime debug participant carries local values", /local_values: snapshot\.local_values\.clone\(\)/], + ["wasmtime runtime resumes after freeze", /epoch\.continue_all\(\)/], +]) { + expect(nodeRuntimeSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["wasmtime smoke runs debug freeze resume mode", /--debug-freeze-resume/], + ["wasmtime smoke verifies frozen state", /debugReport\.frozen_state, "Frozen"/], + ["wasmtime smoke verifies resumed state", /debugReport\.resumed_state, "Running"/], + ["wasmtime smoke verifies frame local values", /debugReport\.local_values[\s\S]*wasm_local_0/], + ["wasmtime smoke proves node runtime reached wasm task", /node_runtime_reached_wasm_task/], + ["wasmtime smoke proves node captured locals", /node_runtime_captured_wasm_locals/], +]) { + expect(wasmtimeSmoke, name, pattern); +} + +for (const [name, pattern] of [ + ["debug model freezes wasm and command participants", /fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks\(\)/], + ["debug model rejects unsupported freeze", /fn debug_epoch_reports_failure_when_no_participant_can_freeze\(\)/], + ["debug model resumes frozen participants", /fn continue_resumes_every_frozen_participant\(\)/], + ["debug model includes captured locals", /local_values/], + ["wasm participants are modeled", /DebugParticipantKind::WasmTask/], + ["controlled native command participants are modeled", /DebugParticipantKind::ControlledNativeCommand/], +]) { + expect(debugCore, name, pattern); +} + +console.log("Node lifecycle contract smoke passed"); diff --git a/scripts/node-signing.js b/scripts/node-signing.js new file mode 100644 index 0000000..e50a27a --- /dev/null +++ b/scripts/node-signing.js @@ -0,0 +1,181 @@ +const crypto = require("crypto"); +const identities = new Map(); + +function nodeIdentity(identityPurpose, node) { + const identityKey = `${identityPurpose}:${node}`; + const existing = identities.get(identityKey); + if (existing) return existing; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + const privateSeed = Buffer.from(privateDer).subarray(-32); + const identity = { + privateKey: `ed25519:${privateSeed.toString("base64")}`, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; + identities.set(identityKey, identity); + return identity; +} + +function nodeIdentityFromPrivateKey(privateKey) { + if (typeof privateKey !== "string" || !privateKey.startsWith("ed25519:")) { + throw new Error("node private key must use ed25519: encoding"); + } + const seed = Buffer.from(privateKey.slice("ed25519:".length), "base64"); + if (seed.length !== 32) throw new Error("node private key must contain 32 bytes"); + const privateKeyObject = crypto.createPrivateKey({ + key: Buffer.concat([ + Buffer.from("302e020100300506032b657004220420", "hex"), + seed, + ]), + format: "der", + type: "pkcs8", + }); + const publicKeyObject = crypto.createPublicKey(privateKeyObject); + const publicDer = publicKeyObject.export({ format: "der", type: "spki" }); + return { + privateKey, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; +} + +function canonicalSignedRequest(value, topLevel = true) { + if (Array.isArray(value)) { + return value.map((entry) => canonicalSignedRequest(entry, false)); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter( + ([key, entry]) => + entry !== null && + (!topLevel || !["agent_signature", "node_signature"].includes(key)) + ) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalSignedRequest(entry, false)]) + ); + } + return value; +} + +function withWireDefaults(request) { + const value = { ...request }; + if (value.type === "report_node_capabilities") { + value.dependency_cache_digests ??= []; + } else if (value.type === "launch_task" || value.type === "launch_child_task") { + value.wait_for_node ??= false; + if (value.task_spec && typeof value.task_spec === "object") { + value.task_spec = { ...value.task_spec }; + value.task_spec.failure_policy ??= "fail_fast"; + } + } else if (value.type === "start_process") { + value.restart ??= false; + } else if (value.type === "report_debug_state") { + value.stack_frames ??= []; + value.local_values ??= []; + value.task_args ??= []; + value.handles ??= []; + value.recent_output ??= []; + } else if (value.type === "report_task_log") { + value.stdout_tail ??= ""; + value.stderr_tail ??= ""; + } else if (value.type === "task_completed") { + value.stdout_tail ??= ""; + value.stderr_tail ??= ""; + value.stdout_truncated ??= false; + value.stderr_truncated ??= false; + } + return value; +} + +function signedRequestPayloadDigest(request) { + return `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify(canonicalSignedRequest(withWireDefaults(request)))) + .digest("hex")}`; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeProof(node, identity, requestKind, request, options = {}) { + const nonce = + options.nonce || + `${requestKind}-${process.pid}-${Date.now()}-${crypto + .randomBytes(8) + .toString("hex")}`; + const issuedAt = + options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + requestKind, + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedNodeHeartbeat(node, identity, options = {}) { + const request = { type: "node_heartbeat", node }; + return signedNodeProof(node, identity, "node_heartbeat", request, options); +} + +function signedNodeRequest(node, identity, requestKind, request, options = {}) { + return { + type: "signed_node", + node, + node_signature: signedNodeProof( + node, + identity, + requestKind, + request, + options + ), + request, + }; +} + +module.exports = { + nodeIdentity, + nodeIdentityFromPrivateKey, + signedNodeProof, + signedRequestPayloadDigest, + signedNodeHeartbeat, + signedNodeRequest, +}; diff --git a/scripts/operator-panel-smoke.js b/scripts/operator-panel-smoke.js new file mode 100755 index 0000000..b817c42 --- /dev/null +++ b/scripts/operator-panel-smoke.js @@ -0,0 +1,385 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const path = require("path"); +const { nodeIdentity } = require("./node-signing"); +const { + ensureRootlessPodman, + repo, + runFlagshipWorker, + send, + startFlagship, + waitForTaskEvent, + waitForJsonLine, + waitForNodeStatus, +} = require("./real-flagship-harness"); + +const panelNode = "panel-node"; +const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode); +const panelProject = path.join(repo, "tests/fixtures/runtime-conformance"); + +function widget(panel, id) { + const item = panel.widgets[id]; + assert(item, `missing panel widget ${id}`); + return item; +} + +const delay = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForBreakpointHit(addr, process) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const status = await send(addr, { + type: "inspect_debug_breakpoints", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + if (status.type !== "debug_breakpoints") { + const events = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + throw new Error( + `breakpoint state disappeared: ${JSON.stringify({ status, events })}` + ); + } + if (status.hit_epoch != null) return status; + await delay(25); + } + throw new Error(`timed out waiting for package breakpoint in ${process}`); +} + +async function waitForDebugEpochFrozen(addr, process, epoch) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const status = await send(addr, { + type: "inspect_debug_epoch", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + epoch, + }); + assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status)); + if (status.failed) { + throw new Error(status.failure_messages.join("; ")); + } + if (status.fully_frozen) return status; + await delay(25); + } + throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + let worker; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + const projectCreated = await send(addr, { + type: "create_project", + tenant: "tenant", + actor_user: "user", + project: "project", + name: "Operator panel smoke", + }); + assert.strictEqual(projectCreated.type, "project_created"); + + worker = await runFlagshipWorker( + addr, + panelNode, + panelNodeIdentity, + panelProject + ); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + const workerCompletion = waitForNodeStatus(worker.child, "completed"); + const flagship = startFlagship(addr, panelProject); + const configuredBreakpoints = await send(addr, { + type: "set_debug_breakpoints", + tenant: "tenant", + project: "project", + actor_user: "user", + process: flagship.process, + probe_symbols: ["clusterflux.probe.package_release"], + }); + assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints"); + await waitForTaskEvent( + addr, + flagship.process, + (event) => event.task_definition === "prepare_source", + "prepare_source after breakpoint configuration" + ); + const breakpointHit = await waitForBreakpointHit(addr, flagship.process); + assert.strictEqual( + breakpointHit.hit_probe_symbol, + "clusterflux.probe.package_release" + ); + const frozenEpoch = await waitForDebugEpochFrozen( + addr, + flagship.process, + breakpointHit.hit_epoch + ); + assert(frozenEpoch.acknowledgements.length >= 2); + const compileEvent = await waitForTaskEvent( + addr, + flagship.process, + (event) => event.task_definition === "compile_linux", + "compile before the package breakpoint" + ); + const report = await workerCompletion; + assert.strictEqual(report.node_status, "completed"); + assert.strictEqual(report.coordinator_response.type, "task_recorded"); + const process = flagship.process; + + const rendered = await send(addr, { + type: "render_operator_panel", + tenant: "tenant", + project: "project", + process, + actor_user: "user", + max_download_bytes: 1024 * 1024, + stopped: false + }); + assert.strictEqual(rendered.type, "operator_panel"); + const panel = rendered.panel; + assert.strictEqual(panel.tenant, "tenant"); + assert.strictEqual(panel.project, "project"); + assert.strictEqual(panel.process, process); + assert.strictEqual(panel.program_ui_events_enabled, true); + + assert.deepStrictEqual(widget(panel, "process-status").kind, { + Text: { value: "running" } + }); + const taskProgress = widget(panel, "task-progress").kind.Progress; + assert(taskProgress.current >= 2); + assert.strictEqual(taskProgress.current, taskProgress.total); + const taskSummary = widget(panel, "task-summary").kind.Text.value; + assert.match( + taskSummary, + new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`) + ); + assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/); + const downloadWidget = widget(panel, "download-artifact").kind; + const artifact = downloadWidget.ArtifactDownload.artifact; + assert( + artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length), + "panel download must point at a real flagship artifact" + ); + assert(!JSON.stringify(downloadWidget).includes("url_path")); + assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest")); + assert.deepStrictEqual(widget(panel, "debug-process").kind, { + Button: { action: "debug-process" } + }); + assert.deepStrictEqual(widget(panel, "cancel-process").kind, { + Button: { action: "cancel-process" } + }); + assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, { + Button: { action: "restart-task" } + }); + assert(panel.control_plane_actions.includes("DebugProcess")); + assert(panel.control_plane_actions.includes("CancelProcess")); + const restartTarget = panel.control_plane_actions.find( + (action) => action.RestartTask + )?.RestartTask; + assert( + restartTarget && taskSummary.includes(`[${restartTarget}]`), + "panel restart must target the real flagship task instance" + ); + assert( + panel.control_plane_actions.some( + (action) => action.DownloadArtifact === artifact + ) + ); + assert(!JSON.stringify(panel).includes(" action.DownloadArtifact === artifact + ) + ); + + const frozenEvent = await send(addr, { + type: "submit_panel_event", + tenant: "tenant", + project: "project", + process, + widget_id: "debug-process", + kind: "ButtonClicked", + max_events: 10 + }); + assert.strictEqual(frozenEvent.type, "error"); + assert.match(frozenEvent.message, /program UI events are disabled/i); + + const crossTenant = await send(addr, { + type: "render_operator_panel", + tenant: "other", + project: "project", + process, + actor_user: "user", + max_download_bytes: 1024 * 1024, + stopped: false + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match( + crossTenant.message, + /scope|tenant|project|requires an active virtual process/i + ); + assert(!crossTenant.message.includes(process)); + + const resumed = await send(addr, { + type: "resume_debug_epoch", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + epoch: breakpointHit.hit_epoch, + }); + assert.strictEqual(resumed.type, "debug_epoch"); + const cleanup = await send(addr, { + type: "abort_process", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + assert.strictEqual(cleanup.type, "process_aborted"); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Operator panel smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/podman-backend-smoke.js b/scripts/podman-backend-smoke.js new file mode 100755 index 0000000..438c2df --- /dev/null +++ b/scripts/podman-backend-smoke.js @@ -0,0 +1,86 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const path = require("path"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +const baseImage = "docker.io/library/alpine:3.20"; + +// Nix's standalone Podman package may not install the distribution-level +// containers/image policy normally found under /etc. Use an isolated test HOME +// without replacing a policy supplied by the host. +configurePodmanTestEnvironment(repo); + +function run(command, args, options = {}) { + return cp.execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: options.stdio || ["ignore", "pipe", "pipe"], + }); +} + +function incomplete(reason) { + const error = new Error(`Linux Podman backend incomplete: ${reason}`); + error.code = "CLUSTERFLUX_PODMAN_INCOMPLETE"; + throw error; +} + +function ensurePodmanBaseImage() { + try { + run("podman", ["--version"]); + } catch (error) { + incomplete(`podman command is unavailable (${error.message})`); + } + + let rootless; + try { + rootless = run("podman", ["info", "--format", "{{.Host.Security.Rootless}}"]).trim(); + } catch (error) { + incomplete(`podman info did not report rootless status (${error.message})`); + } + if (rootless !== "true") { + incomplete(`podman is not running in rootless mode (reported ${JSON.stringify(rootless)})`); + } + + try { + run("podman", ["image", "exists", baseImage]); + } catch (_) { + try { + run("podman", ["pull", baseImage], { stdio: "inherit" }); + } catch (error) { + incomplete(`unable to make ${baseImage} available (${error.message})`); + } + } +} + +try { + ensurePodmanBaseImage(); + + const stdout = run("cargo", [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-podman-smoke" + ]); + const report = JSON.parse(stdout.trim().split("\n").at(-1)); + + assert.strictEqual(report.podman_status, "completed"); + assert.strictEqual(report.status_code, 0); + assert.strictEqual(report.stdout, "podman-ok:node-local source\n"); + assert.strictEqual(report.large_bytes_uploaded, false); + assert.strictEqual(report.uses_full_repo_tarball, false); + assert.strictEqual(report.coordinator_routed_file_reads, false); + assert.strictEqual(report.staged_artifact.path, "/vfs/artifacts/podman-smoke.txt"); + + console.log("Podman backend smoke passed"); +} catch (error) { + if (error.code === "CLUSTERFLUX_PODMAN_INCOMPLETE") { + console.error(error.message); + process.exit(2); + } + throw error; +} diff --git a/scripts/podman-test-env.js b/scripts/podman-test-env.js new file mode 100644 index 0000000..2c0dc3b --- /dev/null +++ b/scripts/podman-test-env.js @@ -0,0 +1,41 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +function configurePodmanTestEnvironment(repo) { + const originalHome = os.homedir(); + if ( + fs.existsSync(path.join(originalHome, ".config/containers/policy.json")) || + fs.existsSync("/etc/containers/policy.json") + ) { + return; + } + + const isolatedHome = path.join(repo, "scripts/containers-home"); + const policy = path.join(isolatedHome, ".config/containers/policy.json"); + fs.mkdirSync(path.dirname(policy), { recursive: true }); + if (!fs.existsSync(policy)) { + fs.writeFileSync( + policy, + `${JSON.stringify( + { default: [{ type: "insecureAcceptAnything" }] }, + null, + 2 + )}\n` + ); + } + + process.env.CARGO_HOME ||= path.join(originalHome, ".cargo"); + process.env.RUSTUP_HOME ||= path.join(originalHome, ".rustup"); + process.env.HOME = isolatedHome; + process.env.XDG_DATA_HOME = path.join( + os.tmpdir(), + `clusterflux-containers-data-${process.getuid?.() ?? "user"}` + ); + process.env.XDG_CACHE_HOME = path.join( + os.tmpdir(), + `clusterflux-containers-cache-${process.getuid?.() ?? "user"}` + ); +} + +module.exports = { configurePodmanTestEnvironment }; diff --git a/scripts/prepare-public-release.js b/scripts/prepare-public-release.js new file mode 100755 index 0000000..eca582e --- /dev/null +++ b/scripts/prepare-public-release.js @@ -0,0 +1,1224 @@ +#!/usr/bin/env node + +const crypto = require("crypto"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const outputRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const publicTree = path.join(outputRoot, "public-tree"); +const assetsDir = path.join(outputRoot, "assets"); +const stagingDir = path.join(outputRoot, "staging"); +const publicBuildTarget = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_BUILD_TARGET_DIR || + path.join(outputRoot, "cargo-target") +); +const hostedBuildTarget = path.resolve( + process.env.CLUSTERFLUX_HOSTED_BUILD_TARGET_DIR || + path.join(outputRoot, "hosted-cargo-target") +); +const candidateManifestPath = process.env.CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST + ? path.resolve(process.env.CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST) + : null; +const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; +const forgejoHost = "git.michelpaulissen.com"; +const args = new Set(process.argv.slice(2)); +const includeForgejoWorkflows = + args.has("--include-forgejo-workflows") || + /^(1|true|yes)$/i.test(process.env.CLUSTERFLUX_INCLUDE_FORGEJO_WORKFLOWS || ""); +const filteredTopLevel = ["private", "internal", "experiments", ".git", "target"]; +const filteredDirectoryNames = [".clusterflux"]; +const archiveIgnoredPathFallbacks = [ + "target", + ".clusterflux", + "vscode-extension/node_modules", + "scripts/containers-home", +]; +const publicRepoBranch = process.env.CLUSTERFLUX_PUBLIC_REPO_BRANCH || "main"; +const publicBinaries = [ + "clusterflux", + "clusterflux-coordinator", + "clusterflux-node", + "clusterflux-debug-dap", +]; +const hostedBinary = "clusterflux-hosted-service"; + +function commandOutput(command, args, options = {}) { + try { + const execOptions = { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + ...options, + }; + execOptions.env = commandEnv(command, options.env); + return cp + .execFileSync(command, args, execOptions) + .trim(); + } catch (_) { + return null; + } +} + +function run(command, args, options = {}) { + const execOptions = { + cwd: repo, + stdio: "inherit", + ...options, + }; + execOptions.env = commandEnv(command, options.env); + cp.execFileSync(command, args, execOptions); +} + +function nonInteractiveGitEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function commandEnv(command, env) { + if (command !== "git") { + return env; + } + return nonInteractiveGitEnv(env); +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function filteredOutPatterns() { + return [ + "private/**", + "internal/**", + "experiments/**", + ".git", + "target", + "git-ignored source paths", + "root/*.md except README.md", + "**/.clusterflux/**", + ...(includeForgejoWorkflows ? [] : [".forgejo/**"]), + ]; +} + +function gitIgnoredSourcePaths() { + const output = commandOutput("git", [ + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + ]); + if (output === null) { + return archiveIgnoredPathFallbacks; + } + return [ + ...new Set([ + ...archiveIgnoredPathFallbacks, + ...output + .split("\0") + .filter(Boolean) + .map((relativePath) => + relativePath.replaceAll("\\", "/").replace(/\/$/, "") + ), + ]), + ]; +} + +function isGitIgnored(relativePath, ignoredSourcePaths) { + const normalized = relativePath.replaceAll(path.sep, "/"); + return ignoredSourcePaths.some( + (ignored) => normalized === ignored || normalized.startsWith(`${ignored}/`) + ); +} + +function isFilteredRootMarkdown(relativePath, entry) { + return ( + !relativePath.includes(path.sep) && + (entry.isFile() || entry.isSymbolicLink()) && + path.extname(entry.name).toLowerCase() === ".md" && + !["README.md", "SECURITY.md"].includes(entry.name) + ); +} + +function shouldFilter(entry, relativePath) { + const parts = relativePath.split(path.sep).filter(Boolean); + const topLevel = parts[0]; + if (filteredTopLevel.includes(topLevel)) return true; + if (parts.some((part) => filteredDirectoryNames.includes(part))) return true; + if (topLevel === ".forgejo" && !includeForgejoWorkflows) return true; + if (isFilteredRootMarkdown(relativePath, entry)) return true; + return false; +} + +function copyFilteredTree(src, dest, ignoredSourcePaths, relative = "") { + ensureDir(dest); + const entries = fs + .readdirSync(src, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)); + + for (const entry of entries) { + const childRelative = relative ? path.join(relative, entry.name) : entry.name; + if ( + shouldFilter(entry, childRelative) || + isGitIgnored(childRelative, ignoredSourcePaths) + ) { + continue; + } + + const from = path.join(src, entry.name); + const to = path.join(dest, entry.name); + if (entry.isDirectory()) { + copyFilteredTree(from, to, ignoredSourcePaths, childRelative); + } else if (entry.isSymbolicLink()) { + fs.symlinkSync(fs.readlinkSync(from), to); + } else if (entry.isFile()) { + fs.copyFileSync(from, to); + fs.chmodSync(to, fs.statSync(from).mode & 0o777); + } + } +} + +function assertFilteredTree(ignoredSourcePaths) { + for (const excluded of ["private", "internal", "experiments"]) { + if (fs.existsSync(path.join(publicTree, excluded))) { + throw new Error(`${excluded}/ leaked into the public tree`); + } + } + for (const file of walkFiles(publicTree)) { + if (file.split(path.sep).includes(".clusterflux")) { + throw new Error(`generated .clusterflux view state leaked into the public tree: ${file}`); + } + } + if (!includeForgejoWorkflows && fs.existsSync(path.join(publicTree, ".forgejo"))) { + throw new Error(".forgejo/ leaked into the host-neutral public tree"); + } + if (!fs.existsSync(path.join(publicTree, "README.md"))) { + throw new Error("product README.md is missing from the public tree"); + } + for (const ignored of ignoredSourcePaths) { + if (fs.existsSync(path.join(publicTree, ...ignored.split("/")))) { + throw new Error(`Git-ignored source path leaked into the public tree: ${ignored}`); + } + } + for (const entry of fs.readdirSync(publicTree, { withFileTypes: true })) { + if (isFilteredRootMarkdown(entry.name, entry)) { + throw new Error(`internal root Markdown file leaked into the public tree: ${entry.name}`); + } + } +} + +function walkFiles(root, relative = "") { + const dir = path.join(root, relative); + const entries = fs + .readdirSync(dir, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)); + const files = []; + for (const entry of entries) { + const childRelative = relative ? path.join(relative, entry.name) : entry.name; + if (entry.isDirectory()) { + files.push(...walkFiles(root, childRelative)); + } else if (entry.isFile()) { + files.push(childRelative); + } else if (entry.isSymbolicLink()) { + files.push(childRelative); + } + } + return files; +} + +function hashTree(root) { + const hash = crypto.createHash("sha256"); + for (const file of walkFiles(root)) { + const absolute = path.join(root, file); + const stat = fs.lstatSync(absolute); + hash.update(file.replaceAll(path.sep, "/")); + hash.update("\0"); + hash.update(String(stat.mode & 0o777)); + hash.update("\0"); + if (stat.isSymbolicLink()) { + hash.update("symlink"); + hash.update("\0"); + hash.update(fs.readlinkSync(absolute)); + } else { + hash.update(fs.readFileSync(absolute)); + } + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function sha256File(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function sha256Buffer(buffer) { + return crypto.createHash("sha256").update(buffer).digest("hex"); +} + +function sourceTreeDigest(sourceCommit) { + const archive = cp.execFileSync("git", ["archive", "--format=tar", sourceCommit], { + cwd: repo, + encoding: null, + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "inherit"], + env: nonInteractiveGitEnv(), + }); + return `sha256:${sha256Buffer(archive)}`; +} + +function evidenceIdentity(value) { + const serialized = JSON.stringify(value === undefined ? null : value); + return `sha256:${sha256Buffer(Buffer.from(serialized, "utf8"))}`; +} + +function sanitizeEvidence(value, key = "") { + const secretKey = /(?:token|secret|password|cookie|authorization|private_key)/iu.test( + key + ); + if (secretKey && typeof value === "string") return "[redacted]"; + if (secretKey && Array.isArray(value)) return ["[redacted]"]; + if (Array.isArray(value)) { + return value.map((entry) => sanitizeEvidence(entry)); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([childKey, childValue]) => [ + childKey, + sanitizeEvidence(childValue, childKey), + ]) + ); + } + return value; +} + +function renderFinalTranscript(result) { + const lines = [ + "Clusterflux final release transcript", + `Source commit: ${result.source_commit}`, + `Source tree digest: ${result.source_tree_digest}`, + `Public tree identity: ${result.public_tree_identity}`, + `Acceptance result recorded: ${result.timestamps.acceptance_result_recorded_at}`, + `Evidence packaged: ${result.timestamps.evidence_packaged_at}`, + `Deployment generation: ${result.deployment.system_generation}`, + `Deployment identity: ${result.deployment.identity}`, + `Configuration identity: ${result.configuration_identity}`, + "", + "Commands:", + ...result.release_commands.map((command) => `- ${command}`), + "", + "Binary digests:", + ...Object.entries(result.binary_digests).map( + ([name, digest]) => `- ${name}: ${digest}` + ), + "", + "Named scenarios:", + ...result.named_scenarios.map( + (scenario) => + `- ${scenario.id}: ${scenario.status} (${scenario.duration_ms} ms)` + ), + "", + "Strict requirement ledger:", + ...result.strict_requirement_ledger.map( + (requirement) => + `- ${requirement.id}: ${requirement.passed ? "passed" : "failed"}` + ), + "", + `Scenario skips: ${JSON.stringify(result.scenario_skips)}`, + `Acceptance result: ${result.acceptance_result}`, + "", + "Failure-injection and complete machine evidence:", + JSON.stringify(result, null, 2), + "", + ]; + return `${lines.join("\n")}\n`; +} + +function tarGz(output, cwd, inputs) { + run("tar", ["-czf", output, "-C", cwd, ...inputs]); +} + +function platformName() { + return `${os.platform()}-${os.arch()}`; +} + +function binaryName(name) { + return process.platform === "win32" ? `${name}.exe` : name; +} + +function buildPublicBinaries() { + run("cargo", ["build", "--locked", "--workspace", "--bins", "--release", "--jobs", "2"], { + cwd: publicTree, + env: { ...process.env, CARGO_TARGET_DIR: publicBuildTarget }, + }); +} + +function buildHostedBinary() { + run( + "cargo", + [ + "build", + "--locked", + "--manifest-path", + "private/hosted-policy/Cargo.toml", + "--bin", + hostedBinary, + "--release", + "--jobs", + "2", + ], + { + cwd: repo, + env: { ...process.env, CARGO_TARGET_DIR: hostedBuildTarget }, + } + ); +} + +function stageBinaryAssets(releaseName) { + const stageRoot = path.join(stagingDir, "binaries"); + const binDir = path.join(stageRoot, "bin"); + fs.rmSync(stageRoot, { recursive: true, force: true }); + ensureDir(binDir); + + for (const binary of publicBinaries) { + const fileName = binaryName(binary); + const built = path.join(publicBuildTarget, "release", fileName); + if (!fs.existsSync(built)) { + throw new Error(`expected release binary ${built}`); + } + const staged = path.join(binDir, fileName); + fs.copyFileSync(built, staged); + fs.chmodSync(staged, 0o755); + } + + const archive = path.join( + assetsDir, + `clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz` + ); + tarGz(archive, stageRoot, ["."]); + return archive; +} + +function stageHostedBinaryAsset(releaseName) { + const stageRoot = path.join(stagingDir, "hosted-binary"); + const binDir = path.join(stageRoot, "bin"); + fs.rmSync(stageRoot, { recursive: true, force: true }); + ensureDir(binDir); + const fileName = binaryName(hostedBinary); + const built = path.join(hostedBuildTarget, "release", fileName); + if (!fs.existsSync(built)) { + throw new Error(`expected hosted release binary ${built}`); + } + const staged = path.join(binDir, fileName); + fs.copyFileSync(built, staged); + fs.chmodSync(staged, 0o755); + const archive = path.join( + assetsDir, + `clusterflux-hosted-${releaseName}-${platformName()}.tar.gz` + ); + tarGz(archive, stageRoot, ["."]); + return archive; +} + +function publicBinaryDigests() { + return Object.fromEntries( + publicBinaries.map((binary) => { + const fileName = binaryName(binary); + const file = path.join(publicBuildTarget, "release", fileName); + if (!fs.existsSync(file)) throw new Error(`missing built release binary ${file}`); + return [fileName, `sha256:${sha256File(file)}`]; + }) + ); +} + +function candidateBinaryDigests() { + const fileName = binaryName(hostedBinary); + const file = path.join(hostedBuildTarget, "release", fileName); + if (!fs.existsSync(file)) throw new Error(`missing hosted release binary ${file}`); + return { + ...publicBinaryDigests(), + [fileName]: `sha256:${sha256File(file)}`, + }; +} + +function reuseCandidateBinaries(candidate, releaseName) { + assertCandidateManifest(candidate); + const asset = candidate.assets.find((entry) => + entry.name.startsWith("clusterflux-public-binaries-") + ); + if (!asset || !fs.existsSync(asset.file)) { + throw new Error("release candidate binary archive is missing"); + } + if (sha256File(asset.file) !== asset.sha256) { + throw new Error("release candidate binary archive digest changed after candidate creation"); + } + const archive = path.join( + assetsDir, + `clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz` + ); + fs.copyFileSync(asset.file, archive); + const extractRoot = path.join(stagingDir, "candidate-binaries"); + fs.rmSync(extractRoot, { recursive: true, force: true }); + ensureDir(extractRoot); + run("tar", ["-xzf", archive, "-C", extractRoot]); + for (const name of publicBinaries.map(binaryName)) { + const digest = candidate.binary_digests[name]; + const binary = path.join(extractRoot, "bin", name); + if (!fs.existsSync(binary) || `sha256:${sha256File(binary)}` !== digest) { + throw new Error(`release candidate binary ${name} does not match ${digest}`); + } + } + return archive; +} + +function reuseCandidateHostedBinary(candidate, releaseName) { + assertCandidateManifest(candidate); + const asset = candidate.assets.find((entry) => + entry.name.startsWith("clusterflux-hosted-") + ); + if (!asset || !fs.existsSync(asset.file)) { + throw new Error("release candidate hosted binary archive is missing"); + } + if (sha256File(asset.file) !== asset.sha256) { + throw new Error("release candidate hosted archive digest changed after creation"); + } + const archive = path.join( + assetsDir, + `clusterflux-hosted-${releaseName}-${platformName()}.tar.gz` + ); + fs.copyFileSync(asset.file, archive); + const extractRoot = path.join(stagingDir, "candidate-hosted-binary"); + fs.rmSync(extractRoot, { recursive: true, force: true }); + ensureDir(extractRoot); + run("tar", ["-xzf", archive, "-C", extractRoot]); + const name = binaryName(hostedBinary); + const binary = path.join(extractRoot, "bin", name); + const digest = candidate.binary_digests[name]; + if (!fs.existsSync(binary) || `sha256:${sha256File(binary)}` !== digest) { + throw new Error(`release candidate hosted binary ${name} does not match ${digest}`); + } + return archive; +} + +function assertCandidateManifest(candidate) { + if (candidate.kind !== "clusterflux-public-release") { + throw new Error("release candidate manifest has the wrong kind"); + } + if (candidate.release_candidate?.mode !== "built-once") { + throw new Error("release finalization requires a built-once candidate manifest"); + } + if (!candidate.binary_digests || !Object.keys(candidate.binary_digests).length) { + throw new Error("release candidate manifest omitted binary digests"); + } + if (!candidate.binary_digests[binaryName(hostedBinary)]) { + throw new Error("release candidate omitted the hosted service binary digest"); + } + if (!candidate.release_candidate.hosted_binary_archive_sha256) { + throw new Error("release candidate omitted the hosted service archive digest"); + } +} + +function stageEvidenceAsset( + releaseName, + sourceCommit, + sourceDigest, + publicTreeIdentity, + binaryDigests, + candidateBinding, + candidateConfigurationIdentity, + candidateProxyConfigurationIdentity +) { + const stage = path.join(stagingDir, "evidence"); + fs.rmSync(stage, { recursive: true, force: true }); + ensureDir(stage); + const evidenceRoot = path.join(repo, "target/acceptance"); + const included = []; + for (const name of ["public-environment.json", "wasmtime-assignment.json"]) { + const source = path.join(evidenceRoot, name); + if (!fs.existsSync(source)) continue; + fs.copyFileSync(source, path.join(stage, name)); + included.push(name); + } + const acceptancePath = path.resolve( + process.env.CLUSTERFLUX_FINAL_RESULT_PATH || + path.join(evidenceRoot, "cli-happy-path-live.json") + ); + const legacyIncompleteEvidenceEnv = [ + "CLUSTERFLUX", + "ALLOW", + "INCOMPLETE", + "RELEASE", + "EVIDENCE", + ].join("_"); + if (process.env[legacyIncompleteEvidenceEnv]) { + throw new Error( + "the legacy incomplete-evidence environment switch is unsupported; use CLUSTERFLUX_RELEASE_STAGE=candidate or final" + ); + } + const releaseStage = + process.env.CLUSTERFLUX_RELEASE_STAGE || + (candidateManifestPath ? "final" : "candidate"); + if (!new Set(["candidate", "final"]).has(releaseStage)) { + throw new Error("CLUSTERFLUX_RELEASE_STAGE must be candidate or final"); + } + const allowIncomplete = releaseStage === "candidate"; + let finalEvidence = { + complete: false, + result: null, + transcript: null, + acceptance_result_recorded_at: null, + deployment_identity: null, + configuration_identity: null, + }; + + if (fs.existsSync(acceptancePath)) { + const liveResult = JSON.parse(fs.readFileSync(acceptancePath, "utf8")); + if (liveResult.source_commit === sourceCommit) { + if (liveResult.acceptance_result !== "passed") { + throw new Error("final live acceptance result is not passed"); + } + if (liveResult.source_tree_digest !== sourceDigest) { + throw new Error("final live acceptance source-tree digest does not match the candidate"); + } + if (liveResult.public_tree_identity !== publicTreeIdentity) { + throw new Error("final live acceptance public-tree identity does not match the candidate"); + } + if ( + JSON.stringify(liveResult.binary_digests) !== JSON.stringify(binaryDigests) || + JSON.stringify(liveResult.release_binding?.binary_digests) !== + JSON.stringify(binaryDigests) + ) { + throw new Error("final live acceptance binaries do not match the exact candidate"); + } + if (liveResult.release_binding?.source_commit !== sourceCommit) { + throw new Error("final live acceptance release binding has the wrong source commit"); + } + if (!Array.isArray(liveResult.scenario_skips) || liveResult.scenario_skips.length) { + throw new Error("final live acceptance result contains unresolved scenario skips"); + } + if ( + !Array.isArray(liveResult.named_scenarios) || + liveResult.named_scenarios.length !== 25 || + liveResult.named_scenarios.some((scenario) => scenario.status !== "passed") + ) { + throw new Error("all twenty-five named final live checks must pass"); + } + if ( + !liveResult.release_binding?.deployment || + !liveResult.release_binding?.configuration || + !liveResult.release_binding?.proxy_configuration + ) { + throw new Error( + "final live evidence must bind deployment, runtime configuration, and proxy configuration identities" + ); + } + if ( + !Array.isArray(liveResult.strict_requirement_ledger) || + !liveResult.strict_requirement_ledger.length || + liveResult.strict_requirement_ledger.some((requirement) => requirement.passed !== true) + ) { + throw new Error("the strict final requirement ledger must be complete and passed"); + } + const deploymentGeneration = + process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || + liveResult.release_binding?.deployment?.system_generation || + null; + if (!deploymentGeneration && !allowIncomplete) { + throw new Error( + "CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION is required for final evidence" + ); + } + const recordedAt = fs.statSync(acceptancePath).mtime.toISOString(); + const packagedAt = new Date().toISOString(); + const sanitized = sanitizeEvidence(liveResult); + const finalResult = { + ...sanitized, + kind: "clusterflux-final-release-result", + source_commit: sourceCommit, + source_tree_digest: sourceDigest, + public_tree_identity: publicTreeIdentity, + binary_digests: binaryDigests, + release_candidate: candidateBinding, + deployment: { + system_generation: deploymentGeneration, + identity: evidenceIdentity(liveResult.release_binding?.deployment), + }, + configuration_identity: evidenceIdentity( + liveResult.release_binding?.configuration + ), + timestamps: { + acceptance_result_recorded_at: recordedAt, + evidence_packaged_at: packagedAt, + }, + release_commands: [ + "nix develop -c ./scripts/acceptance-private.sh", + "nix develop -c ./scripts/acceptance-public.sh", + "node scripts/cli-happy-path-live-smoke.js (strict production-shaped environment)", + ], + }; + const resultName = "FINAL_RELEASE_RESULT.json"; + const transcriptName = "FINAL_RELEASE_TRANSCRIPT.txt"; + const resultPath = path.join(stage, resultName); + const transcriptPath = path.join(stage, transcriptName); + fs.writeFileSync(resultPath, `${JSON.stringify(finalResult, null, 2)}\n`); + fs.writeFileSync(transcriptPath, renderFinalTranscript(finalResult)); + included.push(resultName, transcriptName); + finalEvidence = { + complete: true, + result: { + name: resultName, + sha256: sha256File(resultPath), + }, + transcript: { + name: transcriptName, + sha256: sha256File(transcriptPath), + }, + acceptance_result_recorded_at: recordedAt, + deployment_identity: finalResult.deployment.identity, + configuration_identity: finalResult.configuration_identity, + }; + } else if (!allowIncomplete) { + throw new Error( + `final live acceptance commit ${liveResult.source_commit} does not match ${sourceCommit}` + ); + } + } else if (!allowIncomplete) { + throw new Error(`missing final live acceptance result: ${acceptancePath}`); + } + + const binding = { + kind: "clusterflux-release-evidence-binding", + source_commit: sourceCommit, + source_tree_digest: sourceDigest, + public_tree_identity: publicTreeIdentity, + binary_digests: binaryDigests, + release_candidate: candidateBinding, + candidate_configuration_identity: candidateConfigurationIdentity, + candidate_proxy_configuration_identity: candidateProxyConfigurationIdentity, + configuration_generation: + process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, + final_evidence: finalEvidence, + included_evidence: included.map((name) => ({ + name, + sha256: sha256File(path.join(stage, name)), + })), + }; + fs.writeFileSync( + path.join(stage, "EVIDENCE_BINDING.json"), + `${JSON.stringify(binding, null, 2)}\n` + ); + const archive = path.join( + assetsDir, + `clusterflux-public-evidence-${releaseName}.tar.gz` + ); + tarGz(archive, stage, ["."]); + return { archive, finalEvidence }; +} + +function stageSourceAsset(releaseName) { + const archive = path.join(assetsDir, `clusterflux-public-source-${releaseName}.tar.gz`); + tarGz(archive, publicTree, ["."]); + return archive; +} + +function stageExtensionAsset() { + const packageJson = JSON.parse( + fs.readFileSync(path.join(publicTree, "vscode-extension/package.json"), "utf8") + ); + const archive = path.join( + assetsDir, + `${packageJson.name}-${packageJson.version}.vsix` + ); + run( + "npx", + [ + "--yes", + "@vscode/vsce", + "package", + "--allow-missing-repository", + "--skip-license", + "--out", + archive, + ], + { cwd: path.join(publicTree, "vscode-extension") } + ); + return archive; +} + +function resolverInstructions() { + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS) { + return process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS.trim(); + } + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY) { + return [ + "Add the controlled hosts entry supplied for this release:", + "", + "```", + process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY.trim(), + "```", + ].join("\n"); + } + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP) { + return [ + "Add this controlled hosts entry for the release:", + "", + "```", + `${process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP} clusterflux.michelpaulissen.com`, + "```", + ].join("\n"); + } + return [ + "`clusterflux.michelpaulissen.com` should resolve through public DNS. If it", + "does not resolve yet, wait for DNS propagation or use the fallback hosts", + "entry supplied with the invitation.", + ].join("\n"); +} + +function writeGettingStartedAsset(releaseName, publicTreeIdentity, resolution) { + const file = path.join(assetsDir, `CLUSTERFLUX_GETTING_STARTED-${releaseName}.md`); + fs.writeFileSync( + file, + `# Clusterflux Getting Started + +Use the public repository and release downloads with the hosted coordinator at +\`https://clusterflux.michelpaulissen.com\`. + +## DNS + +${resolution} + +## Install + +1. Download the binary archive for your platform from the Forgejo release. +2. Check the archive against \`SHA256SUMS\`. +3. Extract it and put \`bin/\` on your \`PATH\`. +4. Install \`clusterflux-vscode-*.vsix\` when you want the VS Code debugger: + +\`\`\`bash +code --install-extension clusterflux-vscode-*.vsix +\`\`\` + +## Sign in and run + +\`\`\`bash +clusterflux login --browser +clusterflux auth status +clusterflux project list +clusterflux bundle inspect --project examples/hello-build +\`\`\` + +The CLI opens the server-provided Authentik authorization URL. State, nonce, +PKCE, provider code exchange, and userinfo remain on the hosted service. The CLI +stores only the resulting scoped Clusterflux session. + +Create a node enrollment grant, attach the node, and start the worker: + +\`\`\`bash +clusterflux node enroll --project-id --json +clusterflux node attach --project-id --node workstation \\ + --enrollment-grant "$ENROLLMENT_GRANT" +clusterflux-node --coordinator https://clusterflux.michelpaulissen.com \\ + --tenant --project-id --node workstation \\ + --project-root "$PWD" --worker --emit-ready +\`\`\` + +Then run the workflow: + +\`\`\`bash +clusterflux run --project examples/hello-build build +\`\`\` + +Public tree identity: \`${publicTreeIdentity}\` +Release name: \`${releaseName}\` +`, + "utf8" + ); + return file; +} + +function writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution) { + const file = path.join(assetsDir, `CLUSTERFLUX_RELEASE_NOTES-${releaseName}.md`); + const publicRepo = + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + "https://git.michelpaulissen.com/michel/clusterflux-public"; + const releaseUrl = + process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || + "the Forgejo release attached to the public repository"; + fs.writeFileSync( + file, + `# Clusterflux Release Notes + +Use this release with the public Clusterflux repository and hosted coordinator. + +## Links + +- Public repository: ${publicRepo} +- Release downloads: ${releaseUrl} +- Hosted coordinator: ${defaultHostedCoordinatorEndpoint} +- Public tree identity: ${publicTreeIdentity} +- Release name: ${releaseName} + +## DNS + +${resolution} + +## First run + +1. Download the archive for your platform and \`SHA256SUMS\`. +2. Extract the archive and put \`bin/\` on your \`PATH\`. +3. Optionally install the VS Code extension archive. +4. Run \`clusterflux login --browser\`. +5. Run \`clusterflux node enroll\`, attach your node, and start the worker. +6. Run \`clusterflux run --project examples/hello-build build\`. +`, + "utf8" + ); + return file; +} + +function writeSha256Sums(assets) { + const sumsPath = path.join(assetsDir, "SHA256SUMS"); + const lines = assets + .map((asset) => `${sha256File(asset)} ${path.basename(asset)}`) + .sort() + .join("\n"); + fs.writeFileSync(sumsPath, `${lines}\n`); + return sumsPath; +} + +function boolEnv(name) { + return /^(1|true|yes)$/i.test(process.env[name] || ""); +} + +function writePublicTreeProvenance(sourceCommit, releaseName) { + const provenance = { + kind: "clusterflux-filtered-public-tree", + source_commit: sourceCommit, + release_name: releaseName, + filtered_out: filteredOutPatterns(), + public_export: { + host_neutral: !includeForgejoWorkflows, + include_forgejo_workflows: includeForgejoWorkflows, + }, + forgejo_host: forgejoHost, + default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, + }; + fs.writeFileSync( + path.join(publicTree, "CLUSTERFLUX_PUBLIC_TREE.json"), + `${JSON.stringify(provenance, null, 2)}\n` + ); +} + +function publishPublicTree(releaseName, sourceCommit, publicTreeIdentity) { + const remote = + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + null; + const enabled = boolEnv("CLUSTERFLUX_PUBLISH_PUBLIC_TREE"); + const result = { + enabled, + remote, + branch: publicRepoBranch, + commit: null, + pushed: false, + }; + + if (!enabled) { + return result; + } + if (!remote) { + throw new Error( + "CLUSTERFLUX_PUBLISH_PUBLIC_TREE requires CLUSTERFLUX_PUBLIC_REPO_REMOTE or CLUSTERFLUX_PUBLIC_REPO_URL" + ); + } + if (!remote.includes(forgejoHost)) { + throw new Error(`public repo remote must point at ${forgejoHost}: ${remote}`); + } + + run("git", ["init"], { cwd: publicTree }); + run("git", ["config", "user.name", "Clusterflux release"], { + cwd: publicTree, + }); + run("git", ["config", "user.email", "release@clusterflux.invalid"], { + cwd: publicTree, + }); + run("git", ["add", "."], { cwd: publicTree }); + const tree = commandOutput("git", ["write-tree"], { cwd: publicTree }); + run("git", ["remote", "add", "public", remote], { cwd: publicTree }); + run( + "git", + [ + "fetch", + "public", + `refs/heads/${publicRepoBranch}:refs/remotes/public/${publicRepoBranch}`, + ], + { cwd: publicTree } + ); + const parent = commandOutput( + "git", + ["rev-parse", `refs/remotes/public/${publicRepoBranch}`], + { cwd: publicTree } + ); + result.commit = commandOutput( + "git", + [ + "commit-tree", + tree, + "-p", + parent, + "-m", + `Public release ${releaseName}`, + "-m", + `Source commit: ${sourceCommit}`, + "-m", + `Public tree identity: ${publicTreeIdentity}`, + ], + { cwd: publicTree } + ); + run("git", ["update-ref", `refs/heads/${publicRepoBranch}`, result.commit], { + cwd: publicTree, + }); + run("git", ["push", "public", `${result.commit}:${publicRepoBranch}`], { + cwd: publicTree, + }); + result.pushed = true; + return result; +} + +function main() { + const sourceCommit = commandOutput("git", ["rev-parse", "HEAD"]); + if (!sourceCommit || !/^[0-9a-f]{40}$/u.test(sourceCommit)) { + throw new Error("the public release requires an exact Git HEAD object id"); + } + const acceptanceCommit = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT?.trim(); + if (acceptanceCommit && acceptanceCommit !== sourceCommit) { + throw new Error( + `CLUSTERFLUX_ACCEPTANCE_COMMIT ${acceptanceCommit} does not match source HEAD ${sourceCommit}` + ); + } + const shortCommit = sourceCommit.slice(0, 12); + const releaseName = process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || `release-${shortCommit}`; + const sourceDigest = sourceTreeDigest(sourceCommit); + const candidate = candidateManifestPath + ? JSON.parse(fs.readFileSync(candidateManifestPath, "utf8")) + : null; + if (candidate) { + for (const asset of candidate.assets ?? []) { + asset.file = path.isAbsolute(asset.file) + ? asset.file + : path.resolve(path.dirname(candidateManifestPath), asset.file); + } + } + const releaseStage = + process.env.CLUSTERFLUX_RELEASE_STAGE || + (candidateManifestPath ? "final" : "candidate"); + if (releaseStage === "final" && !candidate) { + throw new Error("final release stage requires CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST"); + } + if (releaseStage === "candidate" && candidate) { + throw new Error("candidate release stage cannot consume a prior candidate manifest"); + } + if ( + releaseStage === "candidate" && + (!process.env.CLUSTERFLUX_CANDIDATE_CONFIGURATION_IDENTITY || + !process.env.CLUSTERFLUX_CANDIDATE_PROXY_CONFIGURATION_IDENTITY) + ) { + throw new Error( + "candidate release stage requires configuration and proxy configuration identities" + ); + } + if (candidate) { + assertCandidateManifest(candidate); + if (path.dirname(candidateManifestPath) === outputRoot) { + throw new Error("candidate and finalized release must use different output directories"); + } + if (candidate.source_commit !== sourceCommit) { + throw new Error("release candidate source commit does not match Git HEAD"); + } + if (candidate.source_tree_digest !== sourceDigest) { + throw new Error("release candidate source tree digest does not match Git HEAD"); + } + if (candidate.release_name !== releaseName) { + throw new Error("release candidate name does not match the finalized release name"); + } + } + const sourceStatus = commandOutput("git", ["status", "--short"]); + if (sourceStatus === null) { + throw new Error("the public release must be prepared from a Git checkout"); + } + if (sourceStatus !== "") { + throw new Error("the public release must be prepared from a clean source tree"); + } + const sourceTreeClean = true; + + fs.rmSync(outputRoot, { recursive: true, force: true }); + ensureDir(publicTree); + ensureDir(assetsDir); + ensureDir(stagingDir); + + const ignoredSourcePaths = gitIgnoredSourcePaths(); + copyFilteredTree(repo, publicTree, ignoredSourcePaths); + assertFilteredTree(ignoredSourcePaths); + writePublicTreeProvenance(sourceCommit, releaseName); + const publicTreeIdentity = hashTree(publicTree); + if (candidate && candidate.public_tree_identity !== publicTreeIdentity) { + throw new Error("release candidate public tree identity changed before finalization"); + } + const sourceArchive = stageSourceAsset(releaseName); + + run("scripts/check-old-name.sh", [], { cwd: publicTree }); + run("node", ["scripts/check-docs.js"], { cwd: publicTree }); + run("scripts/check-code-size.sh", [], { cwd: publicTree }); + const publicTreePublish = candidate + ? candidate.public_tree_publish + : publishPublicTree(releaseName, sourceCommit, publicTreeIdentity); + let binaryArchive; + let hostedBinaryArchive; + let binaryDigests; + let candidateBinding; + if (candidate) { + binaryArchive = reuseCandidateBinaries(candidate, releaseName); + hostedBinaryArchive = reuseCandidateHostedBinary(candidate, releaseName); + binaryDigests = candidate.binary_digests; + candidateBinding = { + mode: "finalized-exact", + manifest: path.relative(outputRoot, candidateManifestPath).split(path.sep).join("/"), + manifest_sha256: sha256File(candidateManifestPath), + binary_archive_sha256: sha256File(binaryArchive), + hosted_binary_archive_sha256: sha256File(hostedBinaryArchive), + }; + } else { + buildPublicBinaries(); + buildHostedBinary(); + binaryArchive = stageBinaryAssets(releaseName); + hostedBinaryArchive = stageHostedBinaryAsset(releaseName); + binaryDigests = candidateBinaryDigests(); + candidateBinding = { + mode: "built-once", + binary_archive_sha256: sha256File(binaryArchive), + hosted_binary_archive_sha256: sha256File(hostedBinaryArchive), + }; + } + const candidateConfigurationIdentity = + candidate?.candidate_configuration_identity || + process.env.CLUSTERFLUX_CANDIDATE_CONFIGURATION_IDENTITY || + null; + const candidateProxyConfigurationIdentity = + candidate?.candidate_proxy_configuration_identity || + process.env.CLUSTERFLUX_CANDIDATE_PROXY_CONFIGURATION_IDENTITY || + null; + const evidence = stageEvidenceAsset( + releaseName, + sourceCommit, + sourceDigest, + publicTreeIdentity, + binaryDigests, + candidateBinding, + candidateConfigurationIdentity, + candidateProxyConfigurationIdentity + ); + const evidenceArchive = evidence.archive; + const extensionArchive = stageExtensionAsset(); + const resolution = resolverInstructions(); + const gettingStarted = writeGettingStartedAsset( + releaseName, + publicTreeIdentity, + resolution + ); + const invite = writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution); + const assets = [ + sourceArchive, + binaryArchive, + hostedBinaryArchive, + evidenceArchive, + extensionArchive, + gettingStarted, + invite, + ]; + const sha256Sums = writeSha256Sums(assets); + + const manifest = { + kind: "clusterflux-public-release", + release_name: releaseName, + source_commit: sourceCommit, + source_tree_digest: sourceDigest, + source_tree_clean: sourceTreeClean, + public_tree_identity: publicTreeIdentity, + public_tree: publicTree, + filtered_out: filteredOutPatterns(), + public_export: { + host_neutral: !includeForgejoWorkflows, + include_forgejo_workflows: includeForgejoWorkflows, + }, + forgejo_host: forgejoHost, + public_repo_url: + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + candidate?.public_repo_url || + publicTreePublish.remote, + public_repo_remote: + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || candidate?.public_repo_remote || null, + public_tree_publish: publicTreePublish, + forgejo_release_url: process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || null, + default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, + dns_publication_state: + process.env.CLUSTERFLUX_DNS_PUBLICATION_STATE || "published", + resolver_override: + process.env.CLUSTERFLUX_RESOLVER_OVERRIDE || "none-required-public-dns", + platform: platformName(), + binary_digests: binaryDigests, + release_candidate: candidateBinding, + candidate_configuration_identity: candidateConfigurationIdentity, + candidate_proxy_configuration_identity: candidateProxyConfigurationIdentity, + configuration_generation: + process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, + final_evidence: evidence.finalEvidence, + tool_versions: { + node: process.version, + rustc: commandOutput("rustc", ["--version"]) || null, + cargo: commandOutput("cargo", ["--version"]) || null, + tar: commandOutput("tar", ["--version"]) || null, + }, + commands: [ + "scripts/check-old-name.sh", + "node scripts/check-docs.js", + "scripts/check-code-size.sh", + ...(publicTreePublish.enabled + ? [`git push public HEAD:${publicRepoBranch}`] + : []), + ...(candidate + ? ["finalize exact public and hosted binaries from CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST"] + : [ + "cargo build --locked --workspace --bins --release --jobs 2", + "cargo build --locked --manifest-path private/hosted-policy/Cargo.toml --bin clusterflux-hosted-service --release --jobs 2", + ]), + ], + assets: [...assets, sha256Sums].map((asset) => ({ + file: path.relative(outputRoot, asset).split(path.sep).join("/"), + name: path.basename(asset), + sha256: sha256File(asset), + })), + notes: [ + "Upload the assets to the Forgejo Release for the filtered public repository.", + "The real service deployment and full e2e release remain separate acceptance evidence.", + ], + }; + + const manifestPath = path.join(outputRoot, "public-release-manifest.json"); + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + console.log(JSON.stringify({ manifest: manifestPath, assets: assetsDir }, null, 2)); +} + +main(); diff --git a/scripts/private-repository-gate.sh b/scripts/private-repository-gate.sh new file mode 100755 index 0000000..842457e --- /dev/null +++ b/scripts/private-repository-gate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$repo" + +scripts/check-old-name.sh +node scripts/check-docs.js +scripts/check-code-size.sh +cargo fmt --all --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +cargo test --locked --manifest-path private/hosted-policy/Cargo.toml +node scripts/vscode-extension-smoke.js diff --git a/scripts/public-local-demo-matrix-smoke.js b/scripts/public-local-demo-matrix-smoke.js new file mode 100755 index 0000000..8cae940 --- /dev/null +++ b/scripts/public-local-demo-matrix-smoke.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing public local demo evidence: ${name}`); +} + +const publicAcceptance = read("scripts/acceptance-public.sh"); +const publicSplit = read("scripts/verify-public-split.sh"); +const cliInstall = read("scripts/cli-install-smoke.js"); +const flagship = read("scripts/flagship-demo-smoke.js"); +const cliLocalRun = read("scripts/cli-local-run-smoke.js"); +const nodeAttach = read("scripts/node-attach-smoke.js"); +const vscodeExtension = read("scripts/vscode-extension-smoke.js"); +const vscodeF5 = read("scripts/vscode-f5-smoke.js"); +const artifactDownload = read("scripts/artifact-download-smoke.js"); +const artifactExport = read("scripts/artifact-export-smoke.js"); + +for (const script of [publicAcceptance, publicSplit]) { + for (const smoke of [ + "scripts/cli-install-smoke.js", + "scripts/flagship-demo-smoke.js", + "scripts/cli-local-run-smoke.js", + "scripts/node-attach-smoke.js", + "scripts/vscode-extension-smoke.js", + "scripts/vscode-f5-smoke.js", + "scripts/artifact-download-smoke.js", + "scripts/artifact-export-smoke.js", + ]) { + assert(script.includes(`node ${smoke}`), `public demo gate must run ${smoke}`); + } +} + +expect(cliInstall, "CLI install from project path", /cargo[\s\S]*install[\s\S]*crates\/clusterflux-cli/); +expect(cliInstall, "CLI install smoke targets runtime fixture", /const project = path\.join\(repo, "tests\/fixtures\/runtime-conformance"\)/); +expect(cliInstall, "installed CLI inspects flagship project", /installedBin[\s\S]*\["bundle", "inspect", "--project", project, "--json"\]/); + +expect(flagship, "flagship project source is conventional Rust workflow", /examples\/hello-build[\s\S]*src\/lib\.rs/); +expect(flagship, "flagship source avoids implementation details", /const forbidden/); +expect(flagship, "flagship builds a real bundle", /clusterflux-cli[\s\S]*build[\s\S]*hello/); + +expect(cliLocalRun, "local run starts node process", /cli_process_started_node_process[\s\S]*true/); +expect(cliLocalRun, "local run records real task events", /events\.events\.length >= 4[\s\S]*prepare_source[\s\S]*compile_linux[\s\S]*package_release/); +expect(cliLocalRun, "local run records artifact metadata", /events\.events\.some\(\(event\) => event\.artifact_path\)/); + +expect(nodeAttach, "Linux node attach creates enrollment grant", /create_node_enrollment_grant/); +expect(nodeAttach, "Linux node attach uses enrollment exchange", /used_enrollment_exchange[\s\S]*true/); +expect(nodeAttach, "attached Linux node proves signed enrollment", /used_enrollment_exchange[\s\S]*signedNodeHeartbeat/); + +expect(vscodeExtension, "extension contributes debugger", /contributes\.debuggers[\s\S]*type === "clusterflux"/); +expect(vscodeF5, "F5 uses local-services backend", /runtimeBackend[\s\S]*local-services/); +expect(vscodeF5, "F5 exposes real coordinator main thread", /threads\.find\(\(thread\) => thread\.name\.includes\("build coordinator main"\)\)[\s\S]*real coordinator-main entrypoint thread/); +expect(vscodeF5, "F5 does not fabricate a terminal event at the entry probe", /coordinator_task_events[\s\S]*value === 0[\s\S]*must not fabricate a terminal task event/); + +expect(artifactDownload, "artifact download creates scoped link", /create_artifact_download_link/); +expect(artifactDownload, "artifact download opens stream", /open_artifact_download_stream/); +expect(artifactExport, "artifact export targets receiver node", /export_artifact_to_node[\s\S]*node-export-receiver/); +expect(artifactExport, "artifact export rejects coordinator bulk relay", /coordinator_bulk_relay_allowed[\s\S]*false/); + +console.log("Public local demo matrix smoke passed"); diff --git a/scripts/public-release-preflight.js b/scripts/public-release-preflight.js new file mode 100755 index 0000000..8176d0a --- /dev/null +++ b/scripts/public-release-preflight.js @@ -0,0 +1,438 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const acceptanceRoot = path.join(repo, "target/acceptance"); +const manifestPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || + path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_PREFLIGHT_REPORT || + path.join(acceptanceRoot, "public-release-preflight.json"); + +function commandOutput(command, args, options = {}) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + ...options, + }) + .trim(); + } catch (_) { + return null; + } +} + +function nonInteractiveEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function expectedSourceCommit() { + const head = commandOutput("git", ["rev-parse", "HEAD"]); + assert(head && /^[0-9a-f]{40}$/u.test(head), "preflight requires an exact Git HEAD"); + const asserted = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT; + if (asserted) { + assert.strictEqual(asserted, head, "acceptance commit must match Git HEAD"); + } + return head; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function sha256File(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function sha256Buffer(buffer) { + return crypto.createHash("sha256").update(buffer).digest("hex"); +} + +function readArchiveMember(archive, member) { + return cp.execFileSync("tar", ["-xOzf", archive, `./${member}`], { + cwd: repo, + encoding: null, + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function parseSha256Sums(file) { + const sums = new Map(); + for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) { + if (!line.trim()) continue; + const match = /^([0-9a-f]{64})\s+(.+)$/.exec(line.trim()); + assert(match, `malformed SHA256SUMS line: ${line}`); + sums.set(path.basename(match[2]), match[1]); + } + return sums; +} + +function remoteHead(remote) { + const output = commandOutput("git", ["ls-remote", remote, "HEAD", "refs/heads/main"], { + env: nonInteractiveEnv(), + timeout: Number(process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE_TIMEOUT_MS || 30000), + }); + if (!output) return null; + const lines = output.split(/\r?\n/).filter(Boolean); + const head = lines.find((line) => line.endsWith("\tHEAD")) || lines[0]; + return head && head.split(/\s+/)[0]; +} + +function publicTreeAlreadyPushed(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || + process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" + ); +} + +function publicTreePushSource(manifest) { + return manifest.public_tree_publish && manifest.public_tree_publish.pushed === true + ? "manifest" + : "external-env"; +} + +function publicRepoRemoteForManifest(manifest) { + return ( + manifest.public_repo_url || + manifest.public_repo_remote || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + null + ); +} + +function publicTreeCommitForManifest(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.commit) || + process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || + null + ); +} + +function envState(name) { + return process.env[name] ? "set" : "unset"; +} + +function staleEvidence(file, currentSourceCommit) { + if (!fs.existsSync(file)) { + return { file, status: "missing", source_commit: null, release_name: null }; + } + const evidence = readJson(file); + if (!evidence.source_commit) { + return { + file, + status: "unversioned", + source_commit: null, + release_name: evidence.release_name || null, + }; + } + return { + file, + status: evidence.source_commit === currentSourceCommit ? "current" : "stale", + source_commit: evidence.source_commit, + release_name: evidence.release_name || null, + }; +} + +assert(fs.existsSync(manifestPath), `missing public release manifest: ${manifestPath}`); +const manifest = readJson(manifestPath); +for (const asset of manifest.assets ?? []) { + asset.file = path.isAbsolute(asset.file) + ? asset.file + : path.resolve(path.dirname(manifestPath), asset.file); +} +const currentSourceCommit = expectedSourceCommit(); +const currentTreeStatus = commandOutput("git", ["status", "--short"]) || ""; +assert.strictEqual( + currentTreeStatus, + "", + "public release preflight requires a clean source tree" +); +assert.strictEqual(manifest.kind, "clusterflux-public-release"); +assert.strictEqual( + manifest.source_commit, + currentSourceCommit, + "public release manifest must be regenerated for the current acceptance commit" +); +assert.strictEqual(manifest.source_tree_clean, true, "public release prep must start clean"); +assert.strictEqual( + publicTreeAlreadyPushed(manifest), + true, + "filtered public tree must be pushed to Forgejo before release publication" +); + +const publicRepoRemote = publicRepoRemoteForManifest(manifest); +assert(publicRepoRemote, "manifest must record public repository URL or remote"); +const publicTreeCommit = publicTreeCommitForManifest(manifest); +const remoteMain = remoteHead(publicRepoRemote); +assert(remoteMain, "Forgejo public repository main branch must be readable"); +if (publicTreeCommit) { + assert.strictEqual( + remoteMain, + publicTreeCommit, + "Forgejo public repository main branch must match the prepared public tree commit" + ); +} + +assert(Array.isArray(manifest.assets) && manifest.assets.length > 0, "manifest assets missing"); +const checksumAsset = manifest.assets.find((asset) => asset.name === "SHA256SUMS"); +assert(checksumAsset, "manifest must include SHA256SUMS"); +assert(fs.existsSync(checksumAsset.file), `missing checksum asset: ${checksumAsset.file}`); +const checksums = parseSha256Sums(checksumAsset.file); +const assets = manifest.assets.map((asset) => { + assert(fs.existsSync(asset.file), `missing release asset: ${asset.file}`); + const actual = sha256File(asset.file); + const expected = checksums.get(asset.name); + if (asset.name !== "SHA256SUMS") { + assert.strictEqual(actual, expected, `checksum mismatch for ${asset.name}`); + } + return { + name: asset.name, + file: asset.file, + bytes: fs.statSync(asset.file).size, + sha256: actual, + }; +}); +const binaryAsset = manifest.assets.find((asset) => + asset.name.startsWith("clusterflux-public-binaries-") +); +const hostedBinaryAsset = manifest.assets.find((asset) => + asset.name.startsWith("clusterflux-hosted-") +); +assert(binaryAsset, "manifest must include the exact candidate binary archive"); +assert(hostedBinaryAsset, "manifest must include the exact hosted binary archive"); +for (const [name, digest] of Object.entries(manifest.binary_digests)) { + const archive = name === "clusterflux-hosted-service" ? hostedBinaryAsset : binaryAsset; + const binary = readArchiveMember(archive.file, `bin/${name}`); + assert.strictEqual( + `sha256:${sha256Buffer(binary)}`, + digest, + `candidate archive binary ${name} does not match the manifest` + ); +} + +assert( + manifest.final_evidence && manifest.final_evidence.complete === true, + "release publication requires complete final result and transcript evidence" +); +const evidenceAsset = manifest.assets.find((asset) => + asset.name.startsWith("clusterflux-public-evidence-") +); +assert(evidenceAsset, "manifest must include the final evidence archive"); +const binding = JSON.parse( + readArchiveMember(evidenceAsset.file, "EVIDENCE_BINDING.json").toString("utf8") +); +assert.strictEqual(binding.source_commit, currentSourceCommit); +assert.strictEqual(binding.source_tree_digest, manifest.source_tree_digest); +assert.strictEqual(binding.public_tree_identity, manifest.public_tree_identity); +assert.deepStrictEqual(binding.binary_digests, manifest.binary_digests); +assert(binding.final_evidence && binding.final_evidence.complete === true); +assert(Array.isArray(binding.included_evidence)); +const includedEvidence = new Map( + binding.included_evidence.map((entry) => [entry.name, entry.sha256]) +); +for (const required of ["FINAL_RELEASE_RESULT.json", "FINAL_RELEASE_TRANSCRIPT.txt"]) { + assert(includedEvidence.has(required), `final evidence archive is missing ${required}`); + const contents = readArchiveMember(evidenceAsset.file, required); + assert.strictEqual( + sha256Buffer(contents), + includedEvidence.get(required), + `final evidence digest mismatch for ${required}` + ); +} +const finalResult = JSON.parse( + readArchiveMember(evidenceAsset.file, "FINAL_RELEASE_RESULT.json").toString("utf8") +); +assert.strictEqual(finalResult.source_commit, currentSourceCommit); +assert.strictEqual(finalResult.source_tree_digest, manifest.source_tree_digest); +assert.strictEqual(finalResult.public_tree_identity, manifest.public_tree_identity); +assert.deepStrictEqual(finalResult.binary_digests, manifest.binary_digests); +assert.strictEqual( + manifest.release_candidate?.mode, + "finalized-exact", + "publication requires exact candidate finalization" +); +assert.strictEqual(finalResult.release_binding?.source_commit, currentSourceCommit); +assert.strictEqual( + finalResult.release_binding?.source_tree_digest, + manifest.source_tree_digest +); +assert.strictEqual( + finalResult.release_binding?.public_tree_identity, + manifest.public_tree_identity +); +assert.deepStrictEqual( + finalResult.release_binding?.binary_digests, + manifest.binary_digests +); +assert.deepStrictEqual( + finalResult.release_candidate, + manifest.release_candidate +); +assert.strictEqual(finalResult.acceptance_result, "passed"); +assert.deepStrictEqual(finalResult.scenario_skips, []); +assert( + finalResult.release_binding?.deployment && + finalResult.release_binding?.configuration && + finalResult.release_binding?.proxy_configuration, + "final evidence must bind deployment, runtime configuration, and proxy configuration identities" +); +if (manifest.candidate_configuration_identity) { + assert.strictEqual( + finalResult.release_binding.configuration.evidence_identity, + manifest.candidate_configuration_identity, + "live configuration identity differs from the candidate binding" + ); +} +if (manifest.candidate_proxy_configuration_identity) { + assert.strictEqual( + finalResult.release_binding.proxy_configuration.evidence_identity, + manifest.candidate_proxy_configuration_identity, + "live proxy configuration identity differs from the candidate binding" + ); +} +assert( + Array.isArray(finalResult.named_scenarios) && + finalResult.named_scenarios.length === 25 && + finalResult.named_scenarios.every((scenario) => scenario.status === "passed"), + "all twenty-five named final checks must be present and passed" +); +assert( + Array.isArray(finalResult.strict_requirement_ledger) && + finalResult.strict_requirement_ledger.length > 0 && + finalResult.strict_requirement_ledger.every((requirement) => requirement.passed === true), + "the strict final requirement ledger must be present and passed" +); +const finalTranscript = readArchiveMember( + evidenceAsset.file, + "FINAL_RELEASE_TRANSCRIPT.txt" +).toString("utf8"); +for (const identity of [ + currentSourceCommit, + manifest.source_tree_digest, + manifest.public_tree_identity, + ...Object.values(manifest.binary_digests), +]) { + assert(finalTranscript.includes(identity), `final transcript is missing identity ${identity}`); +} + +const evidence = [ + staleEvidence( + path.join(acceptanceRoot, "public-release-forgejo-release.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-service.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "hosted-client-compat.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "core-coordinator-compat.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-e2e.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-final.json"), + currentSourceCommit + ), +]; + +const report = { + kind: "clusterflux-public-release-preflight", + source_commit: currentSourceCommit, + release_name: manifest.release_name, + public_tree_commit: publicTreeCommit || remoteMain, + public_tree_push_source: publicTreePushSource(manifest), + public_repo_url: publicRepoRemote, + public_repo_remote_head: remoteMain, + source_tree_clean: currentTreeStatus === "", + local_assets_ready: true, + assets, + final_evidence: { + complete: true, + binding, + }, + evidence, + external_gates: { + forgejo_release_publication: { + status: envState("CLUSTERFLUX_FORGEJO_TOKEN") === "set" ? "ready" : "pending", + env: { + CLUSTERFLUX_FORGEJO_TOKEN: envState("CLUSTERFLUX_FORGEJO_TOKEN"), + }, + }, + live_service_smoke: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR") === "set" && + envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR" + ), + CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" + ), + }, + }, + public_release_e2e: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_E2E") === "set" && + process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E === "1" && + envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_E2E: + process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E || "unset", + CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" + ), + }, + }, + final_evidence: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_FINAL") === "set" && + process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL === "1" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_FINAL: + process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL || "unset", + }, + }, + }, +}; + +fs.mkdirSync(path.dirname(reportPath), { recursive: true }); +fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); +console.log(JSON.stringify({ report: reportPath, release_name: report.release_name }, null, 2)); diff --git a/scripts/public-repository-gate.sh b/scripts/public-repository-gate.sh new file mode 100755 index 0000000..333585d --- /dev/null +++ b/scripts/public-repository-gate.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$repo" + +scripts/verify-public-split.sh diff --git a/scripts/publish-public-release.js b/scripts/publish-public-release.js new file mode 100755 index 0000000..8fd76e4 --- /dev/null +++ b/scripts/publish-public-release.js @@ -0,0 +1,354 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const https = require("https"); +const path = require("path"); +const cp = require("child_process"); + +const repo = path.resolve(__dirname, ".."); +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = path.join( + repo, + "target/acceptance/public-release-forgejo-release.json" +); +const forgejoUrl = ( + process.env.CLUSTERFLUX_FORGEJO_URL || "https://git.michelpaulissen.com" +).replace(/\/+$/, ""); +const token = process.env.CLUSTERFLUX_FORGEJO_TOKEN; +let owner = process.env.CLUSTERFLUX_PUBLIC_REPO_OWNER; +let repoName = process.env.CLUSTERFLUX_PUBLIC_REPO_NAME; + +function requireEnv(name, value) { + if (!value) { + throw new Error(`${name} is required`); + } +} + +function commandOutput(command, args) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }) + .trim(); + } catch (_) { + return null; + } +} + +function expectedSourceCommit() { + return ( + process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || + commandOutput("git", ["rev-parse", "HEAD"]) || + "unknown" + ); +} + +function parseForgejoRepoIdentity(remote) { + if (!remote) { + return null; + } + + let pathname = remote; + try { + pathname = new URL(remote).pathname; + } catch (_) { + const scpLike = /^[^@/]+@[^:]+:(.+)$/.exec(remote); + if (scpLike) { + pathname = scpLike[1]; + } + } + + const parts = pathname + .replace(/^\/+/, "") + .replace(/\.git$/, "") + .split("/") + .filter(Boolean); + if (parts.length < 2) { + return null; + } + return { + owner: parts[parts.length - 2], + repoName: parts[parts.length - 1], + }; +} + +function resolveRepoIdentity(manifest) { + if (owner && repoName) { + return; + } + + const inferred = parseForgejoRepoIdentity( + manifest.public_repo_url || + manifest.public_repo_remote || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE + ); + owner = owner || (inferred && inferred.owner); + repoName = repoName || (inferred && inferred.repoName); + if (!owner || !repoName) { + throw new Error( + "CLUSTERFLUX_PUBLIC_REPO_OWNER and CLUSTERFLUX_PUBLIC_REPO_NAME are required when the manifest does not contain a parseable Forgejo repository URL" + ); + } +} + +function publicTreeAlreadyPushed(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || + process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" + ); +} + +function publicTreeCommit(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.commit) || + process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || + null + ); +} + +function apiPath(pathname) { + return `/api/v1${pathname}`; +} + +function request(method, pathname, { body, headers = {} } = {}) { + const url = new URL(apiPath(pathname), forgejoUrl); + const payload = + body === undefined + ? null + : Buffer.isBuffer(body) + ? body + : Buffer.from(JSON.stringify(body)); + const requestHeaders = { + Accept: "application/json", + Authorization: `token ${token}`, + ...headers, + }; + if (payload) { + requestHeaders["Content-Length"] = payload.length; + if (!requestHeaders["Content-Type"]) { + requestHeaders["Content-Type"] = "application/json"; + } + } + + return new Promise((resolve, reject) => { + const req = https.request( + url, + { + method, + headers: requestHeaders, + }, + (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + let parsed = null; + if (text.trim()) { + try { + parsed = JSON.parse(text); + } catch (_) { + parsed = text; + } + } + if (res.statusCode < 200 || res.statusCode >= 300) { + reject( + new Error( + `${method} ${url.pathname} failed with ${res.statusCode}: ${text}` + ) + ); + return; + } + resolve({ status: res.statusCode, body: parsed }); + }); + } + ); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +function multipartFile(fieldName, file) { + const boundary = `clusterflux-${Date.now()}-${Math.random().toString(16).slice(2)}`; + const name = path.basename(file); + const header = Buffer.from( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="${fieldName}"; filename="${name}"\r\n` + + "Content-Type: application/octet-stream\r\n\r\n" + ); + const footer = Buffer.from(`\r\n--${boundary}--\r\n`); + return { + body: Buffer.concat([header, fs.readFileSync(file), footer]), + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + +async function existingRelease(tagName) { + const releases = await request( + "GET", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases?limit=100` + ); + return (releases.body || []).find((release) => release.tag_name === tagName) || null; +} + +async function createOrReuseRelease(manifest) { + const tagName = manifest.release_name; + const found = await existingRelease(tagName); + if (found) { + return { release: found, created: false }; + } + const targetCommitish = + publicTreeCommit(manifest) || + "main"; + const body = [ + "Clusterflux public release.", + "", + `Default hosted coordinator endpoint: ${manifest.default_hosted_coordinator_endpoint}`, + `DNS publication state: ${manifest.dns_publication_state}`, + `Resolver override: ${manifest.resolver_override}`, + `Public tree identity: ${manifest.public_tree_identity}`, + `Source commit: ${manifest.source_commit}`, + ].join("\n"); + const created = await request( + "POST", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases`, + { + body: { + tag_name: tagName, + target_commitish: targetCommitish, + name: tagName, + body, + draft: false, + prerelease: false, + }, + } + ); + return { release: created.body, created: true }; +} + +async function loadRelease(releaseId) { + const response = await request( + "GET", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${releaseId}` + ); + return response.body; +} + +async function uploadAsset(release, asset) { + const { body, contentType } = multipartFile("attachment", asset.file); + const response = await request( + "POST", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, + { + body, + headers: { + "Content-Type": contentType, + }, + } + ); + return response.body; +} + +function existingAssetByName(release, name) { + return Array.isArray(release.assets) + ? release.assets.find((asset) => asset.name === name) || null + : null; +} + +async function main() { + requireEnv("CLUSTERFLUX_FORGEJO_TOKEN", token); + if (!fs.existsSync(manifestPath)) { + throw new Error(`missing public release manifest: ${manifestPath}`); + } + + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + for (const asset of manifest.assets ?? []) { + asset.file = path.isAbsolute(asset.file) + ? asset.file + : path.resolve(path.dirname(manifestPath), asset.file); + } + resolveRepoIdentity(manifest); + if (manifest.kind !== "clusterflux-public-release") { + throw new Error(`unexpected public release manifest kind: ${manifest.kind}`); + } + if (manifest.source_commit !== expectedSourceCommit()) { + throw new Error( + "public release manifest is stale; regenerate it for the current acceptance commit before publishing the Forgejo Release" + ); + } + if ( + !publicTreeAlreadyPushed(manifest) + ) { + throw new Error( + "public tree must be pushed before publishing the Forgejo Release; run prepare-public-release.js with CLUSTERFLUX_PUBLISH_PUBLIC_TREE=1" + ); + } + if (!Array.isArray(manifest.assets) || manifest.assets.length === 0) { + throw new Error("public release manifest has no assets"); + } + + const { release: releaseResult, created } = await createOrReuseRelease(manifest); + const release = await loadRelease(releaseResult.id); + const uploaded = []; + const reused = []; + for (const asset of manifest.assets) { + if (!fs.existsSync(asset.file)) { + throw new Error(`missing release asset: ${asset.file}`); + } + const existing = existingAssetByName(release, asset.name); + if (existing) { + reused.push(existing); + continue; + } + uploaded.push(await uploadAsset(release, asset)); + } + + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + const report = { + kind: "clusterflux-public-release-forgejo-release", + forgejo_url: forgejoUrl, + owner, + repo: repoName, + release_id: release.id, + release_name: release.name || manifest.release_name, + tag_name: release.tag_name || manifest.release_name, + release_created: created, + default_hosted_coordinator_endpoint: manifest.default_hosted_coordinator_endpoint, + public_tree_identity: manifest.public_tree_identity, + public_tree_commit: publicTreeCommit(manifest), + source_commit: manifest.source_commit, + uploaded_assets: uploaded.map((asset) => ({ + id: asset.id, + name: asset.name, + size: asset.size, + browser_download_url: asset.browser_download_url, + })), + reused_assets: reused.map((asset) => ({ + id: asset.id, + name: asset.name, + size: asset.size, + browser_download_url: asset.browser_download_url, + })), + }; + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log( + JSON.stringify( + { report: reportPath, uploaded: uploaded.length, reused: reused.length }, + null, + 2 + ) + ); +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/quic-smoke.js b/scripts/quic-smoke.js new file mode 100755 index 0000000..5ad4634 --- /dev/null +++ b/scripts/quic-smoke.js @@ -0,0 +1,154 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function rendezvousRequest(overrides = {}) { + return { + type: "request_rendezvous", + scope: { + tenant: "tenant", + project: "project", + process: "vp-quic", + object: { Artifact: "quic-artifact" }, + authorization_subject: "node-a-to-node-b", + }, + source: { + node: "node-a", + advertised_addr: "node-a.mesh.invalid:4433", + public_key_fingerprint: "sha256:node-a-public-key", + }, + destination: { + node: "node-b", + advertised_addr: "node-b.mesh.invalid:4433", + public_key_fingerprint: "sha256:node-b-public-key", + }, + direct_connectivity: true, + failure_reason: "", + ...overrides, + }; +} + +(async () => { + const output = cp.execFileSync( + "cargo", + ["run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-quic-smoke"], + { cwd: repo, encoding: "utf8" } + ); + const report = JSON.parse(output.trim().split("\n").at(-1)); + + assert.strictEqual(report.kind, "clusterflux_quic_smoke"); + assert.strictEqual(report.transport, "NativeQuic"); + assert.strictEqual(report.rust_native_quic, true); + assert.strictEqual(report.authenticated_direct_connection, true); + assert.strictEqual(report.coordinator_assisted_rendezvous, true); + assert.strictEqual(report.coordinator_bulk_relay_allowed, false); + assert.strictEqual(report.source_node, "node-a"); + assert.strictEqual(report.destination_node, "node-b"); + assert.strictEqual(report.scope.tenant, "tenant"); + assert.strictEqual(report.scope.project, "project"); + assert.strictEqual(report.scope.process, "vp-quic"); + assert.deepStrictEqual(report.scope.object, { Artifact: "quic-artifact" }); + assert.ok(report.authorization_digest.startsWith("sha256:")); + assert.ok(report.request_bytes > 0); + assert.strictEqual(report.server_received_request_bytes, report.request_bytes); + assert.ok(report.payload_bytes > 0); + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + + const plan = await send(addr, rendezvousRequest()); + assert.strictEqual(plan.type, "rendezvous_plan"); + assert.strictEqual(plan.charged_rendezvous_attempts, 1); + assert.strictEqual(plan.plan.transport, "NativeQuic"); + assert.strictEqual(plan.plan.scope.tenant, "tenant"); + assert.strictEqual(plan.plan.scope.project, "project"); + assert.strictEqual(plan.plan.source.node, "node-a"); + assert.strictEqual(plan.plan.destination.node, "node-b"); + assert.strictEqual(plan.plan.coordinator_assisted_rendezvous, true); + assert.strictEqual(plan.plan.coordinator_bulk_relay_allowed, false); + assert.ok(plan.plan.authorization_digest.startsWith("sha256:")); + + const failed = await send( + addr, + rendezvousRequest({ + direct_connectivity: false, + failure_reason: "nat traversal failed", + }) + ); + assert.strictEqual(failed.type, "error"); + assert.match(failed.message, /nat traversal failed/); + assert.match(failed.message, /coordinator bulk relay is disabled/); + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("QUIC smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/real-flagship-harness.js b/scripts/real-flagship-harness.js new file mode 100644 index 0000000..1320f23 --- /dev/null +++ b/scripts/real-flagship-harness.js @@ -0,0 +1,285 @@ +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/hello-build"); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.stderr?.on("data", (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 4096) stderr = stderr.slice(-4096); + }); + child.once("exit", (code) => { + const detail = stderr.trim(); + reject(new Error( + `process exited before JSON line with code ${code}${detail ? `: ${detail}` : ""}` + )); + }); + }); +} + +function waitForNodeStatus(child, expectedStatus) { + return new Promise((resolve, reject) => { + let buffer = ""; + let stderr = ""; + const cleanup = () => { + child.stdout.off("data", onData); + child.stderr?.off("data", onStderr); + child.off("exit", onExit); + }; + const onData = (chunk) => { + buffer += chunk.toString(); + while (buffer.includes("\n")) { + const newline = buffer.indexOf("\n"); + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + let event; + try { + event = JSON.parse(line); + } catch (error) { + cleanup(); + reject(error); + return; + } + if (event.node_status === expectedStatus) { + cleanup(); + resolve(event); + return; + } + } + }; + const onStderr = (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 4096) stderr = stderr.slice(-4096); + }; + const onExit = (code) => { + cleanup(); + const detail = stderr.trim(); + reject( + new Error( + `worker exited before node status ${expectedStatus} with code ${code}${ + detail ? `: ${detail}` : "" + }` + ) + ); + }; + child.stdout.on("data", onData); + child.stderr?.on("data", onStderr); + child.once("exit", onExit); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function configureContainerPolicyHome() { + configurePodmanTestEnvironment(repo); +} + +function commandWithPodman(program, args) { + if (cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status === 0) { + return { program, args }; + } + if (cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0) { + return { + program: "nix", + args: ["shell", "nixpkgs#podman", "--command", program, ...args], + }; + } + throw new Error( + "real flagship smoke requires rootless Podman (or Nix to provide it)" + ); +} + +function ensureRootlessPodman() { + configureContainerPolicyHome(); + const invocation = commandWithPodman("podman", [ + "info", + "--format", + "{{.Host.Security.Rootless}}", + ]); + const rootless = cp.execFileSync(invocation.program, invocation.args, { + cwd: repo, + env: process.env, + encoding: "utf8", + }).trim(); + assert.strictEqual(rootless, "true", "flagship worker must use rootless Podman"); +} + +async function runFlagshipWorker(addr, node, identity, projectRoot = project) { + const enrollment = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "user", + ttl_seconds: 60, + }); + assert.strictEqual(enrollment.type, "node_enrollment_grant_created"); + const cargoArgs = [ + "run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node", "--", + "--coordinator", `${addr.host}:${addr.port}`, + "--tenant", "tenant", + "--project-id", "project", + "--node", node, + "--enrollment-grant", enrollment.grant, + "--worker", + "--emit-ready", + "--project-root", projectRoot, + "--assignment-poll-ms", "25", + ]; + const invocation = commandWithPodman("cargo", cargoArgs); + const child = cp.spawn(invocation.program, invocation.args, { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, + }, + }); + return { child, ready: waitForJsonLine(child) }; +} + +const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForTaskEvent(addr, process, predicate, description) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const response = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + assert.strictEqual(response.type, "task_events", JSON.stringify(response)); + const event = response.events.find(predicate); + if (event) return event; + await delay(25); + } + throw new Error(`timed out waiting for real Wasm task ${description}`); +} + +function startFlagship(addr, projectRoot = project) { + const report = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "run", "build", + "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, + "--project", projectRoot, + "--json", + ], + { cwd: repo, env: process.env, encoding: "utf8" } + ) + ); + assert.strictEqual(report.command, "run"); + assert.strictEqual(report.status, "main_launched", JSON.stringify(report)); + assert.strictEqual(report.entry, "build"); + assert.strictEqual(report.task_launch.type, "main_launched"); + assert.strictEqual(report.task_launch.task_instance, report.task_instance); + assert.strictEqual(report.task_launch.task_definition, report.task_definition); + assert.strictEqual(report.worker_placement_requested, true); + assert.match(report.bundle_digest, /^sha256:[0-9a-f]{64}$/); + assert.match(report.entry_export, /^clusterflux_entry_v1_/); + const virtualProcess = report.process; + return { report, process: virtualProcess }; +} + +async function launchFlagship(addr) { + const { report, process: virtualProcess } = startFlagship(addr); + const compileEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task_definition === "compile", + "compile" + ); + const sourceEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task_definition === "snapshot_current_project", + "snapshot_current_project" + ); + const packageEvent = compileEvent; + const buildEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task === report.task_instance && event.executor === "coordinator_main", + "coordinator build main" + ); + for (const event of [sourceEvent, compileEvent, buildEvent]) { + assert.strictEqual(event.terminal_state, "completed", JSON.stringify(event)); + } + return { + report, + process: virtualProcess, + compileEvent, + sourceEvent, + packageEvent, + buildEvent, + }; +} + +function flagshipNodeCapabilities() { + return { + os: "Linux", + arch: process.arch, + capabilities: [ + "Command", + "Containers", + "RootlessPodman", + "SourceFilesystem", + "VfsArtifacts", + ], + environment_backends: ["Container"], + source_providers: ["filesystem"], + }; +} + +module.exports = { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + project, + repo, + runFlagshipWorker, + send, + startFlagship, + waitForTaskEvent, + waitForJsonLine, + waitForNodeStatus, +}; diff --git a/scripts/recovery-build-smoke.js b/scripts/recovery-build-smoke.js new file mode 100755 index 0000000..202d788 --- /dev/null +++ b/scripts/recovery-build-smoke.js @@ -0,0 +1,139 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); +const { DapClient } = require("./dap-client"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/recovery-build"); +const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs")); +const original = fs.readFileSync(sourcePath, "utf8"); +const failing = "\"exit 23\".to_owned()"; +const replacement = + "\"printf 'recovered\\n' > /clusterflux/output/recovering.txt\".to_owned()"; +assert(original.includes(failing), "recovery source must contain the real failing command"); +configurePodmanTestEnvironment(repo); + +(async () => { + const client = new DapClient({ + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_DAP_TIMEOUT_MS: + process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || "120000", + }, + }); + let restored = false; + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "build", + project, + runtimeBackend: "local-services", + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const configured = client.send("configurationDone"); + await client.response(configured, "configurationDone"); + const failed = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "exception" + ); + assert.strictEqual(failed.body.allThreadsStopped, false); + + const threadRequest = client.send("threads"); + const threads = (await client.response(threadRequest, "threads")).body.threads; + const laneThreads = threads + .filter((thread) => /build[_ ]lane/.test(thread.name)) + .sort((left, right) => left.id - right.id); + assert.strictEqual( + laneThreads.length, + 1, + "only the failed recovering lane should remain a live DAP thread" + ); + const recoveringThread = laneThreads[0]; + const views = JSON.parse( + fs.readFileSync(path.join(project, ".clusterflux/views.json"), "utf8") + ); + const nodeReport = JSON.parse( + views.inspector.find((item) => item.label === "Node report").value + ); + const laneSnapshots = nodeReport.task_snapshots.snapshots.filter( + (snapshot) => snapshot.task_definition === "build_lane" + ); + assert.strictEqual(laneSnapshots.length, 2); + assert.notStrictEqual(laneSnapshots[0].task, laneSnapshots[1].task); + assert(laneSnapshots.some((snapshot) => snapshot.state === "completed")); + assert( + laneSnapshots.some( + (snapshot) => snapshot.state === "failed_awaiting_action" + ) + ); + const startedIds = new Set( + client.messages + .filter( + (message) => + message.type === "event" && + message.event === "thread" && + message.body.reason === "started" + ) + .map((message) => message.body.threadId) + ); + const laneStartedIds = [...startedIds] + .filter((id) => id !== 1) + .sort((left, right) => left - right) + .slice(-2); + assert.strictEqual(laneStartedIds.length, 2); + assert.notStrictEqual(laneStartedIds[0], laneStartedIds[1]); + assert(startedIds.has(recoveringThread.id)); + + fs.writeFileSync(sourcePath, original.replace(failing, replacement)); + const restart = client.send("restartFrame", { + threadId: recoveringThread.id, + }); + await client.response(restart, "restartFrame"); + const terminated = await client.waitFor( + (message) => + message.type === "event" && + message.event === "terminated" + ); + assert(terminated); + + const exitedIds = new Set( + client.messages + .filter( + (message) => + message.type === "event" && + message.event === "thread" && + message.body.reason === "exited" + ) + .map((message) => message.body.threadId) + ); + assert(exitedIds.has(laneStartedIds[0])); + assert(exitedIds.has(recoveringThread.id)); + + fs.writeFileSync(sourcePath, original); + restored = true; + await client.close(); + console.log("Recovery build DAP restart smoke passed"); + } finally { + if (!restored) fs.writeFileSync(sourcePath, original); + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + } +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/release-source-scan.sh b/scripts/release-source-scan.sh new file mode 100755 index 0000000..d8bf013 --- /dev/null +++ b/scripts/release-source-scan.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +release_paths=( + Cargo.toml + Cargo.lock + README.md + crates + examples + private + scripts + vscode-extension +) + +existing_release_paths=() +for path in "${release_paths[@]}"; do + if [[ -e "$path" ]]; then + existing_release_paths+=("$path") + fi +done + +prose_scan_paths=( + README.md + crates + examples + private + scripts + vscode-extension +) + +existing_prose_scan_paths=() +for path in "${prose_scan_paths[@]}"; do + if [[ -e "$path" ]]; then + existing_prose_scan_paths+=("$path") + fi +done + +scan_globs=( + --glob '!**/target/**' + --glob '!**/node_modules/**' + --glob '!scripts/release-source-scan.sh' + --glob '!scripts/rename-to-clusterflux.sh' + --glob '!scripts/check-docs.js' +) + +placeholder_pattern='debugger-gate|experiments/debugger-gate|CLUSTERFLUX-DEMO|device-code-placeholder|artifact://demo|vp-local-demo' +if rg -n "${scan_globs[@]}" "$placeholder_pattern" "${existing_release_paths[@]}"; then + echo "release source scan failed: stale experiment/demo placeholder reference found" >&2 + exit 1 +fi + +demo_setup_pattern='undocumented manual state|hidden setup|demo-only credentials?|hard-coded local paths?' +if rg -n "${scan_globs[@]}" "$demo_setup_pattern" "${existing_prose_scan_paths[@]}"; then + echo "release source scan failed: demo requires hidden setup or demo-only state" >&2 + exit 1 +fi + +hidden_local_pattern='file://|/home/[[:alnum:]_.-]+/|/Users/[[:alnum:]_.-]+/|C:\\Users\\|https?://(localhost|127\.0\.0\.1)[^[:space:]]*/artifacts/' +if rg -n "${scan_globs[@]}" "$hidden_local_pattern" "${existing_release_paths[@]}"; then + echo "release source scan failed: hidden local path or local artifact URL found" >&2 + exit 1 +fi + +public_wording_pattern='reddit|hacker news|lobsters|launch forum|traffic source|free tier' +if rg -n "${scan_globs[@]}" "$public_wording_pattern" "${existing_prose_scan_paths[@]}"; then + echo "release source scan failed: public-facing launch-forum/free-tier wording found" >&2 + exit 1 +fi + +echo "Release source scan passed" diff --git a/scripts/resource-metering-contract-smoke.js b/scripts/resource-metering-contract-smoke.js new file mode 100644 index 0000000..62e593d --- /dev/null +++ b/scripts/resource-metering-contract-smoke.js @@ -0,0 +1,230 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing resource metering evidence: ${name}`); +} + +const coreLimits = read("crates/clusterflux-core/src/limits.rs"); +// Phase 3 keeps the protocol dispatch in service.rs and the metered operation +// implementations in focused service modules. Read the complete relevant +// boundary so this contract follows the refactor instead of one mega-file. +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/processes.rs"), + read("crates/clusterflux-coordinator/src/service/process_launch.rs"), + read("crates/clusterflux-coordinator/src/service/artifacts.rs"), +].join("\n"); +const coordinatorQuota = read("crates/clusterflux-coordinator/src/service/quota.rs"); +const coordinatorLogs = read("crates/clusterflux-coordinator/src/service/logs.rs"); +const coordinatorDebug = read("crates/clusterflux-coordinator/src/service/debug.rs"); +const coordinatorTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); +const wasmRuntime = read("crates/clusterflux-wasm-runtime/src/lib.rs"); +const wasmRuntimeTests = read("crates/clusterflux-wasm-runtime/src/tests.rs"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const quicSmoke = read("scripts/quic-smoke.js"); + +for (const [name, pattern] of [ + ["API call limit kind", /\bApiCall,/], + ["spawn limit kind", /\bSpawn,/], + ["log bytes limit kind", /\bLogBytes,/], + ["metadata bytes limit kind", /\bMetadataBytes,/], + ["debug read bytes limit kind", /\bDebugReadBytes,/], + ["UI event limit kind", /\bUiEvent,/], + ["rendezvous attempt limit kind", /\bRendezvousAttempt,/], + ["artifact download bytes limit kind", /\bArtifactDownloadBytes,/], + [ + "preflight can check without consuming", + /pub fn can_charge\([\s\S]*used\.saturating_add\(amount\) > limit/, + ], + [ + "charge goes through preflight", + /pub fn charge\([\s\S]*self\.can_charge\(limits, kind, amount\)\?/, + ], +]) { + expect(coreLimits, name, pattern); +} + +for (const [name, pattern] of [ + [ + "Wasm stores enforce a concrete memory limit", + /StoreLimitsBuilder::new\(\)[\s\S]*memory_size\(runtime\.memory_bytes\)/, + ], + [ + "Wasm compute uses a refillable fuel token bucket", + /struct FuelTokenBucket[\s\S]*fractional_fuel_numerator[\s\S]*fn refill_after/, + ], + ["Wasmtime fuel consumption is enabled", /config\.consume_fuel\(true\)/], + ["Wasmtime epoch interruption is enabled", /config\.epoch_interruption\(true\)/], +]) { + expect(wasmRuntime, name, pattern); +} + +for (const [name, pattern] of [ + [ + "fuel refill preserves fractional credit", + /frequent_refills_preserve_fractional_credit/, + ], + [ + "linear memory growth is bounded per store", + /wasm_linear_memory_growth_is_bounded_per_store/, + ], + [ + "CPU-bound Wasm is interrupted without a host call", + /epoch_interruption_aborts_cpu_bound_wasm_without_a_host_call/, + ], +]) { + expect(`${wasmRuntime}\n${wasmRuntimeTests}`, name, pattern); +} + +for (const [name, pattern] of [ + [ + "rendezvous charges before transport planning", + /handle_request_rendezvous[\s\S]*charge_rendezvous_attempt\([\s\S]*&scope\.tenant[\s\S]*&scope\.project[\s\S]*now_epoch_seconds[\s\S]*plan_authenticated_direct_bulk_transfer/, + ], + [ + "artifact link creation preflights downloadable bytes before link creation", + /handle_create_artifact_download_link[\s\S]*downloadable_size[\s\S]*can_charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*downloadable_size[\s\S]*create_download_link/, + ], + [ + "artifact delivery charges scoped bytes before advancing its offset", + /handle_open_artifact_download_stream[\s\S]*stream_download_chunk\([\s\S]*charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*streamed_bytes[\s\S]*delivered_offset = end/, + ], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, pattern] of [ + [ + "quota keys include tenant/project resource kind and window", + /struct ProjectQuotaScope[\s\S]*tenant: TenantId[\s\S]*project: ProjectId[\s\S]*struct MeterKey[\s\S]*kind: LimitKind[\s\S]*window: u64/, + ], + [ + "quota module discards expired windows for an accessed scope and kind", + /fn meter_mut[\s\S]*self\.meters\.retain[\s\S]*existing\.scope != key\.scope[\s\S]*existing\.kind != kind[\s\S]*existing\.window == key\.window/, + ], + [ + "quota module charges rendezvous attempts through the scoped window meter", + /fn charge_rendezvous_attempt[\s\S]*self\.charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::RendezvousAttempt/, + ], + [ + "quota module charges authenticated API calls through the scoped window meter", + /fn charge_api_call[\s\S]*self\.charge\(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds\)/, + ], + [ + "quota module preflights and charges log bytes through the scoped window meter", + /fn can_charge_log_bytes[\s\S]*LimitKind::LogBytes[\s\S]*fn charge_log_bytes[\s\S]*LimitKind::LogBytes/, + ], + [ + "quota module preflights artifact download bytes through the scoped meter", + /fn can_charge_download[\s\S]*self\.can_charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::ArtifactDownloadBytes/, + ], + [ + "quota status reports current scoped window usage", + /fn project_status[\s\S]*for kind in LimitKind::ALL[\s\S]*self\.used\(tenant, project, kind, now_epoch_seconds\)/, + ], +]) { + expect(coordinatorQuota, name, pattern); +} + +for (const [source, name, pattern] of [ + [ + coordinatorService, + "authenticated API calls are charged after session authorization and before dispatch", + /authenticate_cli_session[\s\S]*authorize_authenticated_user_operation[\s\S]*charge_api_call[\s\S]*match request/, + ], + [ + coordinatorLogs, + "signed node log ingestion preflights and charges bytes before accepting the report", + /handle_report_task_log[\s\S]*authorize_node_for_process_or_termination[\s\S]*can_charge_log_bytes[\s\S]*charge_log_bytes[\s\S]*TaskLogRecorded/, + ], + [ + coordinatorDebug, + "debug reads charge the scoped debug-read budget before audit state is recorded", + /record_debug_audit_event[\s\S]*charge_debug_read[\s\S]*DebugAuditEvent/, + ], + [ + coordinatorTests, + "tests prove API-call and log-byte quota enforcement and project isolation", + /authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch[\s\S]*project-a[\s\S]*project-b[\s\S]*signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes[\s\S]*LogBytes/, + ], +]) { + expect(source, name, pattern); +} + +for (const [name, source, patterns] of [ + [ + "rendezvous smoke", + quicSmoke, + [/charged_rendezvous_attempts, 1/, /coordinator bulk relay is disabled/], + ], + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /downloaded\.response\.charged_download_bytes,[\s\S]*packageEvent\.artifact_size_bytes/, + /retaining_node_reverse_stream/, + /revoked/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [ + /type: "submit_panel_event"/, + /max_events: 1/, + /used_events, 1/, + /rate limit/i, + /max_download_bytes: 1/, + /exceeds download limit/, + ], + ], +]) { + for (const pattern of patterns) { + expect(source, name, pattern); + } +} + +const privateHostedLibSource = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); +const privateHostedTests = maybeRead(["private", "hosted-policy", "src", "tests.rs"]); +const privateHostedLib = privateHostedLibSource + ? [privateHostedLibSource, privateHostedTests].filter(Boolean).join("\n") + : null; +if (privateHostedLib) { + for (const [name, pattern] of [ + [ + "private hosted configuration owns exact control-plane limits and quota windows", + /community_tier_resource_limits[\s\S]*LimitKind::ApiCall[\s\S]*LimitKind::ArtifactDownloadBytes[\s\S]*community_tier_quota_configuration[\s\S]*CoordinatorQuotaConfiguration::new/, + ], + [ + "hosted zero-capability policy rejects native execution capabilities", + /hosted_zero_capability_wasm_rejects_filesystem_and_command_capabilities[\s\S]*Capability::Command[\s\S]*Capability::Network[\s\S]*Capability::ArbitrarySyscalls/, + ], + ]) { + expect(privateHostedLib, name, pattern); + } + assert.doesNotMatch( + privateHostedLib, + /HostedFuel|HostedMemoryBytes|HostedWallClockMs|HostedStateBytes/, + "decorative hosted Wasm quota kinds must not return", + ); +} + +console.log("Resource metering contract smoke passed"); diff --git a/scripts/scheduler-placement-smoke.js b/scripts/scheduler-placement-smoke.js new file mode 100755 index 0000000..0b35350 --- /dev/null +++ b/scripts/scheduler-placement-smoke.js @@ -0,0 +1,397 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const digest = (value) => + `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +const environmentDigest = digest("scheduler-linux-container"); +const dependencyDigest = digest("scheduler-toolchain-dependencies"); +const sourceDigest = digest("scheduler-source-tree"); + +function buildFlagshipBundle() { + const output = cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "build", "--project", "tests/fixtures/runtime-conformance", "--json", + ], + { cwd: repo, encoding: "utf8" } + ); + const report = JSON.parse(output); + const directory = path.resolve(repo, report.bundle_artifact.directory); + const manifest = JSON.parse(fs.readFileSync(path.join(directory, "manifest.json"), "utf8")); + const entrypoints = JSON.parse( + fs.readFileSync(path.join(directory, manifest.entrypoints), "utf8") + ); + const entrypoint = entrypoints.find((candidate) => candidate.name === "build"); + assert(entrypoint, "flagship bundle omitted build entrypoint"); + const taskDescriptors = JSON.parse( + fs.readFileSync(path.join(directory, manifest.task_descriptors), "utf8") + ); + const prepareSource = taskDescriptors.find( + (candidate) => candidate.name === "prepare_source" + ); + assert(prepareSource, "flagship bundle omitted prepare_source task"); + return { + digest: manifest.bundle_digest, + taskExport: prepareSource.export, + wasmModuleBase64: fs.readFileSync(path.join(directory, "module.wasm")).toString("base64"), + }; +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function linuxCapabilities() { + return { + os: "Linux", + arch: "x86_64", + capabilities: [ + "Command", + "Containers", + "RootlessPodman", + "SourceFilesystem", + "VfsArtifacts" + ], + environment_backends: ["Container"], + source_providers: ["filesystem"] + }; +} + +function gitCapabilities() { + const capabilities = linuxCapabilities(); + capabilities.capabilities = [...capabilities.capabilities, "SourceGit"].sort(); + capabilities.source_providers = [...capabilities.source_providers, "git"].sort(); + return capabilities; +} + +async function attachNode(addr, node) { + const identity = nodeIdentity("scheduler-placement-smoke", node); + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node, + public_key: identity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + assert.strictEqual(attached.node, node); + return identity; +} + +async function reportNode(addr, node, identity, locality) { + const recorded = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node, + capabilities: locality.capabilities || linuxCapabilities(), + cached_environment_digests: locality.cached_environment_digests, + dependency_cache_digests: locality.dependency_cache_digests, + source_snapshots: locality.source_snapshots, + artifact_locations: locality.artifact_locations, + direct_connectivity: locality.direct_connectivity !== false, + online: true + })); + assert.strictEqual( + recorded.type, + "node_capabilities_recorded", + JSON.stringify(recorded) + ); + assert.strictEqual(recorded.node, node); + return recorded; +} + +(async () => { + const bundle = buildFlagshipBundle(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const coldNode = await attachNode(addr, "cold-node"); + const warmNode = await attachNode(addr, "warm-node"); + + const cold = await reportNode(addr, "cold-node", coldNode, { + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + assert.strictEqual(cold.node_descriptors, 1); + + const warm = await reportNode(addr, "warm-node", warmNode, { + cached_environment_digests: [environmentDigest], + dependency_cache_digests: [dependencyDigest], + source_snapshots: [sourceDigest], + artifact_locations: ["toolchain-cache"] + }); + assert.strictEqual(warm.node_descriptors, 2); + const reportedNodes = new Set([cold.node, warm.node]); + assert.strictEqual(reportedNodes.size, 2); + assert(reportedNodes.has("cold-node")); + assert(reportedNodes.has("warm-node")); + + const inspected = await send(addr, { + type: "list_node_descriptors", + tenant: "tenant", + project: "project", + actor_user: "operator" + }); + assert.strictEqual(inspected.type, "node_descriptors"); + assert.strictEqual(inspected.actor, "operator"); + assert.strictEqual(inspected.descriptors.length, 2); + const warmDescriptor = inspected.descriptors.find( + (descriptor) => descriptor.id === "warm-node" + ); + assert(warmDescriptor, "warm node descriptor must be visible to inspector state"); + assert(warmDescriptor.capabilities.capabilities.includes("Command")); + assert(warmDescriptor.capabilities.capabilities.includes("RootlessPodman")); + assert(warmDescriptor.cached_environments.includes(environmentDigest)); + assert(warmDescriptor.dependency_caches.includes(dependencyDigest)); + assert(warmDescriptor.source_snapshots.includes(sourceDigest)); + assert(warmDescriptor.artifact_locations.includes("toolchain-cache")); + + const crossScopeInspection = await send(addr, { + type: "list_node_descriptors", + tenant: "other-tenant", + project: "project", + actor_user: "operator" + }); + assert.strictEqual(crossScopeInspection.type, "node_descriptors"); + assert.strictEqual(crossScopeInspection.descriptors.length, 0); + + const crossTenantReport = await send(addr, signedNodeRequest("warm-node", warmNode, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "other-tenant", + project: "project", + node: "warm-node", + capabilities: linuxCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(crossTenantReport.type, "error"); + assert.match(crossTenantReport.message, /tenant\/project scope/); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Linux", + arch: null, + capabilities: ["Containers", "RootlessPodman"] + }, + environment_digest: environmentDigest, + required_capabilities: ["Command"], + dependency_cache: dependencyDigest, + source_snapshot: sourceDigest, + required_artifacts: ["toolchain-cache"], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "warm-node"); + assert.ok(placement.placement.score > 0); + assert.ok(placement.placement.reasons.includes("warm environment cache")); + assert.ok(placement.placement.reasons.includes("warm dependency cache")); + assert.ok(placement.placement.reasons.includes("source snapshot already local")); + assert.ok( + placement.placement.reasons.includes("1 required artifact(s) already local") + ); + + const impossible = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["WindowsCommandDev"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(impossible.type, "error"); + assert.match(impossible.message, /WindowsCommandDev/); + + const started = await send(addr, { + type: "start_process", + tenant: "tenant", + project: "project", + actor_user: "operator", + process: "vp-wait-for-git" + }); + assert.strictEqual(started.type, "process_started"); + + const queued = await send(addr, { + type: "launch_task", + tenant: "tenant", + project: "project", + actor_user: "operator", + task_spec: { + tenant: "tenant", + project: "project", + process: "vp-wait-for-git", + task_definition: "prepare_source", + task_instance: "prepare_source-1", + dispatch: { + kind: "coordinator_node_wasm", + export: bundle.taskExport, + abi: "task_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: ["SourceFilesystem", "SourceGit"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: started.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: true, + artifact_path: "/vfs/artifacts/git-status.txt", + wasm_module_base64: bundle.wasmModuleBase64, + }); + assert.strictEqual(queued.type, "error"); + assert.match(queued.message, /external callers may launch only EntrypointV1/); + + const gitNode = await attachNode(addr, "git-node"); + const gitRecorded = await reportNode(addr, "git-node", gitNode, { + capabilities: gitCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + assert.strictEqual(gitRecorded.type, "node_capabilities_recorded"); + + const pendingAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { + type: "poll_task_assignment", + tenant: "tenant", + project: "project", + node: "git-node" + })); + assert.strictEqual(pendingAssignment.type, "task_assignment"); + assert.strictEqual(pendingAssignment.assignment, null); + + const emptyAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { + type: "poll_task_assignment", + tenant: "tenant", + project: "project", + node: "git-node" + })); + assert.strictEqual(emptyAssignment.type, "task_assignment"); + assert.strictEqual(emptyAssignment.assignment, null); + + await reportNode(addr, "warm-node", warmNode, { + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + const disconnectedTransfer = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["Command"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: ["toolchain-cache"], + prefer_node: null + }); + assert.strictEqual(disconnectedTransfer.type, "error"); + assert.match(disconnectedTransfer.message, /source snapshot unavailable/); + assert.match(disconnectedTransfer.message, /required artifact\(s\) unavailable/); + assert.match(disconnectedTransfer.message, /direct connectivity unavailable/); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Scheduler placement smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/sdk-spawn-runtime-smoke.js b/scripts/sdk-spawn-runtime-smoke.js new file mode 100755 index 0000000..48fa978 --- /dev/null +++ b/scripts/sdk-spawn-runtime-smoke.js @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const sdk = fs.readFileSync(path.join(repo, "crates/clusterflux-sdk/src/lib.rs"), "utf8"); +const productRuntime = fs.readFileSync( + path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), + "utf8" +); + +assert.match(sdk, /pub struct RuntimeSpawnEvent/); +assert.match(sdk, /pub debugger_visible: bool/); +assert.match(sdk, /fn register_runtime_thread/); +assert.match(sdk, /register_runtime_thread\(id, self\.name, self\.env\)/); +assert.match(sdk, /pub fn debugger_visible\(&self\) -> bool/); +assert.match(sdk, /ProductRuntimeConfig::from_env\(\)/); +assert.match(sdk, /start_remote_task\(\s*config,\s*task_id/); +assert.match(sdk, /start_guest_host_task/); +assert.match(sdk, /join_remote_task\(remote\)/); +assert.doesNotMatch(productRuntime, /"type": "launch_task"/); +assert.match(productRuntime, /native SDK task spawning requires coordinator EntrypointV1 execution/); +assert.match(productRuntime, /task_start_v1/); +assert.match(productRuntime, /task_join_v1/); +assert.match(productRuntime, /command_run_v1/); +assert.match(productRuntime, /remote_completion_observed/); +assert.match(productRuntime, /TaskJoinState::Pending/); +assert.doesNotMatch( + productRuntime, + /entry\s*\(/, + "product runtime must not invoke the submitted local Rust closure" +); + +cp.execFileSync( + "cargo", + [ + "test", + "-p", + "clusterflux-sdk", + "spawn_task_start_registers_debugger_visible_runtime_thread", + ], + { cwd: repo, stdio: "inherit" } +); + +console.log("SDK spawn runtime smoke passed"); diff --git a/scripts/self-hosted-coordinator-smoke.js b/scripts/self-hosted-coordinator-smoke.js new file mode 100644 index 0000000..ca35649 --- /dev/null +++ b/scripts/self-hosted-coordinator-smoke.js @@ -0,0 +1,662 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); +const teamArtifactDigest = `sha256:${"a".repeat(64)}`; +const teamEnvironmentDigest = `sha256:${crypto + .createHash("sha256") + .update("env-team-linux") + .digest("hex")}`; +const teamDependencyCacheDigest = `sha256:${crypto + .createHash("sha256") + .update("cargo-cache") + .digest("hex")}`; +const teamSourceDigest = `sha256:${crypto + .createHash("sha256") + .update("source-team") + .digest("hex")}`; +const coordinatorTaskModule = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); +const coordinatorTaskBundleDigest = `sha256:${crypto + .createHash("sha256") + .update(coordinatorTaskModule) + .digest("hex")}`; +const adminToken = "self-hosted-smoke-admin-token"; +const clientSessionSecret = "self-hosted-smoke-client-session-secret"; +const releaseManifestPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || + path.join(repo, "target/public-release/public-release-manifest.json"); + +function sha256(value) { + return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +} + +function digestFromParts(parts) { + const hash = crypto.createHash("sha256"); + for (const value of parts) { + const bytes = Buffer.from(String(value)); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(bytes.length)); + hash.update(length); + hash.update(bytes); + } + return `sha256:${hash.digest("hex")}`; +} + +function adminRequest(token, operation, tenant, actorUser, targetTenant, nonce) { + const issuedAtEpochSeconds = Math.floor(Date.now() / 1000); + return { + type: operation, + tenant, + actor_user: actorUser, + ...(operation === "suspend_tenant" ? { target_tenant: targetTenant } : {}), + admin_proof: digestFromParts([ + "clusterflux-admin-request-proof:v1", + sha256(token), + operation, + tenant, + actorUser, + targetTenant, + nonce, + issuedAtEpochSeconds, + ]), + admin_nonce: nonce, + issued_at_epoch_seconds: issuedAtEpochSeconds, + }; +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function authenticated(request, sessionSecret = clientSessionSecret) { + return { + type: "authenticated", + session_secret: sessionSecret, + request, + }; +} + +function linuxNodeCapabilities() { + return { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "RootlessPodman", "VfsArtifacts", "QuicDirect"], + environment_backends: ["Container"], + source_providers: ["filesystem", "git"] + }; +} + +function nodeIdentity(node) { + void node; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + return { + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; +} + +function signedRequestPayloadDigest(request) { + const canonicalize = (value, topLevel = true) => { + if (Array.isArray(value)) return value.map((entry) => canonicalize(entry, false)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter( + ([key, entry]) => + entry !== null && + (!topLevel || !["agent_signature", "node_signature"].includes(key)) + ) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalize(entry, false)]) + ); + } + return value; + }; + return `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify(canonicalize(request))) + .digest("hex")}`; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeHeartbeat(node, identity) { + const request = { type: "node_heartbeat", node }; + const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + "node_heartbeat", + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedNodeRequest(node, identity, requestKind, request) { + return { + type: "signed_node", + node, + node_signature: signedNodeHeartbeatForKind(node, identity, requestKind, request), + request, + }; +} + +function signedNodeHeartbeatForKind(node, identity, requestKind, request) { + const nonce = `${requestKind}-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + requestKind, + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function commandOutput(command, args) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }) + .trim(); + } catch (_) { + return null; + } +} + +function runJson(command, args, options = {}) { + return JSON.parse( + cp.execFileSync(command, args, { + cwd: options.cwd || repo, + encoding: "utf8", + input: options.input, + }) + ); +} + +function expectedSourceCommit() { + return ( + process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || + commandOutput("git", ["rev-parse", "HEAD"]) + ); +} + +function readReleaseManifest() { + if (!fs.existsSync(releaseManifestPath)) return null; + const manifest = JSON.parse(fs.readFileSync(releaseManifestPath, "utf8")); + if (manifest.kind !== "clusterflux-public-release") return null; + const expectedCommit = expectedSourceCommit(); + if (expectedCommit && manifest.source_commit !== expectedCommit) return null; + return manifest; +} + +function releaseIdentity() { + const manifest = readReleaseManifest(); + return { + sourceCommit: manifest ? manifest.source_commit : expectedSourceCommit(), + releaseName: + (manifest && manifest.release_name) || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || + null, + }; +} + +async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilities()) { + const identity = nodeIdentity(node); + const grant = await send(addr, authenticated({ + type: "create_node_enrollment_grant", + ttl_seconds: 900, + })); + assert.strictEqual(grant.type, "node_enrollment_grant_created"); + + const attached = await send(addr, { + type: "exchange_node_enrollment_grant", + tenant: "team", + project: "self-hosted", + node, + public_key: identity.publicKey, + enrollment_grant: grant.grant, + }); + assert.strictEqual(attached.type, "node_enrollment_exchanged"); + + const heartbeat = await send(addr, { + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity) + }); + assert.strictEqual(heartbeat.type, "node_heartbeat"); + + const reported = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "team", + project: "self-hosted", + node, + capabilities, + cached_environment_digests: [teamEnvironmentDigest], + dependency_cache_digests: [teamDependencyCacheDigest], + source_snapshots: [teamSourceDigest], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(reported.type, "node_capabilities_recorded"); + return identity; +} + +(async () => { + const release = releaseIdentity(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_ADMIN_TOKEN: adminToken, + CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET: clientSessionSecret, + CLUSTERFLUX_SELF_HOSTED_TENANT: "team", + CLUSTERFLUX_SELF_HOSTED_PROJECT: "self-hosted", + CLUSTERFLUX_SELF_HOSTED_USER: "developer", + }, + } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual(ready.client_authority, "strict"); + assert.strictEqual(ready.self_hosted_session_bootstrapped, true); + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const forgedBodyAuthority = await send(addr, { + type: "start_process", + tenant: "victim-tenant", + project: "victim-project", + actor_user: "forged-user", + process: "vp-forged", + }); + assert.strictEqual(forgedBodyAuthority.type, "error"); + assert.match(forgedBodyAuthority.message, /body.*identity.*not authority/i); + const wrongSession = await send( + addr, + authenticated({ type: "auth_status" }, "wrong-session-secret") + ); + assert.strictEqual(wrongSession.type, "error"); + assert.match(wrongSession.message, /session.*not recognized|not recognized.*session/i); + const authStatus = await send(addr, authenticated({ type: "auth_status" })); + assert.strictEqual(authStatus.type, "auth_status"); + assert.strictEqual(authStatus.tenant, "team"); + assert.strictEqual(authStatus.project, "self-hosted"); + assert.strictEqual(authStatus.actor, "developer"); + + const missingAdminCredential = await send( + addr, + adminRequest("", "admin_status", "team", "forged-admin", "team", "admin-empty") + ); + assert.strictEqual(missingAdminCredential.type, "error"); + assert.match(missingAdminCredential.message, /admin.*proof.*invalid/i); + const wrongAdminCredential = await send( + addr, + adminRequest( + "wrong-token", + "admin_status", + "team", + "forged-admin", + "team", + "admin-wrong" + ) + ); + assert.strictEqual(wrongAdminCredential.type, "error"); + assert.match(wrongAdminCredential.message, /admin.*proof.*invalid/i); + const replayableAdminRequest = adminRequest( + adminToken, + "admin_status", + "team", + "self-hosted-admin", + "team", + "admin-replay" + ); + const directAdminStatus = await send(addr, replayableAdminRequest); + assert.strictEqual(directAdminStatus.type, "admin_status"); + const replayedAdminStatus = await send(addr, replayableAdminRequest); + assert.strictEqual(replayedAdminStatus.type, "error"); + assert.match(replayedAdminStatus.message, /nonce was already used/i); + const cargoTargetDir = path.resolve( + process.env.CARGO_TARGET_DIR || path.join(repo, "target") + ); + const cliBin = path.join( + cargoTargetDir, + "debug", + process.platform === "win32" ? "clusterflux.exe" : "clusterflux" + ); + const selfHostedCliProject = path.join( + repo, + "target/acceptance/self-hosted-cli-project" + ); + fs.rmSync(selfHostedCliProject, { recursive: true, force: true }); + fs.mkdirSync(selfHostedCliProject, { recursive: true }); + const cliConnect = runJson( + cliBin, + [ + "auth", + "connect-self-hosted", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--project-id", + "self-hosted", + "--user", + "developer", + "--session-secret-stdin", + "--json", + ], + { cwd: selfHostedCliProject, input: `${clientSessionSecret}\n` } + ); + assert.strictEqual(cliConnect.status, "connected"); + assert.strictEqual(cliConnect.session_secret_read_from_stdin, true); + assert.strictEqual(cliConnect.session_secret_exposed_in_report, false); + assert.strictEqual(cliConnect.coordinator_response.type, "auth_status"); + const sessionPath = path.join( + selfHostedCliProject, + ".clusterflux/session.json" + ); + const storedSession = JSON.parse(fs.readFileSync(sessionPath, "utf8")); + assert.strictEqual(storedSession.kind, "self_hosted"); + assert.strictEqual(storedSession.session_secret, clientSessionSecret); + assert.strictEqual(storedSession.provider_tokens_exposed_to_cli, false); + assert.strictEqual(storedSession.provider_tokens_sent_to_nodes, false); + if (process.platform !== "win32") { + assert.strictEqual(fs.statSync(sessionPath).mode & 0o777, 0o600); + } + const cliAuthStatus = runJson( + cliBin, + ["auth", "status", "--json"], + { cwd: selfHostedCliProject } + ); + assert.strictEqual(cliAuthStatus.active_coordinator, ready.listen); + assert.strictEqual( + cliAuthStatus.coordinator_account_status.used_cli_session_credential, + true + ); + assert.strictEqual( + cliAuthStatus.coordinator_account_status.coordinator_response_type, + "auth_status" + ); + const cliAdminStatus = runJson(cliBin, [ + "admin", + "status", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--user", + "self-hosted-admin", + "--admin-token", + adminToken, + "--json", + ]); + assert.strictEqual(cliAdminStatus.response.type, "admin_status"); + assert.strictEqual(cliAdminStatus.suspended, false); + const cliAdminSuspend = runJson(cliBin, [ + "admin", + "suspend-tenant", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--user", + "self-hosted-admin", + "--target-tenant", + "admin-probe-tenant", + "--admin-token", + adminToken, + "--yes", + "--json", + ]); + assert.strictEqual(cliAdminSuspend.response.type, "tenant_suspended"); + assert.strictEqual(cliAdminSuspend.suspended, true); + const cliAdminProbeStatus = runJson(cliBin, [ + "admin", + "status", + "--coordinator", + ready.listen, + "--tenant", + "admin-probe-tenant", + "--user", + "self-hosted-admin", + "--admin-token", + adminToken, + "--json", + ]); + assert.strictEqual(cliAdminProbeStatus.suspended, true); + + const teamLinuxA = await attachTrustedNode(addr, "team-linux-a"); + const teamLinuxBCapabilities = linuxNodeCapabilities(); + teamLinuxBCapabilities.capabilities = teamLinuxBCapabilities.capabilities.filter( + (capability) => capability !== "VfsArtifacts" + ); + await attachTrustedNode(addr, "team-linux-b", teamLinuxBCapabilities); + + const placement = await send(addr, authenticated({ + type: "schedule_task", + environment: { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "QuicDirect"] + }, + environment_digest: teamEnvironmentDigest, + required_capabilities: ["VfsArtifacts"], + source_snapshot: teamSourceDigest, + required_artifacts: [], + prefer_node: "team-linux-a" + })); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "team-linux-a"); + assert(placement.placement.reasons.includes("preferred node")); + assert(placement.placement.reasons.includes("warm environment cache")); + assert(placement.placement.reasons.includes("source snapshot already local")); + + const started = await send(addr, authenticated({ + type: "start_process", + process: "vp-team-build", + restart: false, + })); + assert.strictEqual(started.type, "process_started"); + + const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", { + type: "reconnect_node", + node: "team-linux-a", + process: "vp-team-build", + epoch: started.epoch + })); + assert.strictEqual(reconnected.type, "node_reconnected"); + + const launched = await send(addr, authenticated({ + type: "launch_task", + task_spec: { + tenant: "team", + project: "self-hosted", + process: "vp-team-build", + task_definition: "compile-linux", + task_instance: "compile-linux", + dispatch: { + kind: "coordinator_node_wasm", + export: "compile_linux", + abi: "task_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: ["VfsArtifacts"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: started.epoch, + bundle_digest: coordinatorTaskBundleDigest, + }, + wait_for_node: false, + artifact_path: "/vfs/artifacts/team-output.txt", + wasm_module_base64: coordinatorTaskModule.toString("base64"), + })); + assert.strictEqual(launched.type, "error", JSON.stringify(launched)); + assert.match(launched.message, /external callers may launch only EntrypointV1/); + + const reportPath = path.join(repo, "target/acceptance/core-coordinator-compat.json"); + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + fs.writeFileSync( + reportPath, + `${JSON.stringify( + { + kind: "clusterflux-core-coordinator-compatibility", + source_commit: release.sourceCommit, + release_name: release.releaseName, + coordinator_implementation: "standalone-core-coordinator", + coordinator_addr: ready.listen, + client_authority: ready.client_authority, + authenticated_session: authStatus.authenticated, + forged_body_authority_denied: forgedBodyAuthority.type, + wrong_session_denied: wrongSession.type, + ping: "pong", + nodes: ["team-linux-a", "team-linux-b"], + task_placement: placement.type, + process_started: started.type, + external_task_v1_denied: launched.type, + self_hosted_admin: { + missing_credential_denied: missingAdminCredential.type, + wrong_credential_denied: wrongAdminCredential.type, + nonce_bound_proof_succeeded: directAdminStatus.type, + replay_denied: replayedAdminStatus.type, + cli_status: cliAdminStatus.response.type, + cli_suspend: cliAdminSuspend.response.type, + suspended_state_observed: cliAdminProbeStatus.suspended, + }, + self_hosted_cli: { + connected: cliConnect.status, + secret_read_from_stdin: cliConnect.session_secret_read_from_stdin, + secret_exposed_in_report: cliConnect.session_secret_exposed_in_report, + session_file_mode: + process.platform === "win32" + ? "windows-best-effort" + : (fs.statSync(sessionPath).mode & 0o777).toString(8), + authenticated_status: + cliAuthStatus.coordinator_account_status.coordinator_response_type, + }, + }, + null, + 2 + )}\n` + ); + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Self-hosted coordinator smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/source-preparation-smoke.js b/scripts/source-preparation-smoke.js new file mode 100755 index 0000000..9f9acd8 --- /dev/null +++ b/scripts/source-preparation-smoke.js @@ -0,0 +1,218 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function sourceCapableNode(sourceProviders = ["git"]) { + return { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "SourceFilesystem", "SourceGit"], + environment_backends: [], + source_providers: sourceProviders + }; +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const pending = await send(addr, { + type: "request_source_preparation", + tenant: "tenant", + project: "project", + provider: "Git" + }); + assert.strictEqual(pending.type, "source_preparation"); + assert.strictEqual(pending.status.preparation.tenant, "tenant"); + assert.strictEqual(pending.status.preparation.project, "project"); + assert.strictEqual(pending.status.preparation.provider, "Git"); + assert.strictEqual( + pending.status.preparation.coordinator_requires_checkout_access, + false + ); + assert.deepStrictEqual(pending.status.preparation.required_capabilities, [ + "SourceGit" + ]); + assert.match(pending.status.disposition.Pending.reason, /waiting|node/i); + + const nodeIdentities = new Map(); + for (const node of ["source-cold", "source-ready"]) { + const identity = nodeIdentity("source-preparation-smoke", node); + nodeIdentities.set(node, identity); + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node, + public_key: identity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + } + + const cold = await send(addr, signedNodeRequest("source-cold", nodeIdentities.get("source-cold"), "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: "source-cold", + capabilities: sourceCapableNode(), + cached_environment_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(cold.type, "node_capabilities_recorded"); + + const readyReport = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: "source-ready", + capabilities: sourceCapableNode(), + cached_environment_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(readyReport.type, "node_capabilities_recorded"); + + const assigned = await send(addr, { + type: "request_source_preparation", + tenant: "tenant", + project: "project", + provider: "Git" + }); + assert.strictEqual(assigned.type, "source_preparation"); + assert(["source-cold", "source-ready"].includes(assigned.status.disposition.Assigned.node)); + assert.strictEqual( + assigned.status.preparation.coordinator_requires_checkout_access, + false + ); + + const crossTenantCompletion = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { + type: "complete_source_preparation", + tenant: "other", + project: "project", + node: "source-ready", + provider: "Git", + source_snapshot: "sha256:source-prepared" + })); + assert.strictEqual(crossTenantCompletion.type, "error"); + assert.match(crossTenantCompletion.message, /tenant\/project scope/i); + + const completed = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { + type: "complete_source_preparation", + tenant: "tenant", + project: "project", + node: "source-ready", + provider: "Git", + source_snapshot: "sha256:source-prepared" + })); + assert.strictEqual(completed.type, "source_preparation_completed"); + assert.strictEqual(completed.node, "source-ready"); + assert.strictEqual(completed.provider, "Git"); + assert.strictEqual(completed.source_snapshot, "sha256:source-prepared"); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["SourceGit"], + source_snapshot: "sha256:source-prepared", + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "source-ready"); + assert( + placement.placement.reasons.includes("source snapshot already local"), + "completed source preparation must update node source locality" + ); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Source preparation smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/tenant-isolation-contract-smoke.js b/scripts/tenant-isolation-contract-smoke.js new file mode 100644 index 0000000..fb93e49 --- /dev/null +++ b/scripts/tenant-isolation-contract-smoke.js @@ -0,0 +1,181 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing tenant-isolation evidence: ${name}`); +} + +const auth = read("crates/clusterflux-core/src/auth.rs"); +const artifact = read("crates/clusterflux-core/src/artifact.rs"); +const operatorPanel = read("crates/clusterflux-core/src/operator_panel.rs"); +const source = read("crates/clusterflux-core/src/source.rs"); +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/nodes.rs"), + read("crates/clusterflux-coordinator/src/service/keys.rs"), + read("crates/clusterflux-coordinator/src/service/artifacts.rs"), + read("crates/clusterflux-coordinator/src/service/processes.rs"), + read("crates/clusterflux-coordinator/src/service/process_launch.rs"), + read("crates/clusterflux-coordinator/src/service/tests.rs"), +].join("\n"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); +const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); + +for (const [name, pattern] of [ + ["auth contexts carry tenant and project", /pub struct AuthContext[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["enrollment grants carry tenant and project", /pub struct EnrollmentGrant[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["node credentials carry tenant and project", /pub struct NodeCredential[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["same tenant/project denies tenant mismatch", /pub fn same_tenant_project[\s\S]*tenant mismatch/], + ["same tenant/project denies project mismatch", /pub fn same_tenant_project[\s\S]*project mismatch/], + ["auth unit denies cross-tenant access", /fn tenant_project_scope_denies_cross_tenant_access\(\)/], +]) { + expect(auth, name, pattern); +} + +for (const [name, pattern] of [ + ["artifact metadata carries tenant and project", /pub struct ArtifactMetadata[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["download links carry tenant project process and actor", /pub struct DownloadLink[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId[\s\S]*pub actor: Actor/], + ["downloads authorize against scoped metadata", /let authz = same_tenant_project\(context, &scope\)/], + ["artifact test denies cross-tenant download", /fn cross_tenant_download_is_denied_even_with_known_artifact_id\(\)/], + ["artifact test denies cross-project download", /fn cross_project_download_is_denied_even_with_known_artifact_id\(\)/], +]) { + expect(artifact, name, pattern); +} + +for (const [name, pattern] of [ + ["panel state carries tenant project and process", /pub struct PanelState[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], + ["panel events carry tenant project and process", /pub struct PanelEvent[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], + ["panel events reject scope mismatch", /PanelError::ScopeMismatch/], + ["panel scope validation compares tenant project process", /self\.tenant != event\.tenant[\s\S]*self\.project != event\.project[\s\S]*self\.process != event\.process/], +]) { + expect(operatorPanel, name, pattern); +} + +expect( + source, + "source preparation carries tenant and project", + /pub struct SourcePreparation[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/ +); + +for (const [name, pattern] of [ + ["project creation rejects foreign tenant reuse", /project id is outside the signed-in tenant scope/], + ["project selection rejects foreign tenant", /project is outside the signed-in tenant scope/], + ["project listing uses tenant-scoped context", /CoordinatorRequest::ListProjects[\s\S]*list_projects\(&context\)/], + ["node capability reports check enrollment tenant scope", /node capability report is outside the enrolled tenant\/project scope/], + ["operator panel stop state is tenant/project/process keyed", /type PanelStopKey = \(TenantId, ProjectId, ProcessId\)/], + ["download service tests tenant mismatch", /cross_tenant\.to_string\(\)\.contains\("tenant mismatch"\)/], + ["download service tests project mismatch", /cross_project\.to_string\(\)\.contains\("project mismatch"\)/], + ["node capability test rejects cross-scope report", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], + ["source preparation test rejects cross-scope completion", /fn service_rejects_source_preparation_completion_outside_node_scope\(\)/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, sourceText, patterns] of [ + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /const crossTenant = await send/, + /const crossProject = await send/, + /const crossTenantOpen = await send/, + /const crossProjectOpen = await send/, + /tenant mismatch/, + /project mismatch/, + /token is invalid/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [/const crossTenant = await send/, /render_operator_panel/, /scope\|tenant\|project/], + ], + [ + "scheduler and capability smoke", + schedulerSmoke, + [ + /const crossScopeInspection = await send/, + /assert\.strictEqual\(crossScopeInspection\.descriptors\.length, 0\)/, + /const crossTenantReport = await send/, + /tenant\\\/project scope/, + ], + ], + [ + "source preparation smoke", + sourcePreparationSmoke, + [/const crossTenantCompletion = await send/, /complete_source_preparation/, /tenant\\\/project scope/i], + ], +]) { + for (const pattern of patterns) { + expect(sourceText, name, pattern); + } +} + +const hostedLibRoot = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); +const hostedServiceSource = maybeRead([ + "private", + "hosted-policy", + "src", + "bin", + "clusterflux-hosted-service.rs", +]); +const hostedLib = hostedLibRoot; +const hostedSmoke = maybeRead([ + "private", + "hosted-policy", + "scripts", + "hosted-client-compat-smoke.js", +]); + +if (hostedLib && hostedServiceSource && hostedSmoke) { + for (const [name, pattern] of [ + ["hosted private layer is a compact identity broker", /pub struct HostedIdentityBroker[\s\S]*cli_session_ttl_seconds/], + ["hosted private layer has no parallel coordinator", /Runtime, persistence, nodes, processes,[\s\S]*remain owned by public CoordinatorService/], + ["hosted service delegates Client state to Core", /core_coordinator: CoordinatorService/], + ["hosted login creates project through Core", /issue_cli_session[\s\S]*AuthenticatedCoordinatorRequest::CreateProject/], + ]) { + expect(`${hostedLib}\n${hostedServiceSource}`, name, pattern); + } + + for (const forbidden of [ + "HostedCommunityControlPlane", + "HostedObservabilitySnapshot", + "agent_public_keys: BTreeMap", + "node_statuses: BTreeMap", + "process_statuses: BTreeMap", + "debug_sessions: BTreeMap", + ]) { + assert( + !hostedLib.includes(forbidden), + `private hosted policy must not reimplement Core state: ${forbidden}` + ); + } + + for (const [name, pattern] of [ + ["client-supplied identity is denied", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], + ["second OIDC subject receives a different tenant", /assert\.notStrictEqual\(victimLogin\.session\.tenant, session\.tenant\)/], + ["cross-tenant process events are denied", /const crossTenantTaskEventsDenied = await sendHostedControl[\s\S]*vp-victim[\s\S]*scope\|denied\|unauthorized/], + ]) { + expect(hostedSmoke, name, pattern); + } +} + +console.log("Tenant isolation contract smoke passed"); diff --git a/scripts/user-session-token-boundary-smoke.js b/scripts/user-session-token-boundary-smoke.js new file mode 100755 index 0000000..d00a9da --- /dev/null +++ b/scripts/user-session-token-boundary-smoke.js @@ -0,0 +1,120 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(file) { + return fs.readFileSync(path.join(repo, file), "utf8"); +} + +function extractBalancedBlock(source, marker) { + const start = source.indexOf(marker); + assert(start >= 0, `missing marker ${marker}`); + const open = source.indexOf("{", start); + assert(open >= 0, `missing opening brace for ${marker}`); + let depth = 0; + for (let index = open; index < source.length; index += 1) { + const char = source[index]; + if (char === "{") depth += 1; + if (char === "}") { + depth -= 1; + if (depth === 0) return source.slice(start, index + 1); + } + } + throw new Error(`missing closing brace for ${marker}`); +} + +function extractEnumVariant(source, variant) { + const marker = ` ${variant} {`; + return extractBalancedBlock(source, marker); +} + +const forbiddenUserSessionCredentials = + /\b(BrowserSession|CliDeviceSession|BrowserLoginFlow|CliLoginFlow|authorization_url|verification_url|device_code|access_token|refresh_token|provider_token|provider_tokens|session_token|user_token|oauth_token)\b/i; + +function assertNoUserSessionCredential(surface, text) { + assert.doesNotMatch( + text, + forbiddenUserSessionCredentials, + `${surface} must not carry user OAuth/browser/session credentials` + ); +} + +const coordinatorService = `${read("crates/clusterflux-coordinator/src/service/protocol.rs")}\n${read("crates/clusterflux-coordinator/src/service/protocol/responses.rs")}`; +for (const variant of [ + "AdminStatus", + "SuspendTenant", + "AttachNode", + "RegisterAgentPublicKey", + "ListAgentPublicKeys", + "RotateAgentPublicKey", + "RevokeAgentPublicKey", + "NodeHeartbeat", + "ReportNodeCapabilities", + "RevokeNodeCredential", + "RequestRendezvous", + "RequestSourcePreparation", + "CompleteSourcePreparation", + "StartProcess", + "ReconnectNode", + "CancelTask", + "CancelProcess", + "PollTaskControl", + "RestartTask", + "DebugAttach", + "TaskCompleted", +]) { + assertNoUserSessionCredential( + `CoordinatorRequest::${variant}`, + extractEnumVariant(coordinatorService, variant) + ); +} + +const nodeRuntime = [ + read("crates/clusterflux-node/src/lib.rs"), + read("crates/clusterflux-node/src/command_runner.rs"), +].join("\n"); +for (const marker of [ + "pub struct LinuxCommandRunPlan", + "pub struct LinuxCommandTaskOutput", + "pub struct CapturedCommandLogs", + "pub struct VirtualThreadCommand", + "pub struct CommandOutput", +]) { + assertNoUserSessionCredential(marker, extractBalancedBlock(nodeRuntime, marker)); +} + +const coreExecution = read("crates/clusterflux-core/src/execution.rs"); +assertNoUserSessionCredential( + "CommandInvocation", + extractBalancedBlock(coreExecution, "pub struct CommandInvocation") +); + +const dapAdapter = read("crates/clusterflux-dap/src/variables.rs"); +assertNoUserSessionCredential( + "DAP variables response", + extractBalancedBlock(dapAdapter, "fn variables_response") +); + +const panel = read("crates/clusterflux-core/src/operator_panel.rs"); +assertNoUserSessionCredential( + "PanelEvent", + extractBalancedBlock(panel, "pub struct PanelEvent") +); + +const auth = read("crates/clusterflux-core/src/auth.rs"); +assert.match( + auth, + /task_credentials_do_not_contain_user_session/, + "core auth must keep the task credential user-session guard" +); +assert.match( + auth, + /CredentialKind::BrowserSession \| CredentialKind::CliDeviceSession/, + "task credential guard must reject browser and CLI sessions" +); + +console.log("User session token boundary smoke passed"); diff --git a/scripts/verify-public-split.sh b/scripts/verify-public-split.sh new file mode 100755 index 0000000..f264477 --- /dev/null +++ b/scripts/verify-public-split.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_commit="$(git -C "$repo_root" rev-parse HEAD)" +export CLUSTERFLUX_ACCEPTANCE_COMMIT="$source_commit" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +tar \ + --exclude='./.git' \ + --exclude='./target' \ + --exclude='./private' \ + --exclude='./internal' \ + --exclude='./experiments' \ + --exclude='./.clusterflux' \ + --exclude='./vscode-extension/node_modules' \ + --exclude='./scripts/containers-home' \ + -C "$repo_root" \ + -cf - . | tar -C "$tmp_dir" -xf - + +if find "$tmp_dir" -path "$tmp_dir/private" -print -quit | grep -q .; then + echo "private directory leaked into public split" >&2 + exit 1 +fi + +if find "$tmp_dir" -path "$tmp_dir/experiments" -print -quit | grep -q .; then + echo "experiments directory leaked into public split" >&2 + exit 1 +fi + +if find "$tmp_dir" -path "$tmp_dir/internal" -print -quit | grep -q .; then + echo "internal directory leaked into public split" >&2 + exit 1 +fi + +(cd "$tmp_dir" && scripts/check-old-name.sh) +(cd "$tmp_dir" && CLUSTERFLUX_FILTERED_PUBLIC_TREE=1 node scripts/check-docs.js) +(cd "$tmp_dir" && scripts/check-code-size.sh) +(cd "$tmp_dir" && node scripts/resource-metering-contract-smoke.js) +(cd "$tmp_dir" && node scripts/hostile-input-contract-smoke.js) +(cd "$tmp_dir" && node scripts/tenant-isolation-contract-smoke.js) +(cd "$tmp_dir" && cargo fmt --all --check) +CARGO_TARGET_DIR="$tmp_dir/target" cargo test \ + --workspace \ + --all-targets \ + --manifest-path "$tmp_dir/Cargo.toml" +CARGO_TARGET_DIR="$tmp_dir/target" cargo build \ + --workspace \ + --all-targets \ + --manifest-path "$tmp_dir/Cargo.toml" +(cd "$tmp_dir" && node scripts/cli-install-smoke.js) +(cd "$tmp_dir" && node scripts/flagship-demo-smoke.js) +(cd "$tmp_dir" && node scripts/cli-local-run-smoke.js) +(cd "$tmp_dir" && node scripts/node-attach-smoke.js) +(cd "$tmp_dir" && node scripts/vscode-extension-smoke.js) +(cd "$tmp_dir" && node scripts/vscode-f5-smoke.js) +(cd "$tmp_dir" && node scripts/artifact-download-smoke.js) +(cd "$tmp_dir" && node scripts/artifact-export-smoke.js) + +public_digest="$( + find "$tmp_dir" \ + -path "$tmp_dir/target" -prune -o \ + -type f -print0 \ + | LC_ALL=C sort -z \ + | xargs -0 sha256sum \ + | sha256sum \ + | cut -d' ' -f1 +)" +printf 'Filtered public tree passed: commit=%s digest=sha256:%s\n' \ + "$source_commit" \ + "$public_digest" diff --git a/scripts/vscode-extension-smoke.js b/scripts/vscode-extension-smoke.js new file mode 100755 index 0000000..7eddcea --- /dev/null +++ b/scripts/vscode-extension-smoke.js @@ -0,0 +1,229 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const assert = require("assert"); + +const extension = require("../vscode-extension/extension"); +const packageJson = require("../vscode-extension/package.json"); +const extensionSource = fs.readFileSync( + path.join(__dirname, "../vscode-extension/extension.js"), + "utf8" +); +const repo = path.resolve(__dirname, ".."); + +assert.strictEqual(packageJson.main, "./extension.js"); +assert(fs.existsSync(path.join(__dirname, "../vscode-extension", packageJson.main))); +assert.deepStrictEqual(packageJson.dependencies || {}, {}); +const root = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-vscode-")); +fs.mkdirSync(path.join(root, "envs/linux"), { recursive: true }); +fs.writeFileSync(path.join(root, "envs/linux/Containerfile"), "FROM alpine\n"); +fs.mkdirSync(path.join(root, ".clusterflux"), { recursive: true }); +fs.writeFileSync( + extension.clusterfluxViewStatePath(root), + JSON.stringify({ + nodes: [{ id: "node-linux", status: "online", capabilities: "Command RootlessPodman" }], + processes: [{ id: "vp-build", status: "running", entry: "build" }], + logs: [{ task: "compile-linux", message: "stdout=12 stderr=0", bytes: 12 }], + artifacts: [{ path: "/vfs/artifacts/app.tar.zst", status: "retained", size: 12 }], + inspector: [{ label: "debug", value: "attached" }] + }) +); + +const envs = extension.discoverEnvironmentNames(root); +assert.deepStrictEqual(envs, ["linux"]); + +const diagnostics = extension.diagnoseEnvReferences( + 'let _ = env!("linux"); let _ = env!("windows");', + envs +); +assert.strictEqual(diagnostics.length, 1); +assert.strictEqual(diagnostics[0].name, "windows"); +assert.match(diagnostics[0].message, /envs\/windows\/Containerfile/); + +const inspectCommand = extension.bundleInspectCommand(root, "/repo"); +assert.strictEqual(inspectCommand.command, "cargo"); +assert.deepStrictEqual(inspectCommand.args.slice(0, 8), [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "bundle" +]); +assert(inspectCommand.args.includes("inspect")); +assert(inspectCommand.args.includes("--project")); +assert(inspectCommand.args.includes(root)); +assert(inspectCommand.args.includes("--json")); + +const refreshed = extension.refreshBundleBeforeLaunch(root, "/repo", (command, args, options) => { + assert.strictEqual(command, "cargo"); + assert(args.includes("bundle")); + assert.strictEqual(options.cwd, "/repo"); + return { + status: 0, + stdout: JSON.stringify({ metadata: { identity: "sha256:abc" } }), + stderr: "" + }; +}); +assert.strictEqual(refreshed.metadata.identity, "sha256:abc"); + +const launch = extension.resolveClusterfluxDebugConfiguration( + { uri: { fsPath: root } }, + {} +); +assert.deepStrictEqual(launch, { + name: "Clusterflux: Launch Virtual Process", + type: "clusterflux", + request: "launch", + entry: "build", + project: root, + runtimeBackend: "local-services" +}); +assert.strictEqual( + extension.clusterfluxProcessId("/workspace/app", "build"), + "vp-e4bd6ef50539" +); +assert.strictEqual( + extension.existingProcessRelationship( + { process: "vp-e4bd6ef50539", state: "running" }, + "vp-e4bd6ef50539" + ), + "same_launch_target" +); +assert.strictEqual( + extension.existingProcessRelationship( + { process: "vp-other", state: "running" }, + "vp-e4bd6ef50539" + ), + "different_launch_target" +); + +const liveProcesses = extension.loadLiveProcesses(root, "/repo", (_command, args) => { + assert.deepStrictEqual(args.slice(-3), ["process", "list", "--json"]); + return { + status: 0, + stdout: JSON.stringify({ + coordinator: "https://clusterflux.michelpaulissen.com", + tenant: "tenant-live", + project: "project-live", + user: "user-live", + processes: [{ process: "vp-live", state: "cancelling" }] + }), + stderr: "" + }; +}); +assert.deepStrictEqual(liveProcesses.processes, [ + { process: "vp-live", state: "cancelling" } +]); +assert.strictEqual(liveProcesses.project, "project-live"); + +const adapter = extension.debugAdapterExecutableSpec(root, repo); +assert.strictEqual(adapter.command, "cargo"); +assert.deepStrictEqual(adapter.args, [ + "run", + "-q", + "-p", + "clusterflux-dap", + "--bin", + "clusterflux-debug-dap" +]); +assert.deepStrictEqual(adapter.options, { cwd: repo }); + +const releasedAdapterPath = path.join(root, process.platform === "win32" ? "clusterflux-debug-dap.exe" : "clusterflux-debug-dap"); +fs.writeFileSync(releasedAdapterPath, ""); +const releasedAdapter = extension.debugAdapterExecutableSpec(root, repo); +assert.strictEqual(releasedAdapter.command, releasedAdapterPath); +assert.deepStrictEqual(releasedAdapter.args, []); +assert.deepStrictEqual(releasedAdapter.options, { cwd: root }); + +assert.throws( + () => + extension.refreshBundleBeforeLaunch(root, "/repo", () => ({ + status: 1, + stdout: "", + stderr: "missing environment linux" + })), + /missing environment linux/ +); + +assert( + packageJson.contributes.viewsContainers.activitybar.some( + (container) => container.id === "clusterflux" && container.title === "Clusterflux" + ), + "package.json must contribute a Clusterflux activity-bar container" +); +assert( + fs.existsSync(path.join(__dirname, "../vscode-extension/resources/clusterflux.svg")), + "Clusterflux activity-bar icon must exist" +); +const packageViewIds = packageJson.contributes.views.clusterflux.map((view) => view.id).sort(); +const descriptorViewIds = extension.clusterfluxViewDescriptors().map((view) => view.id).sort(); +assert.deepStrictEqual(descriptorViewIds, [ + "clusterflux.artifacts", + "clusterflux.inspector", + "clusterflux.logs", + "clusterflux.nodes", + "clusterflux.processes" +]); +assert.deepStrictEqual(packageViewIds, descriptorViewIds); +for (const viewId of descriptorViewIds) { + const items = extension.clusterfluxViewItems( + extension.loadClusterfluxViewState(root), + viewId + ); + assert( + items.length > 0 && !items[0].label.startsWith("No "), + `${viewId} should render state-backed items` + ); +} +assert.deepStrictEqual( + extension.clusterfluxViewItems(extension.loadClusterfluxViewState(root), "clusterflux.nodes")[0], + { + label: "node-linux", + description: "online Command RootlessPodman" + } +); + +assert( + packageJson.contributes.debuggers.some((debuggerContribution) => debuggerContribution.type === "clusterflux"), + "package.json must contribute the clusterflux debugger type" +); +for (const viewId of descriptorViewIds) { + assert( + packageJson.activationEvents.includes(`onView:${viewId}`), + `${viewId} should activate the extension when opened` + ); +} +const launchProperties = + packageJson.contributes.debuggers[0].configurationAttributes.launch.properties; +assert.strictEqual(launchProperties.runtimeBackend.default, "local-services"); +assert(launchProperties.runtimeBackend.enum.includes("live-services")); +assert.strictEqual(launchProperties.coordinatorEndpoint.default, undefined); +assert.strictEqual(launchProperties.tenant, undefined); +assert.strictEqual(launchProperties.projectId, undefined); +assert.strictEqual(launchProperties.actorUser, undefined); +assert( + packageJson.contributes.debuggers[0].configurationAttributes.attach, + "package.json must contribute an attach configuration" +); +for (const command of [ + "clusterflux.refreshProcesses", + "clusterflux.process.attach", + "clusterflux.process.cancel", + "clusterflux.process.abort" +]) { + assert( + packageJson.contributes.commands.some((entry) => entry.command === command), + `${command} must be contributed` + ); +} +assert.match(extensionSource, /registerDebugAdapterDescriptorFactory\("clusterflux"/); +assert.match(extensionSource, /clusterflux-debug-dap/); +assert.match(extensionSource, /\.clusterflux\/views\.json/); + +fs.rmSync(root, { recursive: true, force: true }); +console.log("VS Code extension smoke passed"); diff --git a/scripts/vscode-f5-smoke.js b/scripts/vscode-f5-smoke.js new file mode 100755 index 0000000..d7b7893 --- /dev/null +++ b/scripts/vscode-f5-smoke.js @@ -0,0 +1,321 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const extension = require("../vscode-extension/extension"); + +class DapClient { + constructor(spec) { + this.child = cp.spawn(spec.command, spec.args, { + cwd: spec.options && spec.options.cwd, + env: spec.options && spec.options.env, + detached: process.platform !== "win32" + }); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(message)); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error( + `DAP ${command} failed: ${message.message || JSON.stringify(message)}\n${this.stderr}` + ); + } + return message; + } + + terminate() { + if (this.child.exitCode !== null) return; + if (process.platform === "win32") { + this.child.kill("SIGKILL"); + return; + } + try { + process.kill(-this.child.pid, "SIGKILL"); + } catch (_) { + this.child.kill("SIGKILL"); + } + } + + waitFor(predicate, timeoutMs = 240000) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.terminate(); + reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +(async () => { + const repo = path.resolve(__dirname, ".."); + const project = path.join(repo, "examples/hello-build"); + const buildSourceLines = fs + .readFileSync(path.join(project, "src/lib.rs"), "utf8") + .split(/\r?\n/); + const sourceLine = (needle) => { + const index = buildSourceLines.findIndex((line) => line.includes(needle)); + assert(index >= 0, `flagship source must contain ${needle}`); + return index + 1; + }; + const buildMainLine = sourceLine("async fn build()"); + const launchConfig = extension.resolveClusterfluxDebugConfiguration( + { uri: { fsPath: project } }, + {} + ); + + assert.strictEqual(launchConfig.type, "clusterflux"); + assert.strictEqual(launchConfig.request, "launch"); + assert.strictEqual(launchConfig.entry, "build"); + assert.strictEqual(launchConfig.project, project); + assert.strictEqual(launchConfig.runtimeBackend, "local-services"); + + const inspection = extension.refreshBundleBeforeLaunch(project, repo); + assert.match(inspection.metadata.identity, /^sha256:/); + + // Keep the timed attach focused on the runtime boundary. A completely fresh + // public checkout may otherwise spend most of that window compiling the + // coordinator or node after the adapter has already started waiting. + cp.execFileSync( + "cargo", + [ + "build", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-node", + ], + { cwd: repo, stdio: "inherit" } + ); + const executableSuffix = process.platform === "win32" ? ".exe" : ""; + const adapterSpec = extension.debugAdapterExecutableSpec(repo); + adapterSpec.options = { + ...(adapterSpec.options || {}), + env: { + ...process.env, + CLUSTERFLUX_COORDINATOR_BIN: path.join( + repo, + "target", + "debug", + `clusterflux-coordinator${executableSuffix}` + ), + CLUSTERFLUX_NODE_BIN: path.join( + repo, + "target", + "debug", + `clusterflux-node${executableSuffix}` + ), + }, + }; + + const client = new DapClient(adapterSpec); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", launchConfig); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const breakpoints = client.send("setBreakpoints", { + source: { path: path.join(project, "src/lib.rs") }, + breakpoints: [{ line: buildMainLine }] + }); + const breakpointResponse = await client.response(breakpoints, "setBreakpoints"); + assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, false); + assert.match( + breakpointResponse.body.breakpoints[0].message, + /Pending coordinator breakpoint installation/ + ); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const installedBreakpoint = await client.waitFor( + (message) => + message.type === "event" && + message.event === "breakpoint" && + message.body?.breakpoint?.verified === true + ); + assert.strictEqual(installedBreakpoint.body.breakpoint.line, buildMainLine); + const stopped = await client.waitFor( + (message) => message.type === "event" && message.event === "stopped" + ); + assert.strictEqual(stopped.body.allThreadsStopped, true); + assert.strictEqual(stopped.body.reason, "breakpoint"); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body.threads; + const mainThread = threads.find((thread) => thread.name.includes("build coordinator main")); + assert(mainThread, "F5 launch must expose the real coordinator-main entrypoint thread"); + + const stackRequest = client.send("stackTrace", { + threadId: mainThread.id, + startFrame: 0, + levels: 1 + }); + const stack = (await client.response(stackRequest, "stackTrace")).body.stackFrames; + assert.strictEqual(stack[0].line, buildMainLine); + assert.strictEqual(stack[0].source.path, path.join(project, "src/lib.rs")); + assert.strictEqual(stack[0].source.sourceReference || 0, 0); + + const sourceRequest = client.send("source", { source: stack[0].source }); + const source = (await client.response(sourceRequest, "source")).body; + assert.match(source.content, /compile/); + + const scopesRequest = client.send("scopes", { frameId: stack[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const localsScope = scopes.find((scope) => scope.name === "Source Locals"); + const argsScope = scopes.find((scope) => scope.name === "Task Args and Handles"); + const runtimeScope = scopes.find((scope) => scope.name === "Clusterflux Runtime"); + const outputScope = scopes.find((scope) => scope.name === "Recent Output"); + assert(localsScope, "F5 launch must expose source locals scope"); + assert(argsScope, "F5 launch must expose task args and handles"); + assert(runtimeScope, "F5 launch must expose Clusterflux runtime state"); + assert(outputScope, "F5 launch must expose recent output state"); + + const localsRequest = client.send("variables", { + variablesReference: localsScope.variablesReference + }); + const locals = (await client.response(localsRequest, "variables")).body.variables; + assert( + locals.some( + (variable) => + variable.name === "unavailable-local-diagnostic" && + String(variable.value).includes("cannot be inspected") + ), + "source locals scope must report unavailable real Rust locals explicitly" + ); + + const runtimeRequest = client.send("variables", { + variablesReference: runtimeScope.variablesReference + }); + const runtime = (await client.response(runtimeRequest, "variables")).body.variables; + assert( + runtime.some( + (variable) => variable.name === "runtime_backend" && variable.value === "LocalServices" + ), + "extension-resolved F5 launch must use the real local-services backend" + ); + assert( + runtime.some( + (variable) => variable.name === "coordinator_task_events" && variable.value === 0 + ), + "a task frozen at its entry probe must not fabricate a terminal task event" + ); + assert( + runtime.some( + (variable) => + variable.name === "command_status" && + String(variable.value).includes( + "frozen through local services at executing Wasm probe" + ) + ) + ); + assert( + runtime.some( + (variable) => variable.name === "state" && variable.value === "Frozen" + ), + "F5 must expose the node-acknowledged frozen participant state" + ); + assert(runtime.some((variable) => variable.name === "command_spec")); + assert(runtime.some((variable) => variable.name === "stdout_tail")); + assert(runtime.some((variable) => variable.name === "stderr_tail")); + + const outputRequest = client.send("variables", { + variablesReference: outputScope.variablesReference + }); + const output = (await client.response(outputRequest, "variables")).body.variables; + assert(output.some((variable) => variable.name === "stdout_tail")); + assert(output.some((variable) => variable.name === "stderr_tail")); + + await client.close(); + } catch (error) { + client.terminate(); + throw error; + } + + console.log("VS Code F5 smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/wasmtime-assignment-smoke.js b/scripts/wasmtime-assignment-smoke.js new file mode 100755 index 0000000..9ec2683 --- /dev/null +++ b/scripts/wasmtime-assignment-smoke.js @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const sdkRuntime = fs.readFileSync( + path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), + "utf8" +); + +assert.doesNotMatch( + sdkRuntime, + /"type": "launch_task"/, + "native SDK code must not submit external TaskV1 work" +); +assert.match( + sdkRuntime, + /native SDK task spawning requires coordinator EntrypointV1 execution/ +); + +cp.execFileSync("node", ["scripts/cli-local-run-smoke.js"], { + cwd: repo, + stdio: "inherit", +}); +cp.execFileSync( + "cargo", + ["test", "-p", "clusterflux-node", "wasmtime_runtime_runs_named_task_export"], + { cwd: repo, stdio: "inherit" } +); +cp.execFileSync( + "cargo", + [ + "test", + "-p", + "clusterflux-coordinator", + "signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only", + ], + { cwd: repo, stdio: "inherit" } +); + +console.log("Wasmtime assignment smoke passed"); diff --git a/scripts/wasmtime-node-smoke.js b/scripts/wasmtime-node-smoke.js new file mode 100644 index 0000000..7cbb1ac --- /dev/null +++ b/scripts/wasmtime-node-smoke.js @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const wasmTarget = path.join( + repo, + "target", + "wasm32-unknown-unknown", + "release", + "runtime_conformance.wasm" +); + +cp.execFileSync( + "cargo", + [ + "build", + "--release", + "-p", + "runtime-conformance", + "--target", + "wasm32-unknown-unknown", + ], + { + cwd: repo, + stdio: "inherit", + env: { + ...process.env, + CARGO_PROFILE_RELEASE_OPT_LEVEL: "z", + CARGO_PROFILE_RELEASE_LTO: "thin", + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1", + }, + } +); + +assert(fs.existsSync(wasmTarget), `missing compiled Wasm module at ${wasmTarget}`); + +const output = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + wasmTarget, + "task_add_one", + "41", + "42", + ], + { cwd: repo, encoding: "utf8" } +); + +const report = JSON.parse(output); +assert.strictEqual(report.type, "wasmtime_task_smoke"); +assert.strictEqual(report.export, "task_add_one"); +assert.strictEqual(report.arg, 41); +assert.strictEqual(report.result, 42); + +const debugOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--debug-freeze-resume", + wasmTarget, + "task_add_one", + "41", + "42", + ], + { cwd: repo, encoding: "utf8" } +); +const debugReport = JSON.parse(debugOutput); +assert.strictEqual(debugReport.type, "wasmtime_debug_freeze_resume_smoke"); +assert.strictEqual(debugReport.export, "task_add_one"); +assert.strictEqual(debugReport.task, "task_add_one"); +assert.strictEqual(debugReport.frozen_state, "Frozen"); +assert.strictEqual(debugReport.resumed_state, "Running"); +assert(debugReport.stack_frames.some((frame) => String(frame).includes("task_add_one"))); +assert( + debugReport.local_values.some( + ([name, value]) => name === "wasm_local_0" && String(value).includes("41") + ), + "Wasmtime debug snapshot must expose the real i32 argument as a frame local" +); +assert.strictEqual(debugReport.node_runtime_captured_wasm_locals, true); +assert.strictEqual(debugReport.result, 42); +assert.strictEqual(debugReport.node_runtime_reached_wasm_task, true); + +const hostCommandOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--host-command", + wasmTarget, + "compile_linux", + ], + { cwd: repo, encoding: "utf8" } +); +const hostCommandReport = JSON.parse(hostCommandOutput); +assert.strictEqual(hostCommandReport.type, "wasmtime_host_command_smoke"); +assert.strictEqual(hostCommandReport.task, "compile_linux"); +assert.match(hostCommandReport.export, /^clusterflux_task_v1_[0-9a-f]{64}$/); +assert.strictEqual(hostCommandReport.program, "cc"); +assert.deepStrictEqual(hostCommandReport.args, [ + "-Os", + "-static", + "-s", + "fixture/hello-clusterflux.c", + "-o", + "/clusterflux/output/hello-clusterflux", +]); +assert.strictEqual(hostCommandReport.working_directory, "/workspace"); +assert.deepStrictEqual(hostCommandReport.environment_variables, { + SOURCE_DATE_EPOCH: "0", +}); +assert.strictEqual(hostCommandReport.timeout_ms, 180000); +assert.strictEqual(hostCommandReport.network, "disabled"); +assert.strictEqual(hostCommandReport.status_code, 0); +assert.strictEqual(hostCommandReport.stdout, ""); +assert.strictEqual(hostCommandReport.artifact_name, "hello-clusterflux"); +assert.strictEqual(hostCommandReport.artifact_size_bytes, "hello-clusterflux".length); +assert.match(hostCommandReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); +assert.strictEqual(hostCommandReport.node_host_import, "clusterflux.command_run_v1"); +assert.strictEqual(hostCommandReport.artifact_host_import, "clusterflux.vfs_operation_v1"); +assert.strictEqual(hostCommandReport.flagship_linux_build_task, true); +assert.strictEqual(hostCommandReport.node_executed_host_command, true); +assert.strictEqual(hostCommandReport.hosted_control_plane_ran_command, false); + +const artifactOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--task-artifact", + wasmTarget, + "package_release", + ], + { cwd: repo, encoding: "utf8" } +); +const artifactReport = JSON.parse(artifactOutput); +assert.strictEqual(artifactReport.type, "wasmtime_task_artifact_smoke"); +assert.strictEqual(artifactReport.task, "package_release"); +assert.strictEqual(artifactReport.artifact_name, "release.tar"); +assert.strictEqual(artifactReport.artifact_size_bytes, "release.tar".length); +assert.match(artifactReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); +assert.strictEqual(artifactReport.host_import, "clusterflux.vfs_operation_v1"); +assert.strictEqual(artifactReport.host_issued_handle_returned, true); +assert.ok(artifactReport.artifact.id.endsWith(artifactReport.artifact_digest.slice("sha256:".length))); + +console.log("Wasmtime node smoke passed"); diff --git a/scripts/windows-best-effort-smoke.js b/scripts/windows-best-effort-smoke.js new file mode 100755 index 0000000..bdb0442 --- /dev/null +++ b/scripts/windows-best-effort-smoke.js @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const digest = (value) => + `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +const environmentDigest = digest("windows-command-dev-environment"); +const sourceDigest = digest("windows-best-effort-source"); +const windowsArtifactBytes = Buffer.from("windows-output-data"); +const windowsArtifactDigest = digest(windowsArtifactBytes); +const emptyWasm = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); +const emptyWasmDigest = digest(emptyWasm); +const nodeDaemon = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/daemon.rs"), "utf8"); +const taskReports = fs.readFileSync( + path.join(repo, "crates/clusterflux-node/src/task_reports.rs"), + "utf8" +); +const nodeLib = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/lib.rs"), "utf8"); +const windowsDev = fs.readFileSync( + path.join(repo, "crates/clusterflux-node/src/windows_dev.rs"), + "utf8" +); +const executionCore = fs.readFileSync( + path.join(repo, "crates/clusterflux-core/src/execution.rs"), + "utf8" +); +const dapSource = [ + fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/demo_backend.rs"), "utf8"), + fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/tests.rs"), "utf8"), +].join("\n"); +const windowsNode = "windows-node"; +const windowsIdentity = nodeIdentity("windows-best-effort-smoke", windowsNode); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function windowsCapabilities() { + return { + os: "Windows", + arch: "x86_64", + capabilities: ["Command", "VfsArtifacts", "WindowsCommandDev"], + environment_backends: ["WindowsCommandDev"], + source_providers: ["filesystem"] + }; +} + +function assertWindowsBackendBoundary() { + assert.match(nodeDaemon, /CoordinatorSession::connect\(&args\.coordinator\)/); + assert.match(nodeDaemon, /record_completed_task\(/); + assert.match(taskReports, /"type": "task_completed"/); + assert.doesNotMatch(nodeDaemon, /WindowsCommandDev|windows-command-dev|cfg\(windows\)/); + + assert.match(nodeLib, /impl CommandBackend for LinuxRootlessPodmanBackend/); + assert.match(nodeLib, /mod windows_dev/); + assert.match(nodeLib, /pub use windows_dev::\{WindowsCommandDevBackend, WindowsSandboxStubBackend\}/); + assert.match(windowsDev, /impl CommandBackend for WindowsCommandDevBackend/); + assert.match(windowsDev, /impl CommandBackend for WindowsSandboxStubBackend/); + assert.doesNotMatch(windowsDev, /LinuxRootlessPodmanBackend/); + assert.match(executionCore, /\bLinuxRootlessPodman\b/); + assert.match(executionCore, /\bWindowsCommandDev\b/); + assert.match(executionCore, /\bStubbedWindowsSandbox\b/); + + assert.match(dapSource, /thread\(WINDOWS_THREAD, "compile-windows", "compile windows"/); + assert.match(dapSource, /fn launch_threads_include_windows_task_in_same_virtual_process\(\)/); + assert.match(dapSource, /fn windows_thread_runtime_variables_share_virtual_process\(\)/); +} + +(async () => { + assertWindowsBackendBoundary(); + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node: windowsNode, + public_key: windowsIdentity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + assert.strictEqual(attached.node, windowsNode); + + const recorded = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: windowsNode, + capabilities: windowsCapabilities(), + cached_environment_digests: [environmentDigest], + dependency_cache_digests: [], + source_snapshots: [sourceDigest], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(recorded.type, "node_capabilities_recorded"); + assert.strictEqual(recorded.node, windowsNode); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"] + }, + environment_digest: environmentDigest, + required_capabilities: ["Command"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, windowsNode); + assert.ok(placement.placement.reasons.includes("warm environment cache")); + assert.ok(placement.placement.reasons.includes("source snapshot already local")); + + const started = await send(addr, { + type: "start_process", + tenant: "tenant", + project: "project", + process: "vp-windows" + }); + assert.strictEqual(started.type, "process_started"); + assert.strictEqual(started.process, "vp-windows"); + + const reconnected = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "reconnect_node", { + type: "reconnect_node", + node: windowsNode, + process: "vp-windows", + epoch: started.epoch + })); + assert.strictEqual(reconnected.type, "node_reconnected"); + + const launched = await send(addr, { + type: "launch_task", + tenant: "tenant", + project: "project", + actor_user: "operator", + task_spec: { + tenant: "tenant", + project: "project", + process: "vp-windows", + task_definition: "windows-command-dev", + task_instance: "windows-command-dev", + dispatch: { + kind: "coordinator_node_wasm", + export: "windows_command_dev", + abi: "task_v1", + }, + environment_id: "windows", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"], + }, + environment_digest: environmentDigest, + required_capabilities: ["Command", "WindowsCommandDev"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: [], + args: [{ SourceSnapshot: sourceDigest }], + vfs_epoch: started.epoch, + bundle_digest: emptyWasmDigest, + }, + wait_for_node: false, + artifact_path: "/vfs/artifacts/windows-output.txt", + wasm_module_base64: emptyWasm.toString("base64"), + }); + assert.strictEqual(launched.type, "error", JSON.stringify(launched)); + assert.match(launched.message, /external callers may launch only EntrypointV1/); + + const recordedTask = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "task_completed", { + type: "task_completed", + tenant: "tenant", + project: "project", + process: "vp-windows", + node: windowsNode, + task: "windows-command-dev", + status_code: 0, + stdout_bytes: 18, + stderr_bytes: 0, + stdout_tail: "", + stderr_tail: "", + stdout_truncated: false, + stderr_truncated: false, + artifact_path: "/vfs/artifacts/windows-output.txt", + artifact_digest: windowsArtifactDigest, + artifact_size_bytes: windowsArtifactBytes.length + })); + assert.strictEqual(recordedTask.type, "error"); + assert.match(recordedTask.message, /coordinator-issued task instance|issued active task|active task/); + + const events = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process: "vp-windows" + }); + assert.strictEqual(events.type, "task_events"); + assert.strictEqual(events.events.length, 0); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact: "windows-output.txt", + max_bytes: 1024 + }); + assert.strictEqual(link.type, "error"); + assert.match(link.message, /does not exist|not found|unavailable/); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Windows best-effort smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/windows-runner-smoke.js b/scripts/windows-runner-smoke.js new file mode 100755 index 0000000..b7b3628 --- /dev/null +++ b/scripts/windows-runner-smoke.js @@ -0,0 +1,479 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const forgejoWindowsNode = "forgejo-windows-node"; +const forgejoWindowsIdentity = nodeIdentity("windows-runner-smoke", forgejoWindowsNode); +const windowsEnvironmentDigest = `sha256:${crypto + .createHash("sha256") + .update("clusterflux/windows-command-dev/v1") + .digest("hex")}`; +const sourceSnapshot = `sha256:${crypto + .createHash("sha256") + .update("clusterflux/windows-runner/source/v1") + .digest("hex")}`; + +class DapClient { + constructor() { + this.child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-dap", "--bin", "clusterflux-debug-dap"], + { cwd: repo } + ); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(coordinatorWireRequest(message))); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error(`DAP ${command} failed: ${message.message || JSON.stringify(coordinatorWireRequest(message))}`); + } + return message; + } + + waitFor(predicate, timeoutMs = 120000) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.child.kill("SIGKILL"); + reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function runWindowsAttach(addr, grant) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "node", + "attach", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + forgejoWindowsNode, + "--public-key", + forgejoWindowsIdentity.publicKey, + "--enrollment-grant", + grant, + "--cap", + "windows-command-dev", + "--json" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Windows node attach failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject( + new Error(`Windows node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) + ); + } + }); + }); +} + +function runWindowsNode(addr, grant) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-node", + "--", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + forgejoWindowsNode, + "--public-key", + forgejoWindowsIdentity.publicKey, + "--enrollment-grant", + grant, + "--process", + "vp-forgejo-windows", + "--task", + "windows-command-dev", + "--command", + "cmd", + "--arg", + "/C", + "--arg", + "echo clusterflux-windows-runner", + "--artifact", + "/vfs/artifacts/windows-runner-output.txt" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Windows node smoke failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout.trim().split(/\r?\n/).at(-1))); + } catch (error) { + reject(new Error(`node output was not JSON: ${stdout}\n${error.stack || error.message}`)); + } + }); + }); +} + +async function assertDebuggerShowsWindowsThread() { + const client = new DapClient(); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "build", + project: path.join(repo, "tests/fixtures/runtime-conformance"), + runtimeBackend: "simulated" + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body.threads; + assert( + threads.some((thread) => thread.name.includes("compile windows")), + "debugger must represent the Windows task as a virtual thread" + ); + + await client.close(); + } catch (error) { + client.child.kill("SIGKILL"); + throw error; + } +} + +(async () => { + if (process.platform !== "win32") { + throw new Error( + `Windows runner validation requires win32; current platform is ${process.platform}` + ); + } + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const attachGrant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(attachGrant.type, "node_enrollment_grant_created"); + assert.strictEqual(attachGrant.scope, "node:attach"); + + const attach = await runWindowsAttach(addr, attachGrant.grant); + assert.strictEqual(attach.plan.node, forgejoWindowsNode); + assert.strictEqual(attach.boundary.cli_contacted_coordinator, true); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + assert.strictEqual(attach.coordinator_response.type, "node_enrollment_exchanged"); + assert.strictEqual(attach.coordinator_response.credential.scope, "node:attach"); + assert(attach.plan.capabilities.capabilities.includes("WindowsCommandDev")); + + const runtimeGrant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(runtimeGrant.type, "node_enrollment_grant_created"); + + const report = await runWindowsNode(addr, runtimeGrant.grant); + assert.strictEqual(report.node_status, "completed"); + assert.strictEqual(report.virtual_thread, "windows-command-dev"); + assert.strictEqual(report.status_code, 0); + assert.strictEqual(report.large_bytes_uploaded, false); + assert.strictEqual( + report.staged_artifact.path, + "/vfs/artifacts/windows-runner-output.txt" + ); + assert.strictEqual(report.registration_response.type, "node_enrollment_exchanged"); + assert.strictEqual(report.registration_response.credential.scope, "node:attach"); + assert.strictEqual(report.coordinator_response.type, "task_recorded"); + + const recorded = await send(addr, signedNodeRequest(forgejoWindowsNode, forgejoWindowsIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: forgejoWindowsNode, + capabilities: { + os: "Windows", + arch: "x86_64", + capabilities: ["Command", "WindowsCommandDev", "VfsArtifacts"], + environment_backends: ["WindowsCommandDev"], + source_providers: ["filesystem"] + }, + cached_environment_digests: [windowsEnvironmentDigest], + source_snapshots: [sourceSnapshot], + artifact_locations: ["windows-runner-output.txt"], + direct_connectivity: true, + online: true + })); + assert.strictEqual(recorded.type, "node_capabilities_recorded"); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"] + }, + environment_digest: windowsEnvironmentDigest, + required_capabilities: ["Command"], + source_snapshot: sourceSnapshot, + required_artifacts: ["windows-runner-output.txt"], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "forgejo-windows-node"); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact: "windows-runner-output.txt", + max_bytes: 1024 + }); + assert.strictEqual(link.type, "artifact_download_link"); + assert.deepStrictEqual(link.link.source, { + RetainedNode: "forgejo-windows-node" + }); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + await assertDebuggerShowsWindowsThread(); + + const outDir = path.join(repo, "target", "acceptance"); + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync( + path.join(outDir, "windows-runner.json"), + `${JSON.stringify( + { + kind: "clusterflux_windows_runner_validation", + platform: process.platform, + runner: process.env.FORGEJO_RUNNER_NAME || process.env.RUNNER_NAME || null, + validated: true + }, + null, + 2 + )}\n` + ); + + console.log("Windows runner smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/tests/fixtures/runtime-conformance/Cargo.toml b/tests/fixtures/runtime-conformance/Cargo.toml new file mode 100644 index 0000000..8558f01 --- /dev/null +++ b/tests/fixtures/runtime-conformance/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "runtime-conformance" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[lib] +crate-type = ["rlib", "cdylib"] + +[dependencies] +clusterflux = { package = "clusterflux-sdk", path = "../../../crates/clusterflux-sdk" } +futures-executor.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/tests/fixtures/runtime-conformance/README.md b/tests/fixtures/runtime-conformance/README.md new file mode 100644 index 0000000..15eed25 --- /dev/null +++ b/tests/fixtures/runtime-conformance/README.md @@ -0,0 +1,5 @@ +# Runtime conformance fixture + +This is test-only coverage for raw task ABI, cancellation, long joins, placement, +debug probes, and failure injection. It is intentionally not a public SDK +example. Start with examples/hello-build. diff --git a/tests/fixtures/runtime-conformance/envs/linux/Containerfile b/tests/fixtures/runtime-conformance/envs/linux/Containerfile new file mode 100644 index 0000000..ba87ba3 --- /dev/null +++ b/tests/fixtures/runtime-conformance/envs/linux/Containerfile @@ -0,0 +1,3 @@ +FROM docker.io/library/alpine:3.20 +RUN apk add --no-cache build-base tar zstd +WORKDIR /workspace diff --git a/tests/fixtures/runtime-conformance/envs/windows/Dockerfile b/tests/fixtures/runtime-conformance/envs/windows/Dockerfile new file mode 100644 index 0000000..2978ba6 --- /dev/null +++ b/tests/fixtures/runtime-conformance/envs/windows/Dockerfile @@ -0,0 +1,2 @@ +# User-attached Windows development execution contract. +# This is not a managed untrusted Windows sandbox. diff --git a/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c b/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c new file mode 100644 index 0000000..79b67fd --- /dev/null +++ b/tests/fixtures/runtime-conformance/fixture/hello-clusterflux.c @@ -0,0 +1,6 @@ +#include + +int main(void) { + puts("hello from a real Clusterflux build"); + return 0; +} diff --git a/tests/fixtures/runtime-conformance/src/lib.rs b/tests/fixtures/runtime-conformance/src/lib.rs new file mode 100644 index 0000000..91cf21f --- /dev/null +++ b/tests/fixtures/runtime-conformance/src/lib.rs @@ -0,0 +1,501 @@ +use clusterflux::{Artifact, EnvRef, SourceSnapshot}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] +pub struct BuildReport { + pub linux_thread: u64, + pub linux_parallel_thread: u64, + pub package_thread: u64, + pub linux_artifact: Artifact, + pub package_artifact: Artifact, + pub source: SourceSnapshot, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] +pub struct PackageInput { + pub release_name: String, + pub source: SourceSnapshot, + pub executable: Option, + pub inputs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] +pub struct IdentityProbeInput { + pub source: SourceSnapshot, + pub label: String, + pub delay_seconds: u64, +} + +pub fn linux_env() -> EnvRef { + clusterflux::env!("linux") +} + +pub fn windows_env() -> EnvRef { + clusterflux::env!("windows") +} + +#[clusterflux::task(capabilities = "source_filesystem")] +pub async fn prepare_source() -> SourceSnapshot { + #[cfg(target_arch = "wasm32")] + { + return clusterflux::source::snapshot() + .await + .expect("the source task should snapshot the node checkout it was placed on"); + } + #[cfg(not(target_arch = "wasm32"))] + SourceSnapshot { + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + } +} + +#[clusterflux::task(capabilities = "command")] +pub async fn compile_linux(source: SourceSnapshot) -> Artifact { + let _ = &source; + #[cfg(target_arch = "wasm32")] + { + let executable = clusterflux::fs::output_path("hello-clusterflux") + .expect("the executable output path should be task-local"); + let output = clusterflux::command::Command::new("cc") + .args([ + "-Os", + "-static", + "-s", + "fixture/hello-clusterflux.c", + "-o", + executable.as_str(), + ]) + .current_dir("/workspace") + .env("SOURCE_DATE_EPOCH", "0") + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the declared Linux environment should provide a real C compiler"); + assert_eq!(output.status_code, Some(0)); + return clusterflux::fs::flush(&executable) + .await + .expect("the command-created executable should flush to node artifact storage"); + } + #[cfg(not(target_arch = "wasm32"))] + Artifact { + id: "hello-clusterflux-native-test".to_owned(), + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + size_bytes: 0, + } +} + +#[clusterflux::task] +#[unsafe(no_mangle)] +pub extern "C" fn task_add_one(input: i32) -> i32 { + input + 1 +} + +#[clusterflux::task] +#[unsafe(no_mangle)] +pub extern "C" fn task_trap(_input: i32) -> i32 { + #[cfg(target_arch = "wasm32")] + core::arch::wasm32::unreachable(); + #[cfg(not(target_arch = "wasm32"))] + panic!("intentional task trap") +} + +#[clusterflux::task(capabilities = "command")] +pub async fn package_release(input: PackageInput) -> Artifact { + #[cfg(target_arch = "wasm32")] + { + assert_eq!(input.release_name, "release.tar"); + let executable = input + .executable + .as_ref() + .expect("the package input should contain the compiler artifact"); + assert!(input.inputs.iter().any(|artifact| artifact == executable)); + let executable = clusterflux::fs::materialize(&executable, "package/hello-clusterflux") + .await + .expect("the retaining node should materialize the compiler artifact locally"); + let release = clusterflux::fs::output_path("release.tar") + .expect("the release output path should be task-local"); + let output = clusterflux::command::Command::new("tar") + .args([ + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "--mode=0755", + "-cf", + release.as_str(), + "-C", + "/clusterflux/output/package", + "hello-clusterflux", + ]) + .current_dir("/workspace") + .env("SOURCE_DATE_EPOCH", "0") + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the declared Linux environment should package the materialized executable"); + assert_eq!(output.status_code, Some(0)); + assert_eq!( + executable.as_str(), + "/clusterflux/output/package/hello-clusterflux" + ); + return clusterflux::fs::flush(&release) + .await + .expect("the command-created release archive should flush as the final artifact"); + } + #[cfg(not(target_arch = "wasm32"))] + { + let _ = input; + Artifact { + id: "release-native-test".to_owned(), + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + size_bytes: 0, + } + } +} + +#[clusterflux::task(capabilities = "command")] +pub async fn abort_probe(source: SourceSnapshot) -> i32 { + let _ = &source; + #[cfg(target_arch = "wasm32")] + { + let output = clusterflux::command::Command::new("sh") + .args(["-c", "sleep 90"]) + .timeout(std::time::Duration::from_secs(120)) + .network_disabled() + .output() + .await + .expect("abort probe command should either finish or be stopped by process abort"); + return output.status_code.unwrap_or(-1); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +#[clusterflux::task(capabilities = "command")] +pub async fn long_join_probe(source: SourceSnapshot) -> i32 { + let _ = &source; + #[cfg(target_arch = "wasm32")] + { + let output = clusterflux::command::Command::new("sh") + .args(["-c", "sleep 125"]) + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the controlled long-running command should complete normally"); + return output.status_code.unwrap_or(-1); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +#[clusterflux::task(capabilities = "command")] +pub async fn identity_probe(input: IdentityProbeInput) -> String { + let _ = &input.source; + #[cfg(target_arch = "wasm32")] + { + let script = format!( + "sleep {}; printf '%s' '{}'", + input.delay_seconds, input.label + ); + let output = clusterflux::command::Command::new("sh") + .args(["-c", script.as_str()]) + .timeout(std::time::Duration::from_secs(60)) + .network_disabled() + .output() + .await + .expect("identity probe command should complete"); + assert_eq!(output.status_code, Some(0)); + return input.label; + } + #[cfg(not(target_arch = "wasm32"))] + input.label +} + +#[clusterflux::task] +pub fn cooperative_cancellation_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + loop { + if clusterflux::process::cancellation_requested() + .expect("task control should remain available while the Wasm task is running") + { + // Cancellation is a request observed and handled by task code. Returning + // normally is intentionally distinct from the runtime's forced abort path. + return 17; + } + } + #[cfg(not(target_arch = "wasm32"))] + 17 +} + +#[clusterflux::task] +pub fn debug_child_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + loop { + if clusterflux::process::cancellation_requested() + .expect("child debug participant should retain task control") + { + return 23; + } + } + #[cfg(not(target_arch = "wasm32"))] + 23 +} + +#[clusterflux::task] +pub async fn debug_parent_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let child = clusterflux::spawn::task(debug_child_probe) + .name("debug child participant") + .start() + .await + .expect("parent debug participant should spawn its child"); + return child + .join() + .await + .expect("parent debug participant should join its child"); + } + #[cfg(not(target_arch = "wasm32"))] + 23 +} + +#[clusterflux::main] +pub async fn build_main() -> BuildReport { + run_build_workflow().await +} + +#[clusterflux::main(name = "fail")] +pub async fn fail_main() -> Result { + #[cfg(target_arch = "wasm32")] + { + let child = clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) + .task_id("task_trap") + .name("intentional failing child") + .start() + .await + .map_err(|error| format!("failing entrypoint could not launch its child: {error}"))?; + return child + .join() + .await + .map_err(|error| format!("intentional child failure: {error}")); + } + #[cfg(not(target_arch = "wasm32"))] + Err("intentional child failure".to_owned()) +} + +#[clusterflux::main(name = "restart")] +pub async fn restart_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + return clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) + .task_id("task_trap") + .name("edited restart probe") + .failure_policy(clusterflux::spawn::TaskFailurePolicy::AwaitOperator) + .start() + .await + .expect("restart probe should launch its child") + .join() + .await + .expect("restart probe child should complete"); + } + #[cfg(not(target_arch = "wasm32"))] + task_add_one(41) +} + +#[clusterflux::main(name = "park-wake")] +pub async fn park_wake_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let mut value = 0_i32; + for _ in 0..16 { + let next = clusterflux::spawn::task_with_arg(value, |input| task_add_one(input)) + .task_id("task_add_one") + .name("park/wake fuel refill probe") + .start() + .await + .expect("park/wake probe should launch") + .join() + .await + .expect("park/wake probe should complete"); + value = i32::try_from(next).expect("park/wake result should remain in i32 range"); + } + return value; + } + #[cfg(not(target_arch = "wasm32"))] + 16 +} + +#[clusterflux::main(name = "long-join")] +pub async fn long_join_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let source = clusterflux::spawn::async_task(prepare_source) + .name("prepare source for controlled long-running task") + .start() + .await + .expect("long join source preparation should launch") + .join() + .await + .expect("long join source preparation should complete"); + return clusterflux::spawn::async_task_with_arg(source, long_join_probe) + .name("controlled task longer than two minutes") + .env(linux_env()) + .start() + .await + .expect("long join probe should launch") + .join() + .await + .expect("long join probe should remain joinable beyond two minutes"); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +#[clusterflux::main(name = "identity")] +pub async fn identity_main() -> Vec { + #[cfg(target_arch = "wasm32")] + { + let source = clusterflux::spawn::async_task(prepare_source) + .name("prepare source for identity probes") + .start() + .await + .expect("identity source preparation should launch") + .join() + .await + .expect("identity source preparation should complete"); + let slow = clusterflux::spawn::async_task_with_arg( + IdentityProbeInput { + source: source.clone(), + label: "slow-first".to_owned(), + delay_seconds: 30, + }, + identity_probe, + ) + .name("identity probe slow") + .env(linux_env()) + .start() + .await + .expect("slow identity probe should launch"); + let fast = clusterflux::spawn::async_task_with_arg( + IdentityProbeInput { + source, + label: "fast-second".to_owned(), + delay_seconds: 20, + }, + identity_probe, + ) + .name("identity probe fast") + .env(linux_env()) + .start() + .await + .expect("fast identity probe should launch"); + let fast_result = fast + .join() + .await + .expect("fast identity probe should complete first"); + let slow_result = slow + .join() + .await + .expect("slow identity probe should remain independently joinable"); + return vec![fast_result, slow_result]; + } + #[cfg(not(target_arch = "wasm32"))] + vec!["fast-second".to_owned(), "slow-first".to_owned()] +} + +pub async fn run_build_workflow() -> BuildReport { + let source = clusterflux::spawn::async_task(prepare_source) + .name("prepare source snapshot") + .start() + .await + .unwrap() + .join() + .await + .unwrap(); + let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) + .name("compile linux") + .env(linux_env()) + .start() + .await + .unwrap(); + let linux_parallel = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) + .name("compile linux in parallel") + .env(linux_env()) + .start() + .await + .unwrap(); + let linux_thread = linux.virtual_thread_id(); + let linux_parallel_thread = linux_parallel.virtual_thread_id(); + let linux_artifact = linux.join().await.unwrap(); + let linux_parallel_artifact = linux_parallel.join().await.unwrap(); + assert_eq!(linux_artifact.digest, linux_parallel_artifact.digest); + let package = clusterflux::spawn::async_task_with_arg( + PackageInput { + release_name: "release.tar".to_owned(), + source: source.clone(), + executable: Some(linux_artifact.clone()), + inputs: vec![linux_artifact.clone()], + }, + package_release, + ) + .name("package artifacts") + .env(linux_env()) + .start() + .await + .unwrap(); + + let package_thread = package.virtual_thread_id(); + let package_artifact = package.join().await.unwrap(); + + BuildReport { + linux_thread, + linux_parallel_thread, + package_thread, + linux_artifact, + package_artifact, + source, + } +} + +#[cfg(test)] +mod tests { + use futures_executor::block_on; + + use super::*; + + #[test] + fn flagship_workflow_is_rust_source_with_spawned_virtual_tasks() { + let main_report = block_on(build_main()); + assert_eq!( + main_report.linux_artifact.id, + "hello-clusterflux-native-test" + ); + assert_eq!(task_add_one(41), 42); + assert_eq!(block_on(restart_main()), 42); + assert_eq!(block_on(park_wake_main()), 16); + assert_eq!(block_on(long_join_main()), 0); + assert_eq!( + block_on(identity_main()), + vec!["fast-second".to_owned(), "slow-first".to_owned()] + ); + assert_eq!(linux_env().name, "linux"); + assert_eq!(windows_env().name, "windows"); + + let report = block_on(run_build_workflow()); + + assert_ne!(report.linux_thread, report.package_thread); + assert_ne!(report.linux_thread, report.linux_parallel_thread); + assert_eq!(report.linux_artifact.id, "hello-clusterflux-native-test"); + assert_eq!(report.package_artifact.id, "release-native-test"); + assert_eq!( + report.source.digest, + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } +} diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json deleted file mode 100644 index d6aebd7..0000000 --- a/vscode-extension/package-lock.json +++ /dev/null @@ -1,3810 +0,0 @@ -{ - "name": "clusterflux-vscode", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "clusterflux-vscode", - "version": "0.1.0", - "devDependencies": { - "@vscode/vsce": "3.9.2" - }, - "engines": { - "vscode": "^1.80.0" - } - }, - "node_modules/@azu/format-text": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz", - "integrity": "sha512-Swi4N7Edy1Eqq82GxgEECXSSLyn6GOb5htRFPzBDdUkECGXtlf12ynO5oJSpWKPwCaUssOu7NfhDcCWpIC6Ywg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@azu/style-format": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@azu/style-format/-/style-format-1.0.1.tgz", - "integrity": "sha512-AHcTojlNBdD/3/KxIKlg8sxIWHfOtQszLvOpagLTO+bjC3u7SAszu1lf//u7JJC50aUSH+BVWDD/KvaA6Gfn5g==", - "dev": true, - "license": "WTFPL", - "dependencies": { - "@azu/format-text": "^1.0.1" - } - }, - "node_modules/@azure/abort-controller": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", - "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", - "dev": true, - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-auth": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", - "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-util": "^1.13.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-client": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.0.tgz", - "integrity": "sha512-JjQWO6akOck45PH/XBrxzsQGAiKrfFl4m5iggJ0ItMIz5omRufOXWpqCPpdjKN3vKDzlSUvFjaMb7Zwf0gvAdA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-rest-pipeline": "^1.22.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.25.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", - "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "@typespec/ts-http-runtime": "^0.3.4", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-tracing": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", - "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-util": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", - "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/identity": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.1.tgz", - "integrity": "sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.9.0", - "@azure/core-client": "^1.9.2", - "@azure/core-rest-pipeline": "^1.17.0", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.0.0", - "@azure/msal-browser": "^5.5.0", - "@azure/msal-node": "^5.1.0", - "open": "^10.1.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/logger": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", - "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/msal-browser": { - "version": "5.17.1", - "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-5.17.1.tgz", - "integrity": "sha512-zBhRGzABKSI7hfWh5EaZmril5ybZ7imBN1qEZl5sDTaelr+l8SnPjZO50Q4dnKnm347YPIlBMSnXKZyh3Yu5DQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/msal-common": "16.11.2" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-common": { - "version": "16.11.2", - "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.11.2.tgz", - "integrity": "sha512-yDhtBOGDCdK9ipQ9g3+wmlMEPnZx2pXaDicDd9jYyR1L+7lEbvEohTDmF5qejZDutZY3m9pWPxeYxzNC701A2w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-node": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-5.4.1.tgz", - "integrity": "sha512-yqgoyOIMCH7TNaSLMBTP+4LUlbMMf1zgC8nzOFG95lmW82CmsAEtUT0J93e4BdqDcnX5qle/9X+yb7A8Mw9M0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/msal-common": "16.11.2", - "jsonwebtoken": "^9.0.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", - "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.29.7", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", - "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@secretlint/config-creator": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/config-creator/-/config-creator-10.2.2.tgz", - "integrity": "sha512-BynOBe7Hn3LJjb3CqCHZjeNB09s/vgf0baBaHVw67w7gHF0d25c3ZsZ5+vv8TgwSchRdUCRrbbcq5i2B1fJ2QQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/config-loader": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/config-loader/-/config-loader-10.2.2.tgz", - "integrity": "sha512-ndjjQNgLg4DIcMJp4iaRD6xb9ijWQZVbd9694Ol2IszBIbGPPkwZHzJYKICbTBmh6AH/pLr0CiCaWdGJU7RbpQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/profiler": "^10.2.2", - "@secretlint/resolver": "^10.2.2", - "@secretlint/types": "^10.2.2", - "ajv": "^8.17.1", - "debug": "^4.4.1", - "rc-config-loader": "^4.1.3" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/core": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/core/-/core-10.2.2.tgz", - "integrity": "sha512-6rdwBwLP9+TO3rRjMVW1tX+lQeo5gBbxl1I5F8nh8bgGtKwdlCMhMKsBWzWg1ostxx/tIG7OjZI0/BxsP8bUgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/profiler": "^10.2.2", - "@secretlint/types": "^10.2.2", - "debug": "^4.4.1", - "structured-source": "^4.0.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/formatter": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/formatter/-/formatter-10.2.2.tgz", - "integrity": "sha512-10f/eKV+8YdGKNQmoDUD1QnYL7TzhI2kzyx95vsJKbEa8akzLAR5ZrWIZ3LbcMmBLzxlSQMMccRmi05yDQ5YDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/resolver": "^10.2.2", - "@secretlint/types": "^10.2.2", - "@textlint/linter-formatter": "^15.2.0", - "@textlint/module-interop": "^15.2.0", - "@textlint/types": "^15.2.0", - "chalk": "^5.4.1", - "debug": "^4.4.1", - "pluralize": "^8.0.0", - "strip-ansi": "^7.1.0", - "table": "^6.9.0", - "terminal-link": "^4.0.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/formatter/node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/@secretlint/node": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/node/-/node-10.2.2.tgz", - "integrity": "sha512-eZGJQgcg/3WRBwX1bRnss7RmHHK/YlP/l7zOQsrjexYt6l+JJa5YhUmHbuGXS94yW0++3YkEJp0kQGYhiw1DMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/config-loader": "^10.2.2", - "@secretlint/core": "^10.2.2", - "@secretlint/formatter": "^10.2.2", - "@secretlint/profiler": "^10.2.2", - "@secretlint/source-creator": "^10.2.2", - "@secretlint/types": "^10.2.2", - "debug": "^4.4.1", - "p-map": "^7.0.3" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/profiler": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/profiler/-/profiler-10.2.2.tgz", - "integrity": "sha512-qm9rWfkh/o8OvzMIfY8a5bCmgIniSpltbVlUVl983zDG1bUuQNd1/5lUEeWx5o/WJ99bXxS7yNI4/KIXfHexig==", - "dev": true, - "license": "MIT" - }, - "node_modules/@secretlint/resolver": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/resolver/-/resolver-10.2.2.tgz", - "integrity": "sha512-3md0cp12e+Ae5V+crPQYGd6aaO7ahw95s28OlULGyclyyUtf861UoRGS2prnUrKh7MZb23kdDOyGCYb9br5e4w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@secretlint/secretlint-formatter-sarif": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-formatter-sarif/-/secretlint-formatter-sarif-10.2.2.tgz", - "integrity": "sha512-ojiF9TGRKJJw308DnYBucHxkpNovDNu1XvPh7IfUp0A12gzTtxuWDqdpuVezL7/IP8Ua7mp5/VkDMN9OLp1doQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "node-sarif-builder": "^3.2.0" - } - }, - "node_modules/@secretlint/secretlint-rule-no-dotenv": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-no-dotenv/-/secretlint-rule-no-dotenv-10.2.2.tgz", - "integrity": "sha512-KJRbIShA9DVc5Va3yArtJ6QDzGjg3PRa1uYp9As4RsyKtKSSZjI64jVca57FZ8gbuk4em0/0Jq+uy6485wxIdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/secretlint-rule-preset-recommend": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-preset-recommend/-/secretlint-rule-preset-recommend-10.2.2.tgz", - "integrity": "sha512-K3jPqjva8bQndDKJqctnGfwuAxU2n9XNCPtbXVI5JvC7FnQiNg/yWlQPbMUlBXtBoBGFYp08A94m6fvtc9v+zA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/source-creator": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/source-creator/-/source-creator-10.2.2.tgz", - "integrity": "sha512-h6I87xJfwfUTgQ7irWq7UTdq/Bm1RuQ/fYhA3dtTIAop5BwSFmZyrchph4WcoEvbN460BWKmk4RYSvPElIIvxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2", - "istextorbinary": "^9.5.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/types": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/types/-/types-10.2.2.tgz", - "integrity": "sha512-Nqc90v4lWCXyakD6xNyNACBJNJ0tNCwj2WNk/7ivyacYHxiITVgmLUFXTBOeCdy79iz6HtN9Y31uw/jbLrdOAg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@sindresorhus/merge-streams": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", - "integrity": "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@textlint/ast-node-types": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-15.7.1.tgz", - "integrity": "sha512-Wii5UgUKFEh9Uv6wbq1zr4/Kf+dtjiUuzPrrXzKp8H+ifkvKNzi23V4Nz+6wVyHQn5T28AFuc8VH8OtzvGYecA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/linter-formatter": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/linter-formatter/-/linter-formatter-15.7.1.tgz", - "integrity": "sha512-TdwZ/debWYFD05K3CcoHtwvnCrza29wZxD+BjDTk/V5N7iRqkK1dTTHSD4A8AIgROLiDkHJmIKQbasbmsg8AvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azu/format-text": "^1.0.2", - "@azu/style-format": "^1.0.1", - "@textlint/module-interop": "15.7.1", - "@textlint/resolver": "15.7.1", - "@textlint/types": "15.7.1", - "chalk": "^4.1.2", - "debug": "^4.4.3", - "js-yaml": "^4.1.1", - "lodash": "^4.18.1", - "pluralize": "^2.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1", - "table": "^6.9.0", - "text-table": "^0.2.0" - } - }, - "node_modules/@textlint/linter-formatter/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@textlint/linter-formatter/node_modules/pluralize": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-2.0.0.tgz", - "integrity": "sha512-TqNZzQCD4S42De9IfnnBvILN7HAW7riLqsCyp8lgjXeysyPlX5HhqKAcJHHHb9XskE4/a+7VGC9zzx8Ls0jOAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/linter-formatter/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@textlint/module-interop": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/module-interop/-/module-interop-15.7.1.tgz", - "integrity": "sha512-Jg+sQW2L/cRJypk59wtcMUVVpt8vmit5ZMT3gUnFwevP3A6Qp1HfOtUy9ObT4hBX3lOSGT/ekcCDxR1pL7uH1g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/resolver": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/resolver/-/resolver-15.7.1.tgz", - "integrity": "sha512-8XnO0pgF6mXnm41VvWmBbEIdGPhiCUt31uLZkOis1ECeg/1SoUcIT6Mx/F0e1rukq8l0UlOSeY9a31CsvRMK0g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/types": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/types/-/types-15.7.1.tgz", - "integrity": "sha512-Vye/GmFNBTgVzZFtIFJTmLB+s2A7oIADxNG6r9UhfPuY+Czv0z5G3xeyFZZudPlfxURsKUyPIU5XsjOFqVp33A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@textlint/ast-node-types": "15.7.1" - } - }, - "node_modules/@types/normalize-package-data": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", - "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/sarif": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@types/sarif/-/sarif-2.1.7.tgz", - "integrity": "sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@typespec/ts-http-runtime": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.7.tgz", - "integrity": "sha512-JVUD8X2tfDMWjcjLs4yVxxVrS8yR5vnh386GAXT9Qj79nBxxXSaHFQZg5FweLmT8HlPQ3kii6noUB+Z9RN7DvQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@vscode/vsce": { - "version": "3.9.2", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.9.2.tgz", - "integrity": "sha512-XSxMosEEDO6vLxELAHVkwmhC0qe0ijZni2jB9Rcs8kQsW4lhTDQ/wMzmwFs/buotAWSnpmUp/dRWD2ufG3UYKA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/identity": "^4.1.0", - "@secretlint/node": "^10.1.2", - "@secretlint/secretlint-formatter-sarif": "^10.1.2", - "@secretlint/secretlint-rule-no-dotenv": "^10.1.2", - "@secretlint/secretlint-rule-preset-recommend": "^10.1.2", - "@vscode/vsce-sign": "^2.0.0", - "azure-devops-node-api": "^12.5.0", - "chalk": "^4.1.2", - "cheerio": "^1.0.0-rc.9", - "cockatiel": "^3.1.2", - "commander": "^12.1.0", - "form-data": "^4.0.0", - "glob": "^13.0.6", - "hosted-git-info": "^4.0.2", - "jsonc-parser": "^3.2.0", - "leven": "^3.1.0", - "markdown-it": "^14.1.0", - "mime": "^1.3.4", - "minimatch": "^10.2.2", - "parse-semver": "^1.1.1", - "read": "^1.0.7", - "secretlint": "^10.1.2", - "semver": "^7.5.2", - "tmp": "^0.2.3", - "typed-rest-client": "^1.8.4", - "url-join": "^4.0.1", - "xml2js": "^0.5.0", - "yauzl": "^3.2.1", - "yazl": "^2.2.2" - }, - "bin": { - "vsce": "vsce" - }, - "engines": { - "node": ">= 20" - }, - "optionalDependencies": { - "keytar": "^7.7.0" - } - }, - "node_modules/@vscode/vsce-sign": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign/-/vsce-sign-2.0.9.tgz", - "integrity": "sha512-8IvaRvtFyzUnGGl3f5+1Cnor3LqaUWvhaUjAYO8Y39OUYlOf3cRd+dowuQYLpZcP3uwSG+mURwjEBOSq4SOJ0g==", - "dev": true, - "hasInstallScript": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optionalDependencies": { - "@vscode/vsce-sign-alpine-arm64": "2.0.6", - "@vscode/vsce-sign-alpine-x64": "2.0.6", - "@vscode/vsce-sign-darwin-arm64": "2.0.6", - "@vscode/vsce-sign-darwin-x64": "2.0.6", - "@vscode/vsce-sign-linux-arm": "2.0.6", - "@vscode/vsce-sign-linux-arm64": "2.0.6", - "@vscode/vsce-sign-linux-x64": "2.0.6", - "@vscode/vsce-sign-win32-arm64": "2.0.6", - "@vscode/vsce-sign-win32-x64": "2.0.6" - } - }, - "node_modules/@vscode/vsce-sign-alpine-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-arm64/-/vsce-sign-alpine-arm64-2.0.6.tgz", - "integrity": "sha512-wKkJBsvKF+f0GfsUuGT0tSW0kZL87QggEiqNqK6/8hvqsXvpx8OsTEc3mnE1kejkh5r+qUyQ7PtF8jZYN0mo8Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "alpine" - ] - }, - "node_modules/@vscode/vsce-sign-alpine-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-x64/-/vsce-sign-alpine-x64-2.0.6.tgz", - "integrity": "sha512-YoAGlmdK39vKi9jA18i4ufBbd95OqGJxRvF3n6ZbCyziwy3O+JgOpIUPxv5tjeO6gQfx29qBivQ8ZZTUF2Ba0w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "alpine" - ] - }, - "node_modules/@vscode/vsce-sign-darwin-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-arm64/-/vsce-sign-darwin-arm64-2.0.6.tgz", - "integrity": "sha512-5HMHaJRIQuozm/XQIiJiA0W9uhdblwwl2ZNDSSAeXGO9YhB9MH5C4KIHOmvyjUnKy4UCuiP43VKpIxW1VWP4tQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@vscode/vsce-sign-darwin-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-x64/-/vsce-sign-darwin-x64-2.0.6.tgz", - "integrity": "sha512-25GsUbTAiNfHSuRItoQafXOIpxlYj+IXb4/qarrXu7kmbH94jlm5sdWSCKrrREs8+GsXF1b+l3OB7VJy5jsykw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@vscode/vsce-sign-linux-arm": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm/-/vsce-sign-linux-arm-2.0.6.tgz", - "integrity": "sha512-UndEc2Xlq4HsuMPnwu7420uqceXjs4yb5W8E2/UkaHBB9OWCwMd3/bRe/1eLe3D8kPpxzcaeTyXiK3RdzS/1CA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-linux-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm64/-/vsce-sign-linux-arm64-2.0.6.tgz", - "integrity": "sha512-cfb1qK7lygtMa4NUl2582nP7aliLYuDEVpAbXJMkDq1qE+olIw/es+C8j1LJwvcRq1I2yWGtSn3EkDp9Dq5FdA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-linux-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-x64/-/vsce-sign-linux-x64-2.0.6.tgz", - "integrity": "sha512-/olerl1A4sOqdP+hjvJ1sbQjKN07Y3DVnxO4gnbn/ahtQvFrdhUi0G1VsZXDNjfqmXw57DmPi5ASnj/8PGZhAA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-win32-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-arm64/-/vsce-sign-win32-arm64-2.0.6.tgz", - "integrity": "sha512-ivM/MiGIY0PJNZBoGtlRBM/xDpwbdlCWomUWuLmIxbi1Cxe/1nooYrEQoaHD8ojVRgzdQEUzMsRbyF5cJJgYOg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@vscode/vsce-sign-win32-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-x64/-/vsce-sign-win32-x64-2.0.6.tgz", - "integrity": "sha512-mgth9Kvze+u8CruYMmhHw6Zgy3GRX2S+Ed5oSokDEK5vPEwGGKnmuXua9tmFhomeAnhgJnL4DCna3TiNuGrBTQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-escapes": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", - "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", - "dev": true, - "license": "MIT", - "dependencies": { - "environment": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/azure-devops-node-api": { - "version": "12.5.0", - "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-12.5.0.tgz", - "integrity": "sha512-R5eFskGvOm3U/GzeAuxRkUsAl0hrAwGgWn6zAd2KrZmrEhWZVqLew4OOupbQlXUuojUzpGtq62SmdhJ06N88og==", - "dev": true, - "license": "MIT", - "dependencies": { - "tunnel": "0.0.6", - "typed-rest-client": "^1.8.4" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/binaryextensions": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-6.11.0.tgz", - "integrity": "sha512-sXnYK/Ij80TO3lcqZVV2YgfKN5QjUWIRk/XSm2J/4bd/lPko3lvk0O4ZppH6m+6hB2/GTu+ptNwVFe1xh+QLQw==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "editions": "^6.21.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true, - "license": "ISC" - }, - "node_modules/boundary": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz", - "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/bundle-name": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", - "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "run-applescript": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, - "node_modules/cheerio-select": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", - "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-select": "^5.1.0", - "css-what": "^6.1.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/cockatiel": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/cockatiel/-/cockatiel-3.2.1.tgz", - "integrity": "sha512-gfrHV6ZPkquExvMh9IOkKsBzNDk6sDuZ6DdBGUBkvFnTCqCxzpuq48RySgP0AnaqQkw2zynOFj9yly6T1Q2G5Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/css-select": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", - "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-what": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", - "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/default-browser": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", - "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "bundle-name": "^4.1.0", - "default-browser-id": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", - "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", - "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "BSD-2-Clause" - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", - "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/editions": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/editions/-/editions-6.22.0.tgz", - "integrity": "sha512-UgGlf8IW75je7HZjNDpJdCv4cGJWIi6yumFdZ0R7A8/CIhQiWUjyGLCxdHpd8bmyD1gnkfUNK0oeOXqUS2cpfQ==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "version-range": "^4.15.0" - }, - "engines": { - "ecmascript": ">= es5", - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/encoding-sniffer": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", - "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", - "dev": true, - "license": "MIT", - "dependencies": { - "iconv-lite": "^0.6.3", - "whatwg-encoding": "^3.1.1" - }, - "funding": { - "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/environment": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", - "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", - "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "dev": true, - "license": "(MIT OR WTFPL)", - "optional": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-glob": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.8" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/fastq": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", - "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, - "license": "ISC", - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/fs-extra": { - "version": "11.3.6", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.6.tgz", - "integrity": "sha512-w8ZNZr2mKIc7qeNaQ9AVPT1+iFaI+Avd4xudVOvdDJ8VytREi1Ft5Ih7hd9jjehod8vAM5GMsfQ/TpPf4EyoEA==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-14.1.0.tgz", - "integrity": "sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sindresorhus/merge-streams": "^2.1.0", - "fast-glob": "^3.3.3", - "ignore": "^7.0.3", - "path-type": "^6.0.0", - "slash": "^5.1.0", - "unicorn-magic": "^0.3.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hosted-git-info": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", - "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, - "node_modules/htmlparser2/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/ignore": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", - "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/index-to-position": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", - "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/is-docker": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", - "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "dev": true, - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", - "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-wsl": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", - "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-inside-container": "^1.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/istextorbinary": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-9.5.0.tgz", - "integrity": "sha512-5mbUj3SiZXCuRf9fT3ibzbSSEWiy63gFfksmGfdOzujPjW3k+z8WvIBxcJHBoQNlaZaiyB25deviif2+osLmLw==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "binaryextensions": "^6.11.0", - "editions": "^6.21.0", - "textextensions": "^6.11.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsonc-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsonfile": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", - "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/jsonwebtoken": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", - "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", - "dev": true, - "license": "MIT", - "dependencies": { - "jws": "^4.0.1", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=12", - "npm": ">=6" - } - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", - "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jwa": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/keytar": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/keytar/-/keytar-7.9.0.tgz", - "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-addon-api": "^4.3.0", - "prebuild-install": "^7.0.1" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/linkify-it": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz", - "integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "uc.micro": "^2.0.0" - } - }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.truncate": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", - "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/markdown-it": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.3.0.tgz", - "integrity": "sha512-RCEsPjR+sr0x+AuYp601tKTkgFG4YEPLCzHST3cQ/fhlJkqAkz1L2/Qbp1j9qw5SBwQHFBoW8+hoN5xssOF0Tw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.5.0", - "linkify-it": "^5.0.2", - "mdurl": "^2.0.0", - "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" - }, - "bin": { - "markdown-it": "bin/markdown-it.mjs" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "dev": true, - "license": "MIT" - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true, - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "license": "MIT", - "optional": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "dev": true, - "license": "ISC" - }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/node-abi": { - "version": "3.94.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", - "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-addon-api": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", - "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/node-sarif-builder": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/node-sarif-builder/-/node-sarif-builder-3.4.0.tgz", - "integrity": "sha512-tGnJW6OKRii9u/b2WiUViTJS+h7Apxx17qsMUjsUeNDiMMX5ZFf8F8Fcz7PAQ6omvOxHZtvDTmOYKJQwmfpjeg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/sarif": "^2.1.7", - "fs-extra": "^11.1.1" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/normalize-package-data": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz", - "integrity": "sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^7.0.0", - "semver": "^7.3.5", - "validate-npm-package-license": "^3.0.4" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/normalize-package-data/node_modules/hosted-git-info": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", - "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^10.0.1" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/normalize-package-data/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "optional": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", - "dev": true, - "license": "MIT", - "dependencies": { - "default-browser": "^5.2.1", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-map": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.5.tgz", - "integrity": "sha512-e8vJF4XdVkzqqSHguEMz41mQO1wKwxKm5ENrUJQUu9kLDCtn83cxbyHZcszr4QC5zEA7WffRRC4gsTecC7J9oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parse-json": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", - "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.26.2", - "index-to-position": "^1.1.0", - "type-fest": "^4.39.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parse-semver": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/parse-semver/-/parse-semver-1.1.1.tgz", - "integrity": "sha512-Eg1OuNntBMH0ojvEKSrvDSnwLmvVuUOSdylH/pSCPNMIspLlweJyIWXCE+k/5hm3cj/EBUYwmWkjhBALNP4LXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^5.1.0" - } - }, - "node_modules/parse-semver/node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/parse5": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", - "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "entities": "^6.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", - "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "domhandler": "^5.0.3", - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-parser-stream": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", - "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", - "dev": true, - "license": "MIT", - "dependencies": { - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "11.5.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", - "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/path-type": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-6.0.0.tgz", - "integrity": "sha512-Vj7sf++t5pBD637NSfkxpHSMfWaeig5+DKWLhcqIYx6mWQz5hdJTGDVMQiJcw1ZYkhs7AazKDGpRVji1LJCZUQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pluralize": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", - "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pump": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", - "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "es-define-property": "^1.0.1", - "side-channel": "^1.1.1" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "optional": true, - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc-config-loader": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/rc-config-loader/-/rc-config-loader-4.1.4.tgz", - "integrity": "sha512-3GiwEzklkbXTDp52UR5nT8iXgYAx1V9ZG/kDZT7p60u2GCv2XTwQq4NzinMoMpNtXhmt3WkhYXcj6HH8HdwCEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "js-yaml": "^4.1.1", - "json5": "^2.2.3", - "require-from-string": "^2.0.2" - } - }, - "node_modules/read": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", - "integrity": "sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "mute-stream": "~0.0.4" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/read-pkg": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-9.0.1.tgz", - "integrity": "sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/normalize-package-data": "^2.4.3", - "normalize-package-data": "^6.0.0", - "parse-json": "^8.0.0", - "type-fest": "^4.6.0", - "unicorn-magic": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/read-pkg/node_modules/unicorn-magic": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz", - "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reusify": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", - "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/run-applescript": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", - "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, - "license": "MIT" - }, - "node_modules/sax": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", - "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=11.0.0" - } - }, - "node_modules/secretlint": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/secretlint/-/secretlint-10.2.2.tgz", - "integrity": "sha512-xVpkeHV/aoWe4vP4TansF622nBEImzCY73y/0042DuJ29iKIaqgoJ8fGxre3rVSHHbxar4FdJobmTnLp9AU0eg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/config-creator": "^10.2.2", - "@secretlint/formatter": "^10.2.2", - "@secretlint/node": "^10.2.2", - "@secretlint/profiler": "^10.2.2", - "debug": "^4.4.1", - "globby": "^14.1.0", - "read-pkg": "^9.0.1" - }, - "bin": { - "secretlint": "bin/secretlint.js" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/side-channel": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", - "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4", - "side-channel-list": "^1.0.1", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/slash": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", - "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/spdx-correct": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", - "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", - "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", - "dev": true, - "license": "CC-BY-3.0" - }, - "node_modules/spdx-expression-parse": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", - "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.23", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", - "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/structured-source": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz", - "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boundary": "^2.0.0" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-hyperlinks": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.2.0.tgz", - "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=14.18" - }, - "funding": { - "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" - } - }, - "node_modules/table": { - "version": "6.9.0", - "resolved": "https://registry.npmjs.org/table/-/table-6.9.0.tgz", - "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/table/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/table/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tar-fs": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", - "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/terminal-link": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-4.0.0.tgz", - "integrity": "sha512-lk+vH+MccxNqgVqSnkMVKx4VLJfnLjDBGzH16JVZjKE2DoxP57s6/vt6JmXV5I3jBcfGrxNrYtC+mPtU7WJztA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-escapes": "^7.0.0", - "supports-hyperlinks": "^3.2.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true, - "license": "MIT" - }, - "node_modules/textextensions": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-6.11.0.tgz", - "integrity": "sha512-tXJwSr9355kFJI3lbCkPpUH5cP8/M0GGy2xLO34aZCjMXBaK3SoPnZwr/oWmo1FdCnELcs4npdCIOFtq9W3ruQ==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "editions": "^6.21.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/tmp": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", - "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.14" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD" - }, - "node_modules/tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6.11 <=0.7.0 || >=0.7.3" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/type-fest": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typed-rest-client": { - "version": "1.8.11", - "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.11.tgz", - "integrity": "sha512-5UvfMpd1oelmUPRbbaVnq+rHP7ng2cE4qoQkQeAqxRL6PklkxsM0g32/HL0yfvruK6ojQ5x8EE+HF4YV6DtuCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "qs": "^6.9.1", - "tunnel": "0.0.6", - "underscore": "^1.12.1" - } - }, - "node_modules/uc.micro": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "dev": true, - "license": "MIT" - }, - "node_modules/underscore": { - "version": "1.13.8", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", - "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/unicorn-magic": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", - "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/url-join": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", - "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", - "dev": true, - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, - "node_modules/version-range": { - "version": "4.15.0", - "resolved": "https://registry.npmjs.org/version-range/-/version-range-4.15.0.tgz", - "integrity": "sha512-Ck0EJbAGxHwprkzFO966t4/5QkRuzh+/I1RxhLgUKKwEn+Cd8NwM60mE3AqBZg5gYODoXW0EFsQvbZjRlvdqbg==", - "dev": true, - "license": "Artistic-2.0", - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/whatwg-encoding": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", - "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", - "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", - "dev": true, - "license": "MIT", - "dependencies": { - "iconv-lite": "0.6.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/whatwg-mimetype": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", - "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-wsl": "^3.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, - "node_modules/yauzl": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", - "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yazl": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/yazl/-/yazl-2.5.1.tgz", - "integrity": "sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3" - } - } - } -} diff --git a/vscode-extension/package.json b/vscode-extension/package.json index 2099a95..729c300 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -7,12 +7,6 @@ "engines": { "vscode": "^1.80.0" }, - "scripts": { - "package:vsix": "vsce package --allow-missing-repository --skip-license" - }, - "devDependencies": { - "@vscode/vsce": "3.9.2" - }, "categories": [ "Debuggers", "Other"