diff --git a/.gitignore b/.gitignore index 28c74c7..4bdddd1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /target/ -/web/target/ /.clusterflux/ **/.clusterflux/ /vscode-extension/node_modules/ +/private/*/Cargo.lock +!/private/hosted-policy/Cargo.lock +/private/*/target/ +/scripts/containers-home/ diff --git a/CLUSTERFLUX_PUBLIC_TREE.json b/CLUSTERFLUX_PUBLIC_TREE.json new file mode 100644 index 0000000..01b9859 --- /dev/null +++ b/CLUSTERFLUX_PUBLIC_TREE.json @@ -0,0 +1,22 @@ +{ + "kind": "clusterflux-filtered-public-tree", + "source_commit": "98d969b26488b4cda8b2381fa870276a00ca98ea", + "release_name": "release-98d969b26488", + "filtered_out": [ + "private/**", + "internal/**", + "experiments/**", + ".git", + "target", + "git-ignored source paths", + "root/*.md except README.md", + "**/.clusterflux/**", + ".forgejo/**" + ], + "public_export": { + "host_neutral": true, + "include_forgejo_workflows": false + }, + "forgejo_host": "git.michelpaulissen.com", + "default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com" +} diff --git a/Cargo.lock b/Cargo.lock index 26517cc..bb110d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,20 +290,6 @@ dependencies = [ "wasmparser 0.245.1", ] -[[package]] -name = "clusterflux-client" -version = "0.1.0" -dependencies = [ - "base64", - "clusterflux-control", - "clusterflux-coordinator", - "clusterflux-core", - "serde", - "serde_json", - "thiserror 1.0.69", - "tokio", -] - [[package]] name = "clusterflux-control" version = "0.1.0" @@ -981,13 +967,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hello-build" -version = "0.1.0" -dependencies = [ - "clusterflux-sdk", -] - [[package]] name = "hex" version = "0.4.3" @@ -1165,6 +1144,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "launch-build-demo" +version = "0.1.0" +dependencies = [ + "clusterflux-sdk", + "futures-executor", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1681,14 +1670,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "recovery-build" -version = "0.1.0" -dependencies = [ - "clusterflux-sdk", - "serde", -] - [[package]] name = "redox_syscall" version = "0.5.18" diff --git a/Cargo.toml b/Cargo.toml index b4ff4d4..74c9bad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,7 @@ [workspace] resolver = "2" -exclude = ["web"] members = [ "crates/clusterflux-cli", - "crates/clusterflux-client", "crates/clusterflux-control", "crates/clusterflux-coordinator", "crates/clusterflux-core", @@ -12,14 +10,13 @@ members = [ "crates/clusterflux-node", "crates/clusterflux-sdk", "crates/clusterflux-wasm-runtime", - "examples/hello-build", - "examples/recovery-build", + "examples/launch-build-demo", ] [workspace.package] edition = "2021" license = "Apache-2.0 OR MIT" -repository = "https://clusterflux.lesstuff.com" +repository = "https://git.michelpaulissen.com/michel/clusterflux-public" [workspace.dependencies] anyhow = "1.0" diff --git a/README.md b/README.md index e2c11ac..1e2d733 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,14 @@ # Clusterflux -Clusterflux runs a Rust-defined workflow as one distributed virtual process. The -async main runs serverless, provisioning nodes to run tasks through rootless -containers. Tasks on nodes exchange data simply and efficiently. - -The user experience is built to be as much as possible like building a regular -program. The processes are debuggable as normal through a debugger adapter. - -The primary use case is to consolidate build processes into a single streamlined -developer experience. An example program can be seen below: - -~~~rust -use clusterflux::prelude::*; - -#[clusterflux::task(capabilities = "command")] -pub async fn compile(source: SourceSnapshot) -> Result { - let executable = fs::output("hello-clusterflux")?; - Command::new("cc") - .args([ - "-Os", - "-static", - "-s", - "fixture/hello-clusterflux.c", - "-o", - executable.as_str(), - ]) - .cwd(source.mount()?) - .env("SOURCE_DATE_EPOCH", "0") - .network_disabled() - .run() - .await?; - fs::publish(&executable).await -} - -#[clusterflux::main] -pub async fn build() -> Result { - let source = source::current_project().snapshot().await?; - let compile = clusterflux::spawn!(compile(source)) - .on(clusterflux::env!("linux")) - .await?; - compile.join().await -} -~~~ - -After setup, this build pipeline could be deployed as easily as launching it -through your IDE. This repository includes a VS Code extension to make -development as straightforward as possible. A full collection of CLI tools is -included for advanced usage. - -Clusterflux is explicitly local-first. It is trivial to provision existing -hardware as resources. Bulk data will typically not leave the local network, -allowing maximum throughput. The same capability, however, also makes it -possible to leverage cloud resources easily. +Clusterflux runs a Rust-defined workflow as one distributed virtual process. A +coordinator hosts the async main, while attached nodes execute Wasm tasks, +rootless containers, and native commands. Tasks exchange canonical values and +portable typed handles instead of sharing host memory. Start with [Getting started](docs/getting-started.md). It takes you through authentication, project setup, node enrollment, a run, debugging, task restart, and artifact download. -## Build a real executable - -The primary example is intentionally small: [hello-build](examples/hello-build) -snapshots its source project, spawns one container-backed compile task, and -publishes the resulting executable as a retained artifact. - -~~~bash -clusterflux bundle inspect --project examples/hello-build -clusterflux run --project examples/hello-build build -clusterflux artifact list --process -clusterflux artifact download --to ./hello-clusterflux -chmod +x ./hello-clusterflux -./hello-clusterflux -~~~ - -The final command prints `hello from a real Clusterflux build`. The source uses -only the public SDK path: current-project snapshot, `spawn!`, `Command::run`, and -artifact publication. See [recovery-build](examples/recovery-build) for two -same-definition task instances, a real command failure, and operator restart. - ## What you get - One virtual process with distinct task instances and restart attempts. @@ -97,9 +29,6 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ -On NixOS or another system with Nix, the equivalent package is available with -`nix profile install .#clusterflux-tools`. - Rootless Podman is required on Linux nodes that build or run a declared Containerfile environment. Install VS Code when you want the graphical debug workflow. @@ -125,8 +54,8 @@ explicit-key option. Choose an entrypoint and run it: ~~~bash -clusterflux bundle inspect --project examples/hello-build -clusterflux run --project examples/hello-build build +clusterflux bundle inspect --project examples/launch-build-demo +clusterflux run --project examples/launch-build-demo build ~~~ Inspect the result: @@ -136,8 +65,7 @@ clusterflux process status clusterflux task list clusterflux logs clusterflux artifact list -# Use the `artifact` value returned by the list command, for example: -clusterflux artifact download hello-clusterflux-4f61c2... --to ./hello-clusterflux +clusterflux artifact download app.txt --to ./app.txt ~~~ To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md). diff --git a/SECURITY.md b/SECURITY.md index 2c1cd10..a050c6d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained. ## Report a vulnerability -Email security@clusterflux.lesstuff.com with a concise description, affected version +Email security@michelpaulissen.com with a concise description, affected version or source revision, reproduction steps, and impact. Do not open a public issue for an unpatched vulnerability or include credentials, session tokens, private keys, provider tokens, customer data, or operator secrets in a public report. diff --git a/crates/clusterflux-cli/src/admin.rs b/crates/clusterflux-cli/src/admin.rs index ce17d88..83f19a6 100644 --- a/crates/clusterflux-cli/src/admin.rs +++ b/crates/clusterflux-cli/src/admin.rs @@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { .unwrap_or(json!(false)), "response": response, "safe_default": "read_only", - "external_website_required": false, + "private_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result { "command": "admin status", "mode": "self_hosted_local", "safe_default": "read_only", - "external_website_required": false, + "private_website_required": false, })) } @@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "project": project.clone(), "user": user, "coordinator": scope.coordinator, - "external_website_required": false, + "private_website_required": false, "self_hosted_cli_only": true, "project_config_written": project_init .get("project_config_written") @@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> { "step": "start_self_hosted_coordinator", "command": "clusterflux-coordinator --listen 127.0.0.1:0", - "external_website_required": false, + "private_website_required": false, }, { "step": "create_or_link_project", "command": "clusterflux project init --yes", "completed": true, - "external_website_required": false, + "private_website_required": false, }, { "step": "create_node_enrollment_grant", @@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "attach_worker_node", @@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "run_process", @@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}", tenant, project ), - "external_website_required": false, + "private_website_required": false, }, { "step": "inspect_status_logs_artifacts", @@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> "clusterflux artifact list", "clusterflux quota status", ], - "external_website_required": false, + "private_website_required": false, }, { "step": "revoke_access", "command": "clusterflux admin revoke-node --node --yes", - "external_website_required": false, + "private_website_required": false, } ], })) @@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "actor_tenant": actor_tenant, "actor_user": actor_user, "suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"), - "external_website_required": false, + "private_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul "status": "requires_coordinator", "requires_confirmation": !args.yes, "tenant": tenant, - "external_website_required": false, + "private_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/artifact.rs b/crates/clusterflux-cli/src/artifact.rs index a2c0620..a0440be 100644 --- a/crates/clusterflux-cli/src/artifact.rs +++ b/crates/clusterflux-cli/src/artifact.rs @@ -12,7 +12,6 @@ use crate::client::{ }; use crate::config::StoredCliSession; use crate::errors::cli_error_summary_for_category; -use crate::process::hydrate_process_scope; use crate::process_events::{ artifact_download_grant_disclosures, artifact_download_session_summary, artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries, @@ -28,10 +27,9 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result { } pub(crate) fn artifact_list_report_with_session( - mut args: ArtifactListArgs, + args: ArtifactListArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, @@ -55,10 +53,9 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( @@ -146,10 +143,9 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result } pub(crate) fn artifact_export_report_with_session( - mut args: ArtifactExportArgs, + args: ArtifactExportArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); if let Some(coordinator) = &args.scope.coordinator { let mut session = JsonLineSession::connect(coordinator)?; let response = session.request(authenticated_or_local_trusted_request( diff --git a/crates/clusterflux-cli/src/auth.rs b/crates/clusterflux-cli/src/auth.rs index d820cbc..608c9ee 100644 --- a/crates/clusterflux-cli/src/auth.rs +++ b/crates/clusterflux-cli/src/auth.rs @@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result Value { "account_status": "unknown", "suspension_known": false, "account_state_known": false, - "sensitive_moderation_details_exposed": false, + "private_moderation_details_exposed": false, "signup_failure_details_exposed": false, "next_actions": ["clusterflux login --browser"], }) diff --git a/crates/clusterflux-cli/src/config.rs b/crates/clusterflux-cli/src/config.rs index ab744ca..db22a3f 100644 --- a/crates/clusterflux-cli/src/config.rs +++ b/crates/clusterflux-cli/src/config.rs @@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize}; use crate::CliScopeArgs; -pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com"; +pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = + "https://clusterflux.michelpaulissen.com"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct ProjectConfig { diff --git a/crates/clusterflux-cli/src/debug.rs b/crates/clusterflux-cli/src/debug.rs index 356ce8b..d39c780 100644 --- a/crates/clusterflux-cli/src/debug.rs +++ b/crates/clusterflux-cli/src/debug.rs @@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result { "command": "dap", "adapter": dap_binary_path()?.display().to_string(), "args": args.args, - "external_website_required": false, + "private_website_required": false, })) } @@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session( .cloned() .unwrap_or_else(|| json!(0)), "debug_reads_quota_limited": true, - "external_website_required": false, + "private_website_required": false, "coordinator_session_requests": session.requests(), })); } @@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session( "dap": dap, "authorized": "unknown_without_coordinator", "debug_reads_quota_limited": "unknown_without_coordinator", - "external_website_required": false, + "private_website_required": false, })) } diff --git a/crates/clusterflux-cli/src/errors.rs b/crates/clusterflux-cli/src/errors.rs index 282a91d..0dce2be 100644 --- a/crates/clusterflux-cli/src/errors.rs +++ b/crates/clusterflux-cli/src/errors.rs @@ -38,10 +38,7 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s ); object.insert("community_tier_language".to_owned(), json!(true)); object.insert("community_tier_label".to_owned(), json!("community tier")); - object.insert( - "sensitive_abuse_heuristics_exposed".to_owned(), - json!(false), - ); + object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false)); } } summary diff --git a/crates/clusterflux-cli/src/logs.rs b/crates/clusterflux-cli/src/logs.rs index dcad5f7..a12815a 100644 --- a/crates/clusterflux-cli/src/logs.rs +++ b/crates/clusterflux-cli/src/logs.rs @@ -3,7 +3,6 @@ use serde_json::{json, Value}; use crate::client::list_task_events_if_available_with_session; use crate::config::StoredCliSession; -use crate::process::hydrate_process_scope; use crate::process_events::log_entries; use crate::LogsArgs; @@ -13,10 +12,9 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result { } pub(crate) fn logs_report_with_session( - mut args: LogsArgs, + args: LogsArgs, stored_session: Option<&StoredCliSession>, ) -> Result { - hydrate_process_scope(&mut args.scope, stored_session); let events = list_task_events_if_available_with_session( args.scope.coordinator.as_deref(), &args.scope, diff --git a/crates/clusterflux-cli/src/node.rs b/crates/clusterflux-cli/src/node.rs index 3f04516..917310d 100644 --- a/crates/clusterflux-cli/src/node.rs +++ b/crates/clusterflux-cli/src/node.rs @@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result Result Result }; let heartbeat_request = json!({ "type": "node_heartbeat", - "tenant": &tenant, - "project": &project, "node": &plan.node, }); let heartbeat_signature = sign_node_request( diff --git a/crates/clusterflux-cli/src/output.rs b/crates/clusterflux-cli/src/output.rs index 296f142..d5c2b57 100644 --- a/crates/clusterflux-cli/src/output.rs +++ b/crates/clusterflux-cli/src/output.rs @@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String { } push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason"); if let Some(exposed) = account - .get("sensitive_moderation_details_exposed") + .get("private_moderation_details_exposed") .and_then(Value::as_bool) { - lines.push(format!("sensitive moderation details exposed: {exposed}")); + lines.push(format!("private moderation details exposed: {exposed}")); } } if let Some(coordinator_selection) = value.get("coordinator") { @@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String { )); } if let Some(flag) = value - .get("external_website_required") + .get("private_website_required") .and_then(Value::as_bool) { - lines.push(format!("external website required: {flag}")); + lines.push(format!("private website required: {flag}")); } if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) { let actions = next_actions diff --git a/crates/clusterflux-cli/src/process.rs b/crates/clusterflux-cli/src/process.rs index 5e6adde..e7696b4 100644 --- a/crates/clusterflux-cli/src/process.rs +++ b/crates/clusterflux-cli/src/process.rs @@ -15,10 +15,7 @@ use crate::{ ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs, }; -pub(crate) fn hydrate_process_scope( - scope: &mut CliScopeArgs, - stored_session: Option<&StoredCliSession>, -) { +fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) { if scope.coordinator.is_none() { scope.coordinator = stored_session .filter(|session| session.session_secret.is_some()) diff --git a/crates/clusterflux-cli/src/process_events.rs b/crates/clusterflux-cli/src/process_events.rs index 50adc60..fa577c7 100644 --- a/crates/clusterflux-cli/src/process_events.rs +++ b/crates/clusterflux-cli/src/process_events.rs @@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value { "cross_tenant_reuse_allowed": false, "unauthorized_project_reuse_allowed": false, "default_durable_store_assumed": false, - "external_website_required": false, + "private_website_required": false, }]) } @@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option< "current_usage": current_usage, "limits": quota_limits_value(), "next_blocked_action": next_blocked_action, - "sensitive_abuse_heuristics_exposed": false, + "private_abuse_heuristics_exposed": false, }) } diff --git a/crates/clusterflux-cli/src/project.rs b/crates/clusterflux-cli/src/project.rs index 1573880..47140db 100644 --- a/crates/clusterflux-cli/src/project.rs +++ b/crates/clusterflux-cli/src/project.rs @@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result Ok(json!({ "command": "project init", "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, - "external_website_required": false, + "private_website_required": false, "project_config_written": true, "project_config_write_after_coordinator_acceptance": coordinator.is_some(), "coordinator_create_before_local_write": coordinator.is_some(), @@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "config_format": "clusterflux_project_config_v1", "links_current_directory": true, "writes_current_directory_only": true, - "external_website_required": false, + "private_website_required": false, }, "safe_defaults": { "tenant": config.tenant.clone(), @@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result "default_project_id_used": args.new_project == "project", "default_project_name_used": args.name == "Clusterflux Project", "browser_interaction_required": false, - "external_website_required": false, + "private_website_required": false, }, "project_config": config, "config_file": project_config_file(&cwd), @@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "user": user, "projects": projects, "project_count": project_count, - "external_website_required": false, + "private_website_required": false, "response": response, "coordinator_session_requests": session.requests(), })); @@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result "source": "local_project_config", "projects": projects, "project_count": project_count, - "external_website_required": false, + "private_website_required": false, })) } @@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re "source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" }, "selected_project": selected_project, "project_config_written": true, - "external_website_required": false, + "private_website_required": false, "project_config": config, "coordinator_response": coordinator_response, })) diff --git a/crates/clusterflux-cli/src/quota.rs b/crates/clusterflux-cli/src/quota.rs index 9c30880..a37977f 100644 --- a/crates/clusterflux-cli/src/quota.rs +++ b/crates/clusterflux-cli/src/quota.rs @@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "user": effective_scope.user, "coordinator": coordinator, "project_config": config, - "policy_surface": "generic public quota categories; hosted tuning is coordinator-defined", + "policy_surface": "generic public quota categories; hosted tuning remains private policy", "limits": limits, "window_seconds": window_seconds, "current_usage": current_usage, @@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result "next_blocked_action": quota_next_blocked_action(¤t_usage), "quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" }, "quota_tier": quota_tier, - "sensitive_abuse_heuristics_exposed": false, + "private_abuse_heuristics_exposed": false, "quota_response": quota_status, })) } diff --git a/crates/clusterflux-cli/src/run.rs b/crates/clusterflux-cli/src/run.rs index 7dfc187..ea83a9e 100644 --- a/crates/clusterflux-cli/src/run.rs +++ b/crates/clusterflux-cli/src/run.rs @@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu "safe_failure": true, "message": message, "next_actions": next_actions, - "external_website_required": false, + "private_website_required": false, "machine_error": machine_error, }) } @@ -195,14 +195,12 @@ fn coordinator_run_report(plan: RunPlan) -> Result { ); } let process = "vp-current".to_owned(); - let launch_attempt = new_launch_attempt_id(); let mut session = JsonLineSession::connect(&coordinator)?; let mut request = json!({ "type": "start_process", "tenant": tenant.clone(), "project": project.clone(), "process": process.clone(), - "launch_attempt": launch_attempt.clone(), "restart": false, }); request = authenticated_human_or_local_trusted_workflow( @@ -221,7 +219,6 @@ fn coordinator_run_report(plan: RunPlan) -> Result { &tenant, &project, &process, - &launch_attempt, )?; let run_start = run_start_summary(&response); let status = run_start @@ -289,7 +286,6 @@ fn coordinator_run_report(plan: RunPlan) -> Result { tenant: &tenant, project: &project, process: &process, - launch_attempt: &launch_attempt, }; let cleanup = rollback_failed_process_launch_reconnecting( &coordinator, @@ -329,7 +325,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result { "task_launch": launch_task_response, "coordinator_response": response, "coordinator_session_requests": session.requests(), - "external_website_required": false, + "private_website_required": false, })) } @@ -344,7 +340,6 @@ fn request_process_start_with_rollback( tenant: &str, project: &str, process: &str, - launch_attempt: &str, ) -> Result { let rollback = ProcessLaunchRollback { cli_session, @@ -353,25 +348,9 @@ fn request_process_start_with_rollback( tenant, project, process, - launch_attempt, }; match session.request_allow_error(request) { - Ok(response) => { - if response.get("type").and_then(Value::as_str) == Some("process_started") - && response.get("launch_attempt").and_then(Value::as_str) != Some(launch_attempt) - { - let ownership_error = anyhow::anyhow!( - "coordinator returned a process-start acknowledgement owned by a different launch attempt" - ); - rollback_failed_process_launch_reconnecting( - coordinator, - &rollback, - &json!({ "error": ownership_error.to_string(), "phase": "start_process_ownership" }), - )?; - return Err(ownership_error); - } - Ok(response) - } + Ok(response) => Ok(response), Err(start_error) => { let cleanup = rollback_failed_process_launch_reconnecting( coordinator, @@ -391,17 +370,6 @@ fn request_process_start_with_rollback( } } -static NEXT_LAUNCH_ATTEMPT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); - -fn new_launch_attempt_id() -> String { - let sequence = NEXT_LAUNCH_ATTEMPT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - format!("launch-{}-{nanos}-{sequence}", std::process::id()) -} - struct ProcessLaunchRollback<'a> { cli_session: &'a CliSession, fallback_user: &'a str, @@ -409,7 +377,6 @@ struct ProcessLaunchRollback<'a> { tenant: &'a str, project: &'a str, process: &'a str, - launch_attempt: &'a str, } fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> { @@ -435,7 +402,6 @@ fn rollback_failed_process_launch( "tenant": context.tenant, "project": context.project, "process": context.process, - "launch_attempt": context.launch_attempt, }), context.cli_session, context.fallback_user, @@ -1017,10 +983,6 @@ mod transactional_launch_tests { payload.get("process").and_then(Value::as_str), Some("vp-current") ); - assert_eq!( - payload.get("launch_attempt").and_then(Value::as_str), - Some("launch-test") - ); writeln!( stream, "{}", @@ -1041,7 +1003,6 @@ mod transactional_launch_tests { tenant: "tenant-a", project: "project-a", process: "vp-current", - launch_attempt: "launch-test", }; rollback_failed_process_launch( &mut session, @@ -1063,7 +1024,6 @@ mod transactional_launch_tests { .read_line(&mut start_line) .unwrap(); assert!(start_line.contains("\"type\":\"start_process\"")); - assert!(start_line.contains("\"launch_attempt\":\"launch-test\"")); drop(stream); let (mut cleanup_stream, _) = listener.accept().unwrap(); @@ -1072,7 +1032,6 @@ mod transactional_launch_tests { .read_line(&mut cleanup_line) .unwrap(); assert!(cleanup_line.contains("\"type\":\"abort_process\"")); - assert!(cleanup_line.contains("\"launch_attempt\":\"launch-test\"")); writeln!( cleanup_stream, "{}", @@ -1084,12 +1043,6 @@ mod transactional_launch_tests { }) ) .unwrap(); - listener.set_nonblocking(true).unwrap(); - std::thread::sleep(std::time::Duration::from_millis(25)); - assert!(matches!( - listener.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock - )); }); let mut session = JsonLineSession::connect(&address).unwrap(); let error = request_process_start_with_rollback( @@ -1100,7 +1053,6 @@ mod transactional_launch_tests { "project": "project-a", "actor_user": "user-a", "process": "vp-current", - "launch_attempt": "launch-test", "restart": false }), &address, @@ -1110,63 +1062,10 @@ mod transactional_launch_tests { "tenant-a", "project-a", "vp-current", - "launch-test", ) .unwrap_err() .to_string(); assert!(error.contains("closed") || error.contains("response")); server.join().unwrap(); } - - #[test] - fn definitive_start_rejection_does_not_abort_existing_process() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap().to_string(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut line = String::new(); - std::io::BufReader::new(stream.try_clone().unwrap()) - .read_line(&mut line) - .unwrap(); - writeln!( - stream, - "{}", - json!({ - "type": "error", - "message": "project already has active virtual process vp-existing" - }) - ) - .unwrap(); - listener.set_nonblocking(true).unwrap(); - std::thread::sleep(std::time::Duration::from_millis(25)); - assert_eq!( - listener.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock - ); - }); - let mut session = JsonLineSession::connect(&address).unwrap(); - let response = request_process_start_with_rollback( - &mut session, - json!({ - "type": "start_process", - "tenant": "tenant-a", - "project": "project-a", - "actor_user": "user-a", - "process": "vp-current", - "launch_attempt": "launch-rejected", - "restart": false - }), - &address, - &CliSession::Anonymous, - "user-a", - None, - "tenant-a", - "project-a", - "vp-current", - "launch-rejected", - ) - .unwrap(); - assert_eq!(response.get("type").and_then(Value::as_str), Some("error")); - server.join().unwrap(); - } } diff --git a/crates/clusterflux-cli/src/tests.rs b/crates/clusterflux-cli/src/tests.rs index 57b4afe..e2350f8 100644 --- a/crates/clusterflux-cli/src/tests.rs +++ b/crates/clusterflux-cli/src/tests.rs @@ -10,15 +10,6 @@ fn parse(args: &[&str]) -> Cli { Cli::parse_from(args) } -fn launch_attempt_from_wire(line: &str) -> String { - let wire: Value = serde_json::from_str(line).unwrap(); - wire.pointer("/payload/request/launch_attempt") - .or_else(|| wire.pointer("/payload/launch_attempt")) - .and_then(Value::as_str) - .expect("start request must carry a launch attempt") - .to_owned() -} - fn write_runnable_wasm_project(project: &Path) { let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); let sdk = cli_manifest_dir.parent().unwrap().join("clusterflux-sdk"); @@ -124,7 +115,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() { assert_eq!(summary["resource_category"], "api_calls"); assert_eq!(summary["community_tier_language"], true); assert_eq!(summary["community_tier_label"], "community tier"); - assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false); + assert_eq!(summary["private_abuse_heuristics_exposed"], false); let rendered = human_report(&json!({ "command": "run", "machine_error": summary, @@ -207,7 +198,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() { assert_eq!(report["status"], "authentication_required"); assert_eq!(report["non_interactive"], true); assert_eq!(report["browser_opened"], false); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["machine_error"]["category"], "authentication"); assert_eq!(report["machine_error"]["stable_exit_code"], 20); assert_eq!(report["machine_error"]["browser_opened"], false); @@ -447,11 +438,10 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() { assert!(line.contains(r#""nonce":"agent-signature-"#)); assert!(line.contains(r#""signature":"ed25519:"#)); assert!(!line.contains(r#""actor_user""#)); - let launch_attempt = launch_attempt_from_wire(&line); stream .write_all( format!( - r#"{{"type":"process_started","process":"vp-current","launch_attempt":"{launch_attempt}","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"# + r#"{{"type":"process_started","process":"vp-current","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"# ) .as_bytes(), ) @@ -598,32 +588,7 @@ fn run_with_human_session_derives_workflow_scope_from_authenticated_envelope() { assert!(wire["payload"]["request"].get("tenant").is_none()); assert!(wire["payload"]["request"].get("project").is_none()); assert!(wire["payload"]["request"].get("actor_user").is_none()); - if expected_operation == "start_process" { - let launch_attempt = launch_attempt_from_wire(&line); - stream - .write_all( - serde_json::to_string(&json!({ - "type": "process_started", - "process": "vp-current", - "launch_attempt": launch_attempt, - "epoch": 7, - "actor": { - "kind": "user", - "user": "user-session", - "agent": null, - "credential_kind": "cli_device_session", - "public_key_fingerprint": null, - "authenticated_without_browser": false, - "scopes": ["project:read", "project:run"] - } - })) - .unwrap() - .as_bytes(), - ) - .unwrap(); - } else { - stream.write_all(response).unwrap(); - } + stream.write_all(response).unwrap(); stream.write_all(b"\n").unwrap(); } }); @@ -696,18 +661,6 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() { assert!(line.contains(r#""project":"project-live""#)); assert!(line.contains(r#""process":"vp-current""#)); assert!(line.contains(r#""restart":false"#)); - let response = if index == 0 { - let launch_attempt = launch_attempt_from_wire(&line); - serde_json::to_string(&json!({ - "type": "process_started", - "process": "vp-current", - "launch_attempt": launch_attempt, - "epoch": 7, - })) - .unwrap() - } else { - response.to_owned() - }; stream.write_all(response.as_bytes()).unwrap(); stream.write_all(b"\n").unwrap(); if index == 0 { @@ -827,7 +780,7 @@ fn run_rejection_reports_machine_readable_error_category() { assert_eq!(rejected["machine_error"]["resource_category"], "api_calls"); assert_eq!(rejected["machine_error"]["community_tier_language"], true); assert_eq!( - rejected["machine_error"]["sensitive_abuse_heuristics_exposed"], + rejected["machine_error"]["private_abuse_heuristics_exposed"], false ); assert!(rejected["machine_error"]["next_actions"] @@ -1665,11 +1618,12 @@ fn node_attach_refuses_a_symlink_credential_target() { fn hosted_coordinator_remains_a_real_https_control_endpoint() { assert_eq!( control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") + .unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert!(control_endpoint_identity("http://operator.example.test").is_err()); assert_eq!( @@ -1787,7 +1741,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { assert!(!line.contains(r#""actor_user":"user-session""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, + br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1850,7 +1804,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() { "active" ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); } @@ -1932,7 +1886,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() { } #[test] -fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() { +fn auth_status_queries_coordinator_account_state_without_private_moderation_details() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap().to_string(); let server = std::thread::spawn(move || { @@ -1946,7 +1900,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de assert!(line.contains(r#""actor_user":"user-live""#)); stream .write_all( - br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#, + br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#, ) .unwrap(); stream.write_all(b"\n").unwrap(); @@ -1986,7 +1940,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de "account or tenant is suspended by hosted policy" ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); assert_eq!( @@ -2004,13 +1958,13 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de let serialized = serde_json::to_string(&report).unwrap(); assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); - assert!(!serialized.contains("sensitive moderation note")); + assert!(!serialized.contains("private moderation note")); let rendered = human_report(&report); assert!(rendered.contains("account status: suspended")); assert!(rendered.contains("account suspended: true")); - assert!(rendered.contains("sensitive moderation details exposed: false")); - assert!(!rendered.contains("sensitive moderation note")); + assert!(rendered.contains("private moderation details exposed: false")); + assert!(!rendered.contains("private moderation note")); } #[test] @@ -2061,11 +2015,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { "manual_review": manual_review, "sanitized_reason": reason, "next_actions": ["contact the hosted operator"], - "sensitive_moderation_details_exposed": false, + "private_moderation_details_exposed": false, "signup_failure_details_exposed": false, "abuse_score": 99, - "moderation_notes": "sensitive moderation note", - "signup_policy_trace": "sensitive signup trace", + "moderation_notes": "private moderation note", + "signup_policy_trace": "private signup trace", }) ) .unwrap(); @@ -2104,7 +2058,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { true ); assert_eq!( - report["coordinator_account_status"]["sensitive_moderation_details_exposed"], + report["coordinator_account_status"]["private_moderation_details_exposed"], false ); assert_eq!( @@ -2115,11 +2069,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() { assert!(!serialized.contains("abuse_score")); assert!(!serialized.contains("moderation_notes")); assert!(!serialized.contains("signup_policy_trace")); - assert!(!serialized.contains("sensitive moderation note")); + assert!(!serialized.contains("private moderation note")); let rendered = human_report(&report); assert!(rendered.contains(&format!("account status: {status}"))); assert!(rendered.contains(rendered_marker)); - assert!(!rendered.contains("sensitive moderation note")); + assert!(!rendered.contains("private moderation note")); } server.join().unwrap(); } @@ -2234,11 +2188,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { assert_eq!(report["command"], "admin bootstrap"); assert_eq!(report["mode"], "self_hosted_local"); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["self_hosted_cli_only"], true); assert_eq!(report["project_config_written"], true); assert_eq!(report["project_init"]["command"], "project init"); - assert_eq!(report["project_init"]["external_website_required"], false); + assert_eq!(report["project_init"]["private_website_required"], false); assert_eq!( report["project_init"]["project_config"]["project"], "self-hosted" @@ -2264,7 +2218,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() { } assert!(steps.iter().all(|step| { !step - .get("external_website_required") + .get("private_website_required") .and_then(Value::as_bool) .unwrap_or(false) })); @@ -2892,13 +2846,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() { assert_eq!(status["command"], "admin status"); assert_eq!(status["safe_default"], "read_only"); - assert_eq!(status["external_website_required"], false); + assert_eq!(status["private_website_required"], false); assert_eq!(status["suspended"], false); assert_eq!(suspended["command"], "admin suspend-tenant"); assert_eq!(suspended["tenant"], "tenant"); assert_eq!(suspended["actor_tenant"], "admin-tenant"); assert_eq!(suspended["suspended"], true); - assert_eq!(suspended["external_website_required"], false); + assert_eq!(suspended["private_website_required"], false); } #[test] @@ -2967,7 +2921,7 @@ fn debug_attach_reports_public_authorization() { assert_eq!(report["charged_debug_read_bytes"], 1024); assert_eq!(report["used_debug_read_bytes"], 1024); assert_eq!(report["debug_reads_quota_limited"], true); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); } #[test] @@ -3004,14 +2958,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { "restart_task", br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(), ), - ( - "list_task_events", - br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(), - ), - ( - "list_task_events", - br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(), - ), ( "create_artifact_download_link", br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(), @@ -3116,26 +3062,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); - let logs = logs_report_with_session( - LogsArgs { - scope: scope.clone(), - process: Some("vp".to_owned()), - task: Some("task-a".to_owned()), - }, - Some(&session), - ) - .unwrap(); - let artifacts = artifact_list_report_with_session( - ArtifactListArgs { - scope: scope.clone(), - process: Some("vp".to_owned()), - }, - Some(&session), - ) - .unwrap(); - let download = artifact_download_report_with_session( + artifact_download_report_with_session( ArtifactDownloadArgs { - scope, + scope: coordinator_scope.clone(), artifact: "app.txt".to_owned(), to: None, max_bytes: 2048, @@ -3143,9 +3072,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() { Some(&session), ) .unwrap(); - assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n"); - assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt"); - assert_eq!(download["coordinator"], session.coordinator); debug_attach_report_with_dap_and_session( DebugAttachArgs { scope: coordinator_scope, @@ -3338,7 +3264,7 @@ fn project_init_uses_public_create_before_writing_local_config() { created["safe_defaults"]["browser_interaction_required"], false ); - assert_eq!(created["external_website_required"], false); + assert_eq!(created["private_website_required"], false); assert_eq!( read_project_config(temp_success.path()) .unwrap() @@ -3438,7 +3364,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(list["source"], "public_coordinator_api"); assert_eq!(list["project_count"], 1); assert_eq!(list["projects"][0]["id"], "project-a"); - assert_eq!(list["external_website_required"], false); + assert_eq!(list["private_website_required"], false); assert_eq!(list["coordinator_session_requests"], 1); let selected = project_select_report( @@ -3453,7 +3379,7 @@ fn project_list_and_select_use_public_api_without_website() { assert_eq!(selected["source"], "public_coordinator_api"); assert_eq!(selected["selected_project"]["id"], "project-a"); assert_eq!(selected["project_config_written"], true); - assert_eq!(selected["external_website_required"], false); + assert_eq!(selected["private_website_required"], false); assert_eq!( read_project_config(temp.path()).unwrap().unwrap().project, "project-a" @@ -4482,7 +4408,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { let temp = tempfile::tempdir().unwrap(); let project = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../..") - .join("examples/hello-build"); + .join("examples/launch-build-demo"); let output = temp.path().join("bundle"); let report = build_report( @@ -4500,8 +4426,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { assert_eq!(report["command"], "build"); assert_eq!(report["content_addressed"], true); assert_eq!(report["contains_full_repository_upload"], false); - assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2); - assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1); + assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 10); + assert_eq!(report["bundle_artifact"]["entrypoint_count"], 5); assert!(output.join("module.wasm").is_file()); assert!(output.join("manifest.json").is_file()); assert!(output.join("task-descriptors.json").is_file()); @@ -4512,9 +4438,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() { .unwrap() .iter() .any(|descriptor| { - descriptor["name"] == "compile" - && descriptor["argument_schema"] == "source : SourceSnapshot" - && descriptor["result_schema"] == "Result < Artifact >" + descriptor["name"] == "task_add_one" + && descriptor["argument_schema"] == "input : i32" + && descriptor["result_schema"] == "i32" && descriptor["restart_compatibility_hash"] .as_str() .unwrap() @@ -4675,7 +4601,7 @@ fn node_enroll_reports_short_lived_public_api_grant() { assert_eq!(report["command"], "node enroll"); assert_eq!(report["status"], "created"); - assert_eq!(report["external_website_required"], false); + assert_eq!(report["private_website_required"], false); assert_eq!(report["tenant"], "tenant"); assert_eq!(report["project"], "project"); assert_eq!(report["user"], "user"); @@ -4818,7 +4744,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() { ) .unwrap(); assert_eq!(enroll["status"], "requires_coordinator"); - assert_eq!(enroll["external_website_required"], false); + assert_eq!(enroll["private_website_required"], false); assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null); assert_eq!(enroll["requested_ttl_seconds"], 60); diff --git a/crates/clusterflux-client/Cargo.toml b/crates/clusterflux-client/Cargo.toml deleted file mode 100644 index fa2b5cc..0000000 --- a/crates/clusterflux-client/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "clusterflux-client" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -base64.workspace = true -clusterflux-control = { path = "../clusterflux-control" } -clusterflux-core = { path = "../clusterflux-core" } -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tokio = { workspace = true, features = ["sync", "time"] } - -[dev-dependencies] -clusterflux-coordinator = { path = "../clusterflux-coordinator" } diff --git a/crates/clusterflux-client/README.md b/crates/clusterflux-client/README.md deleted file mode 100644 index 89951c1..0000000 --- a/crates/clusterflux-client/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# clusterflux-client - -`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web -backend. It talks to the same versioned control and login endpoints as the CLI. -Callers do not construct JSON envelopes or place session secrets into request -objects. - -The crate intentionally contains no coordinator implementation, persistence -types, scheduler policy, or website-specific business rules. A web backend only -needs this crate to authenticate and work with account status, projects, Agent -keys, node enrollment and liveness, current/recent processes, task attempts, -recent logs, artifacts, quota state, process control, task recovery, and Debug -Epoch state. It does not expose workflow launch or whole-process replay. - -```rust,no_run -use clusterflux_client::{ClusterfluxClient, SessionCredential}; - -# async fn example() -> Result<(), clusterflux_client::ClientError> { -let credential = SessionCredential::from_secret( - std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"), -); -let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")? - .with_session_credential(&credential); - -let account = client.account_status().await?; -let nodes = client.list_nodes().await?; -let processes = client.list_processes(None, 20).await?; -# let _ = (account, nodes, processes); -# Ok(()) -# } -``` - -## Login boundary - -`begin_browser_login` starts the hosted Authentik flow. The identity callback -returns only to the fixed configured website URL with a short-lived handoff. -The website backend calls `exchange_browser_login_handoff` once and stores the -returned `SessionCredential` only in encrypted server-side session storage. -The credential has redacted `Debug` output and deliberately does not implement -Serde serialization. Logout calls the existing session-revocation operation. - -The browser must not receive or persist the Clusterflux session credential, -provider tokens, authorization code, PKCE verifier, or CLI polling secret. - -## Errors, bounds, and transport - -API failures are returned as `ClientError::Api(ApiError)`. Decisions should use -the stable `code`, `category`, `retryable`, and `request_id` fields, while -`message` is for people. The client rejects an error whose request ID does not -match the originating request. - -Paginated list methods require an explicit bounded page size. Process pages are -limited to 100 entries; node, artifact, and recent-log pages to 200. -`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes -its cursor. Recent logs are ephemeral and can report truncation or sequence -loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates -offsets and decodes bounded chunks without materializing the complete artifact. - -`ControlTransport` has bounded connect and I/O timeouts and performs blocking -network work outside the async executor. Dropping a request future cancels the -caller’s wait; any already-started blocking I/O remains bounded by its timeout. -`MockTransport` supplies deterministic responses and records exact envelopes -for application tests. - -`CLIENT_API_VERSION` is the one supported control protocol version. The -checked-in `web_operations.json` contract fixture records every website -operation and its stable error shape, and the crate’s contract suite checks the -fixture. diff --git a/crates/clusterflux-client/src/lib.rs b/crates/clusterflux-client/src/lib.rs deleted file mode 100644 index 93c6feb..0000000 --- a/crates/clusterflux-client/src/lib.rs +++ /dev/null @@ -1,924 +0,0 @@ -mod protocol; -mod transport; -mod types; - -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; - -use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; -use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; -use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION}; -use protocol::{AuthenticatedRequest, LoginRequest, WireResponse}; -use serde_json::json; -use thiserror::Error; -use transport::{ClientTransport, TransportRequest}; - -pub use clusterflux_core::{ - AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind, - Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId, - ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId, - UserId, VfsPath, -}; -pub use transport::{ - ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse, -}; -pub use types::*; - -pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION; - -#[derive(Debug, Error)] -pub enum ClientError { - #[error("Clusterflux API error: {0}")] - Api(ApiError), - #[error(transparent)] - Transport(#[from] ClientTransportError), - #[error("Clusterflux client protocol error: {0}")] - Protocol(String), -} - -#[derive(Clone)] -pub struct ClusterfluxClient { - transport: Arc, - session_secret: Arc>>, - next_request: Arc, -} - -impl ClusterfluxClient { - pub fn connect(endpoint: impl Into) -> Result { - Ok(Self::with_transport(ControlTransport::new(endpoint)?)) - } - - pub fn with_transport(transport: impl ClientTransport) -> Self { - Self { - transport: Arc::new(transport), - session_secret: Arc::new(Mutex::new(None)), - next_request: Arc::new(AtomicU64::new(1)), - } - } - - pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self { - self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone()))); - self - } - - pub fn is_session_configured(&self) -> bool { - self.session_secret - .lock() - .map(|secret| secret.is_some()) - .unwrap_or(false) - } - - pub async fn begin_browser_login(&self) -> Result { - match self - .send_login(LoginRequest::BeginWebBrowserLogin {}) - .await? - { - WireResponse::WebBrowserLoginStarted { - transaction_id, - authorization_url, - expires_at_epoch_seconds, - } => Ok(BrowserLoginStart { - transaction_id, - authorization_url, - expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("web_browser_login_started")), - } - } - - pub async fn exchange_browser_login_handoff( - &self, - transaction_id: impl Into, - handoff_code: impl Into, - ) -> Result { - match self - .send_login(LoginRequest::ExchangeWebLoginHandoff { - transaction_id: transaction_id.into(), - handoff_code: handoff_code.into(), - }) - .await? - { - WireResponse::WebBrowserSession { session } => Ok(BrowserSession { - tenant: session.tenant, - project: session.project, - user: session.user, - credential: SessionCredential(session.session_secret), - expires_at_epoch_seconds: session.expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("web_browser_session")), - } - } - - pub async fn cancel_browser_login( - &self, - transaction_id: impl Into, - ) -> Result<(), ClientError> { - match self - .send_login(LoginRequest::CancelWebBrowserLogin { - transaction_id: transaction_id.into(), - }) - .await? - { - WireResponse::WebBrowserLoginCancelled {} => Ok(()), - _ => Err(unexpected_response("web_browser_login_cancelled")), - } - } - - pub async fn account_status(&self) -> Result { - match self - .send_authenticated(AuthenticatedRequest::AuthStatus) - .await? - { - WireResponse::AuthStatus { - tenant, - project, - actor, - authenticated, - account_status, - suspended, - disabled, - deleted, - manual_review, - sanitized_reason, - next_actions, - } => Ok(AccountStatus { - tenant, - project, - actor, - authenticated, - account_status, - suspended, - disabled, - deleted, - manual_review, - sanitized_reason, - next_actions, - }), - _ => Err(unexpected_response("auth_status")), - } - } - - pub async fn logout(&self) -> Result<(), ClientError> { - match self - .send_authenticated(AuthenticatedRequest::RevokeCliSession) - .await? - { - WireResponse::CliSessionRevoked {} => { - *self.session_secret.lock().map_err(|_| { - ClientError::Protocol("session credential lock was poisoned".to_owned()) - })? = None; - Ok(()) - } - _ => Err(unexpected_response("cli_session_revoked")), - } - } - - pub async fn list_projects(&self) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListProjects) - .await? - { - WireResponse::Projects { projects } => Ok(projects), - _ => Err(unexpected_response("projects")), - } - } - - pub async fn create_project( - &self, - project: ProjectId, - name: impl Into, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateProject { - project, - name: name.into(), - }) - .await? - { - WireResponse::ProjectCreated { project } => Ok(project), - _ => Err(unexpected_response("project_created")), - } - } - - pub async fn select_project(&self, project: ProjectId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::SelectProject { project }) - .await? - { - WireResponse::ProjectSelected { project } => Ok(project), - _ => Err(unexpected_response("project_selected")), - } - } - - pub async fn register_agent_public_key( - &self, - agent: AgentId, - public_key: impl Into, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey { - agent, - public_key: public_key.into(), - }) - .await - } - - pub async fn list_agent_public_keys(&self) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListAgentPublicKeys) - .await? - { - WireResponse::AgentPublicKeys { records } => Ok(records), - _ => Err(unexpected_response("agent_public_keys")), - } - } - - pub async fn rotate_agent_public_key( - &self, - agent: AgentId, - public_key: impl Into, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey { - agent, - public_key: public_key.into(), - }) - .await - } - - pub async fn revoke_agent_public_key( - &self, - agent: AgentId, - ) -> Result { - self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent }) - .await - } - - async fn agent_key_mutation( - &self, - request: AuthenticatedRequest, - ) -> Result { - match self.send_authenticated(request).await? { - WireResponse::AgentPublicKey { record } => Ok(record), - _ => Err(unexpected_response("agent_public_key")), - } - } - - pub async fn create_node_enrollment_grant( - &self, - ttl_seconds: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds }) - .await? - { - WireResponse::NodeEnrollmentGrantCreated { - tenant, - project, - grant, - scope, - expires_at_epoch_seconds, - } => Ok(NodeEnrollmentGrant { - tenant, - project, - grant, - scope, - expires_at_epoch_seconds, - }), - _ => Err(unexpected_response("node_enrollment_grant_created")), - } - } - - pub async fn list_nodes(&self) -> Result, ClientError> { - Ok(self.list_nodes_page(None, 200).await?.nodes) - } - - pub async fn list_nodes_page( - &self, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit }) - .await? - { - WireResponse::NodeSummaries { nodes, next_cursor } => { - Ok(NodePage { nodes, next_cursor }) - } - _ => Err(unexpected_response("node_summaries")), - } - } - - pub async fn revoke_node(&self, node: NodeId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node }) - .await? - { - WireResponse::NodeCredentialRevoked { - node, - tenant, - project, - actor, - descriptor_removed, - queued_assignments_removed, - } => Ok(NodeRevocation { - node, - tenant, - project, - actor, - descriptor_removed, - queued_assignments_removed, - }), - _ => Err(unexpected_response("node_credential_revoked")), - } - } - - pub async fn list_processes( - &self, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit }) - .await? - { - WireResponse::ProcessSummaries { - processes, - next_cursor, - } => Ok(ProcessPage { - processes, - next_cursor, - }), - _ => Err(unexpected_response("process_summaries")), - } - } - - pub async fn cancel_process( - &self, - process: ProcessId, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CancelProcess { process }) - .await? - { - WireResponse::ProcessCancellationRequested { - process, - cancelled_tasks, - affected_nodes, - } => Ok(ProcessCancellation { - process, - affected_tasks: cancelled_tasks, - affected_nodes, - aborted: false, - }), - _ => Err(unexpected_response("process_cancellation_requested")), - } - } - - pub async fn abort_process( - &self, - process: ProcessId, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::AbortProcess { - process, - launch_attempt: None, - }) - .await? - { - WireResponse::ProcessAborted { - process, - aborted_tasks, - affected_nodes, - } => Ok(ProcessCancellation { - process, - affected_tasks: aborted_tasks, - affected_nodes, - aborted: true, - }), - _ => Err(unexpected_response("process_aborted")), - } - } - - pub async fn quota_status(&self) -> Result { - match self - .send_authenticated(AuthenticatedRequest::QuotaStatus) - .await? - { - WireResponse::QuotaStatus { - tenant, - project, - actor, - policy_label, - limits, - window_seconds, - usage, - window_started_epoch_seconds, - } => Ok(QuotaStatus { - tenant, - project, - actor, - policy_label, - limits, - window_seconds, - usage, - window_started_epoch_seconds, - }), - _ => Err(unexpected_response("quota_status")), - } - } - - pub async fn list_task_events( - &self, - process: Option, - ) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListTaskEvents { process }) - .await? - { - WireResponse::TaskEvents { events } => Ok(events), - _ => Err(unexpected_response("task_events")), - } - } - - pub async fn list_task_snapshots( - &self, - process: ProcessId, - ) -> Result, ClientError> { - match self - .send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process }) - .await? - { - WireResponse::TaskSnapshots { snapshots } => Ok(snapshots), - _ => Err(unexpected_response("task_snapshots")), - } - } - - pub async fn list_recent_logs( - &self, - process: ProcessId, - task: Option, - after_sequence: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListRecentLogs { - process, - task, - after_sequence, - limit, - }) - .await? - { - WireResponse::RecentLogs { - entries, - next_sequence, - history_truncated, - } => Ok(RecentLogPage { - entries, - next_sequence, - history_truncated, - }), - _ => Err(unexpected_response("recent_logs")), - } - } - - pub async fn restart_task( - &self, - process: ProcessId, - task: TaskInstanceId, - replacement_bundle: Option, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::RestartTask { - process, - task, - replacement_bundle, - }) - .await? - { - WireResponse::TaskRestart { - process, - task, - restarted_task_instance, - restarted_attempt_id, - actor, - accepted, - clean_boundary_available, - active_task, - completed_event_observed, - requires_whole_process_restart, - message, - audit_event, - } => Ok(TaskRestart { - process, - task, - restarted_task_instance, - restarted_attempt_id, - actor, - accepted, - clean_boundary_available, - active_task, - completed_event_observed, - requires_whole_process_restart, - message, - audit_event, - }), - _ => Err(unexpected_response("task_restart")), - } - } - - pub async fn resolve_task_failure( - &self, - process: ProcessId, - task: TaskInstanceId, - resolution: TaskFailureResolution, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ResolveTaskFailure { - process, - task, - resolution, - }) - .await? - { - WireResponse::TaskFailureResolved { - process, - task, - attempt_id, - resolution, - } => Ok(TaskFailureResolutionResult { - process, - task, - attempt_id, - resolution, - }), - _ => Err(unexpected_response("task_failure_resolved")), - } - } - - pub async fn debug_attach(&self, process: ProcessId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::DebugAttach { process }) - .await? - { - WireResponse::DebugAttach { - process, - actor, - authorization, - audit_event, - } => Ok(DebugAttach { - process, - actor, - authorization, - audit_event, - }), - _ => Err(unexpected_response("debug_attach")), - } - } - - pub async fn create_debug_epoch( - &self, - process: ProcessId, - stopped_task: TaskInstanceId, - reason: impl Into, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateDebugEpoch { - process, - stopped_task, - reason: reason.into(), - }) - .await? - { - WireResponse::DebugEpoch { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - } => Ok(DebugEpochControl { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch")), - } - } - - pub async fn resume_debug_epoch( - &self, - process: ProcessId, - epoch: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch }) - .await? - { - WireResponse::DebugEpoch { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - } => Ok(DebugEpochControl { - process, - actor, - epoch, - command, - affected_tasks, - all_stop_requested, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch")), - } - } - - pub async fn inspect_debug_epoch( - &self, - process: ProcessId, - epoch: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch }) - .await? - { - WireResponse::DebugEpochStatus { - process, - actor, - epoch, - command, - expected_tasks, - acknowledgements, - fully_frozen, - partially_frozen, - fully_resumed, - failed, - failure_messages, - audit_event, - } => Ok(DebugEpochStatus { - process, - actor, - epoch, - command, - expected_tasks, - acknowledgements, - fully_frozen, - partially_frozen, - fully_resumed, - failed, - failure_messages, - audit_event, - }), - _ => Err(unexpected_response("debug_epoch_status")), - } - } - - pub async fn list_artifacts( - &self, - process: Option, - cursor: Option, - limit: u32, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::ListArtifacts { - process, - cursor, - limit, - }) - .await? - { - WireResponse::Artifacts { - artifacts, - next_cursor, - } => Ok(ArtifactPage { - artifacts, - next_cursor, - }), - _ => Err(unexpected_response("artifacts")), - } - } - - pub async fn get_artifact(&self, artifact: ArtifactId) -> Result { - match self - .send_authenticated(AuthenticatedRequest::GetArtifact { artifact }) - .await? - { - WireResponse::Artifact { artifact } => Ok(artifact), - _ => Err(unexpected_response("artifact")), - } - } - - pub async fn begin_artifact_download( - &self, - artifact: ArtifactId, - max_bytes: u64, - ttl_seconds: u64, - chunk_bytes: u64, - ) -> Result { - match self - .send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink { - artifact: artifact.clone(), - max_bytes, - ttl_seconds, - }) - .await? - { - WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload { - client: self.clone(), - artifact, - max_bytes, - chunk_bytes, - token_digest: link.scoped_token_digest.clone(), - link, - expected_offset: 0, - finished: false, - }), - _ => Err(unexpected_response("artifact_download_link")), - } - } - - async fn send_authenticated( - &self, - request: AuthenticatedRequest, - ) -> Result { - let session_secret = self - .session_secret - .lock() - .map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))? - .clone() - .ok_or_else(|| { - ClientError::Api(ApiError::from_message( - "client", - "no authenticated Clusterflux session is configured", - )) - })?; - self.send( - CONTROL_API_PATH, - json!({ - "type": "authenticated", - "session_secret": session_secret, - "request": request, - }), - ) - .await - } - - async fn send_login(&self, request: LoginRequest) -> Result { - let payload = serde_json::to_value(request) - .map_err(|error| ClientError::Protocol(error.to_string()))?; - self.send(LOGIN_API_PATH, payload).await - } - - async fn send( - &self, - api_path: &str, - payload: serde_json::Value, - ) -> Result { - let request_number = self.next_request.fetch_add(1, Ordering::Relaxed); - let request_id = format!("client-{request_number}"); - let envelope = coordinator_wire_request(&request_id, payload); - let body = serde_json::to_vec(&envelope) - .map_err(|error| ClientError::Protocol(error.to_string()))?; - let response = self - .transport - .send(TransportRequest { - api_path: api_path.to_owned(), - body, - }) - .await?; - let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| { - ClientError::Protocol(format!("decode typed API response: {error}")) - })?; - match response { - WireResponse::Error { - code, - category, - message, - retryable, - request_id: response_request_id, - } => { - if response_request_id != request_id { - return Err(ClientError::Protocol(format!( - "error response request_id {response_request_id} does not match {request_id}" - ))); - } - Err(ClientError::Api(ApiError::new( - code, - category, - message, - retryable, - response_request_id, - ))) - } - response => Ok(response), - } - } -} - -pub struct ArtifactDownload { - client: ClusterfluxClient, - artifact: ArtifactId, - max_bytes: u64, - chunk_bytes: u64, - token_digest: Digest, - link: clusterflux_core::DownloadLink, - expected_offset: u64, - finished: bool, -} - -impl ArtifactDownload { - pub fn link(&self) -> &clusterflux_core::DownloadLink { - &self.link - } - - pub async fn next_chunk(&mut self) -> Result { - if self.finished { - return Ok(ArtifactDownloadPoll::Chunk { - offset: self.expected_offset, - bytes: Vec::new(), - eof: true, - }); - } - match self - .client - .send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream { - artifact: self.artifact.clone(), - max_bytes: self.max_bytes, - token_digest: self.token_digest.clone(), - chunk_bytes: self.chunk_bytes, - }) - .await? - { - WireResponse::ArtifactDownloadStream { - content_bytes_available, - content_offset, - content_eof, - content_base64, - .. - } => { - if !content_bytes_available { - return Ok(ArtifactDownloadPoll::Pending); - } - let offset = content_offset.ok_or_else(|| { - ClientError::Protocol( - "artifact response contained bytes without an offset".to_owned(), - ) - })?; - if offset != self.expected_offset { - return Err(ClientError::Protocol(format!( - "artifact chunk offset {offset} does not match expected {}", - self.expected_offset - ))); - } - let bytes = BASE64_STANDARD - .decode(content_base64.unwrap_or_default()) - .map_err(|error| { - ClientError::Protocol(format!( - "artifact response contains invalid base64: {error}" - )) - })?; - self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64); - self.finished = content_eof; - Ok(ArtifactDownloadPoll::Chunk { - offset, - bytes, - eof: content_eof, - }) - } - _ => Err(unexpected_response("artifact_download_stream")), - } - } - - pub async fn cancel(mut self) -> Result<(), ClientError> { - if self.finished { - return Ok(()); - } - match self - .client - .send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink { - artifact: self.artifact.clone(), - token_digest: self.token_digest.clone(), - }) - .await? - { - WireResponse::ArtifactDownloadLinkRevoked {} => { - self.finished = true; - Ok(()) - } - _ => Err(unexpected_response("artifact_download_link_revoked")), - } - } -} - -fn unexpected_response(expected: &str) -> ClientError { - ClientError::Protocol(format!( - "expected typed response {expected}, received another response variant" - )) -} diff --git a/crates/clusterflux-client/src/protocol.rs b/crates/clusterflux-client/src/protocol.rs deleted file mode 100644 index a71ecff..0000000 --- a/crates/clusterflux-client/src/protocol.rs +++ /dev/null @@ -1,403 +0,0 @@ -use std::collections::BTreeMap; - -use clusterflux_core::{ - AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink, - LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId, -}; -use serde::{Deserialize, Serialize}; - -use crate::types::*; - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum AuthenticatedRequest { - AuthStatus, - RevokeCliSession, - CreateProject { - project: ProjectId, - name: String, - }, - SelectProject { - project: ProjectId, - }, - ListProjects, - RegisterAgentPublicKey { - agent: AgentId, - public_key: String, - }, - ListAgentPublicKeys, - RotateAgentPublicKey { - agent: AgentId, - public_key: String, - }, - RevokeAgentPublicKey { - agent: AgentId, - }, - CreateNodeEnrollmentGrant { - ttl_seconds: u64, - }, - ListNodeSummaries { - cursor: Option, - limit: u32, - }, - RevokeNodeCredential { - node: NodeId, - }, - ListProcessSummaries { - cursor: Option, - limit: u32, - }, - CancelProcess { - process: ProcessId, - }, - AbortProcess { - process: ProcessId, - launch_attempt: Option, - }, - QuotaStatus, - ListTaskEvents { - process: Option, - }, - ListTaskSnapshots { - process: ProcessId, - }, - ListRecentLogs { - process: ProcessId, - task: Option, - after_sequence: Option, - limit: u32, - }, - RestartTask { - process: ProcessId, - task: TaskInstanceId, - replacement_bundle: Option, - }, - ResolveTaskFailure { - process: ProcessId, - task: TaskInstanceId, - resolution: TaskFailureResolution, - }, - DebugAttach { - process: ProcessId, - }, - CreateDebugEpoch { - process: ProcessId, - stopped_task: TaskInstanceId, - reason: String, - }, - ResumeDebugEpoch { - process: ProcessId, - epoch: u64, - }, - InspectDebugEpoch { - process: ProcessId, - epoch: u64, - }, - ListArtifacts { - process: Option, - cursor: Option, - limit: u32, - }, - GetArtifact { - artifact: ArtifactId, - }, - CreateArtifactDownloadLink { - artifact: ArtifactId, - max_bytes: u64, - ttl_seconds: u64, - }, - OpenArtifactDownloadStream { - artifact: ArtifactId, - max_bytes: u64, - token_digest: Digest, - chunk_bytes: u64, - }, - RevokeArtifactDownloadLink { - artifact: ArtifactId, - token_digest: Digest, - }, -} - -#[derive(Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum LoginRequest { - BeginWebBrowserLogin {}, - CancelWebBrowserLogin { - transaction_id: String, - }, - ExchangeWebLoginHandoff { - transaction_id: String, - handoff_code: String, - }, -} - -#[derive(Clone, Deserialize)] -pub(crate) struct WireBrowserSession { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub session_secret: String, - pub expires_at_epoch_seconds: u64, -} - -#[allow(clippy::large_enum_variant)] -#[derive(Clone, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub(crate) enum WireResponse { - AuthStatus { - tenant: TenantId, - project: ProjectId, - actor: UserId, - authenticated: bool, - account_status: String, - suspended: bool, - disabled: bool, - deleted: bool, - manual_review: bool, - sanitized_reason: Option, - next_actions: Vec, - }, - CliSessionRevoked {}, - ProjectCreated { - project: Project, - }, - ProjectSelected { - project: Project, - }, - Projects { - projects: Vec, - }, - AgentPublicKey { - record: AgentPublicKey, - }, - AgentPublicKeys { - records: Vec, - }, - NodeEnrollmentGrantCreated { - tenant: TenantId, - project: ProjectId, - grant: String, - scope: String, - expires_at_epoch_seconds: u64, - }, - NodeSummaries { - nodes: Vec, - next_cursor: Option, - }, - NodeCredentialRevoked { - node: NodeId, - tenant: TenantId, - project: ProjectId, - actor: UserId, - descriptor_removed: bool, - queued_assignments_removed: usize, - }, - ProcessSummaries { - processes: Vec, - next_cursor: Option, - }, - ProcessCancellationRequested { - process: ProcessId, - cancelled_tasks: Vec, - affected_nodes: Vec, - }, - ProcessAborted { - process: ProcessId, - aborted_tasks: Vec, - affected_nodes: Vec, - }, - QuotaStatus { - tenant: TenantId, - project: ProjectId, - actor: UserId, - policy_label: Option, - limits: ResourceLimits, - window_seconds: BTreeMap, - usage: BTreeMap, - window_started_epoch_seconds: BTreeMap, - }, - TaskEvents { - events: Vec, - }, - TaskSnapshots { - snapshots: Vec, - }, - RecentLogs { - entries: Vec, - next_sequence: Option, - history_truncated: bool, - }, - TaskRestart { - process: ProcessId, - task: TaskInstanceId, - restarted_task_instance: Option, - restarted_attempt_id: Option, - actor: UserId, - accepted: bool, - clean_boundary_available: bool, - active_task: bool, - completed_event_observed: bool, - requires_whole_process_restart: bool, - message: String, - audit_event: DebugAuditEvent, - }, - TaskFailureResolved { - process: ProcessId, - task: TaskInstanceId, - attempt_id: String, - resolution: TaskFailureResolution, - }, - DebugAttach { - process: ProcessId, - actor: UserId, - authorization: Authorization, - audit_event: DebugAuditEvent, - }, - DebugEpoch { - process: ProcessId, - actor: UserId, - epoch: u64, - command: String, - affected_tasks: Vec, - all_stop_requested: bool, - audit_event: DebugAuditEvent, - }, - DebugEpochStatus { - process: ProcessId, - actor: UserId, - epoch: u64, - command: String, - expected_tasks: Vec, - acknowledgements: Vec, - fully_frozen: bool, - partially_frozen: bool, - fully_resumed: bool, - failed: bool, - failure_messages: Vec, - audit_event: DebugAuditEvent, - }, - Artifacts { - artifacts: Vec, - next_cursor: Option, - }, - Artifact { - artifact: ArtifactSummary, - }, - ArtifactDownloadLink { - link: DownloadLink, - }, - ArtifactDownloadLinkRevoked {}, - ArtifactDownloadStream { - #[serde(rename = "link")] - _link: DownloadLink, - content_bytes_available: bool, - content_offset: Option, - content_eof: bool, - content_base64: Option, - }, - WebBrowserLoginStarted { - transaction_id: String, - authorization_url: String, - expires_at_epoch_seconds: u64, - }, - WebBrowserLoginCancelled {}, - WebBrowserSession { - session: WireBrowserSession, - }, - Error { - code: ApiErrorCode, - category: ApiErrorCategory, - message: String, - retryable: bool, - request_id: String, - }, -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeSet; - - use serde::Deserialize; - use serde_json::Value; - - use super::*; - - #[derive(Deserialize)] - struct OperationFixture { - operation: String, - boundary: String, - request: Value, - response: Value, - } - - #[test] - fn website_operation_contract_fixtures_cover_every_typed_request() { - let fixtures: Vec = - serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap(); - let expected = BTreeSet::from([ - "abort_process", - "auth_status", - "begin_web_browser_login", - "cancel_web_browser_login", - "cancel_process", - "create_artifact_download_link", - "create_debug_epoch", - "create_node_enrollment_grant", - "create_project", - "debug_attach", - "exchange_web_login_handoff", - "get_artifact", - "inspect_debug_epoch", - "list_agent_public_keys", - "list_artifacts", - "list_node_summaries", - "list_process_summaries", - "list_projects", - "list_recent_logs", - "list_task_events", - "list_task_snapshots", - "open_artifact_download_stream", - "quota_status", - "register_agent_public_key", - "resolve_task_failure", - "restart_task", - "resume_debug_epoch", - "revoke_agent_public_key", - "revoke_artifact_download_link", - "revoke_cli_session", - "revoke_node_credential", - "rotate_agent_public_key", - "select_project", - ]) - .into_iter() - .map(str::to_owned) - .collect(); - let mut observed = BTreeSet::new(); - - for fixture in fixtures { - assert_eq!(fixture.request["type"], fixture.operation); - let round_trip = match fixture.boundary.as_str() { - "control" => { - let typed: AuthenticatedRequest = - serde_json::from_value(fixture.request.clone()).unwrap(); - serde_json::to_value(typed).unwrap() - } - "login" => { - let typed: LoginRequest = - serde_json::from_value(fixture.request.clone()).unwrap(); - serde_json::to_value(typed).unwrap() - } - boundary => panic!("unknown fixture boundary {boundary}"), - }; - assert_eq!(round_trip, fixture.request); - let response: WireResponse = serde_json::from_value(fixture.response).unwrap(); - assert!( - !matches!(response, WireResponse::Error { .. }), - "{} must carry its operation-specific success response fixture", - fixture.operation - ); - assert!(observed.insert(fixture.operation)); - } - assert_eq!(observed, expected); - } -} diff --git a/crates/clusterflux-client/src/transport.rs b/crates/clusterflux-client/src/transport.rs deleted file mode 100644 index b480c0f..0000000 --- a/crates/clusterflux-client/src/transport.rs +++ /dev/null @@ -1,250 +0,0 @@ -use std::collections::{BTreeMap, VecDeque}; -use std::future::Future; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use clusterflux_control::{endpoint_identity, ControlSession}; -use thiserror::Error; - -pub type TransportFuture = - Pin> + Send + 'static>>; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransportRequest { - pub api_path: String, - pub body: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransportResponse { - pub body: Vec, -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum ClientTransportError { - #[error("client transport failed: {0}")] - Failed(String), - #[error("client transport task failed: {0}")] - Task(String), -} - -pub trait ClientTransport: Send + Sync + 'static { - fn send(&self, request: TransportRequest) -> TransportFuture; -} - -type ControlSessionPool = Arc>>>; - -pub struct ControlTransport { - endpoint: String, - connect_timeout: Duration, - io_timeout: Duration, - sessions: Arc>>, - next_session: Arc, -} - -// A small pool allows independent HTMX reads to progress concurrently while -// keeping connection and blocking-worker use strictly bounded. -const SESSIONS_PER_API_PATH: usize = 4; - -impl ControlTransport { - pub fn new(endpoint: impl Into) -> Result { - Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30)) - } - - pub fn with_timeouts( - endpoint: impl Into, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result { - let endpoint = endpoint.into(); - endpoint_identity(&endpoint) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?; - Ok(Self { - endpoint, - connect_timeout, - io_timeout, - sessions: Arc::new(Mutex::new(BTreeMap::new())), - next_session: Arc::new(AtomicU64::new(0)), - }) - } -} - -impl ClientTransport for ControlTransport { - fn send(&self, request: TransportRequest) -> TransportFuture { - let endpoint = self.endpoint.clone(); - let connect_timeout = self.connect_timeout; - let io_timeout = self.io_timeout; - let sessions = Arc::clone(&self.sessions); - let next_session = Arc::clone(&self.next_session); - Box::pin(async move { - let pool = { - let mut sessions = sessions.lock().map_err(|_| { - ClientTransportError::Failed( - "client transport pool lock was poisoned".to_owned(), - ) - })?; - Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| { - Arc::new( - (0..SESSIONS_PER_API_PATH) - .map(|_| Mutex::new(None)) - .collect(), - ) - })) - }; - let slot_index = - next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH; - tokio::task::spawn_blocking(move || { - let value = serde_json::from_slice(&request.body) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?; - let mut selected = None; - for offset in 0..SESSIONS_PER_API_PATH { - let index = (slot_index + offset) % SESSIONS_PER_API_PATH; - match pool[index].try_lock() { - Ok(session) if session.is_some() => { - selected = Some(session); - break; - } - Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {} - Err(std::sync::TryLockError::Poisoned(_)) => { - return Err(ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - )); - } - } - } - if selected.is_none() { - for offset in 0..SESSIONS_PER_API_PATH { - let index = (slot_index + offset) % SESSIONS_PER_API_PATH; - match pool[index].try_lock() { - Ok(session) => { - selected = Some(session); - break; - } - Err(std::sync::TryLockError::WouldBlock) => {} - Err(std::sync::TryLockError::Poisoned(_)) => { - return Err(ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - )); - } - } - } - } - let mut session = match selected { - Some(session) => session, - None => pool[slot_index].lock().map_err(|_| { - ClientTransportError::Failed( - "client transport session lock was poisoned".to_owned(), - ) - })?, - }; - if session.is_none() { - *session = Some( - ControlSession::connect_to_api_path_with_timeouts( - &endpoint, - &request.api_path, - connect_timeout, - io_timeout, - ) - .map_err(|error| ClientTransportError::Failed(error.to_string()))?, - ); - } - let response = session - .as_mut() - .expect("session was initialized for the requested API path") - .request(&value); - match response { - Ok(response) => serde_json::to_vec(&response) - .map(|body| TransportResponse { body }) - .map_err(|error| ClientTransportError::Failed(error.to_string())), - Err(error) => { - *session = None; - Err(ClientTransportError::Failed(error.to_string())) - } - } - }) - .await - .map_err(|error| ClientTransportError::Task(error.to_string()))? - }) - } -} - -#[derive(Clone, Default)] -pub struct MockTransport { - state: Arc>, -} - -#[derive(Default)] -struct MockTransportState { - responses: VecDeque, ClientTransportError>>, - requests: Vec, -} - -impl MockTransport { - pub fn from_json_responses(responses: impl IntoIterator>) -> Self { - let responses = responses - .into_iter() - .map(|response| Ok(response.into().into_bytes())) - .collect(); - Self { - state: Arc::new(Mutex::new(MockTransportState { - responses, - requests: Vec::new(), - })), - } - } - - pub fn push_json_response(&self, response: impl Into) { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .responses - .push_back(Ok(response.into().into_bytes())); - } - - pub fn push_error(&self, message: impl Into) { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .responses - .push_back(Err(ClientTransportError::Failed(message.into()))); - } - - pub fn request_bodies(&self) -> Vec { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .requests - .iter() - .map(|request| String::from_utf8_lossy(&request.body).into_owned()) - .collect() - } - - pub fn requests(&self) -> Vec { - self.state - .lock() - .expect("mock transport lock is not poisoned") - .requests - .clone() - } -} - -impl ClientTransport for MockTransport { - fn send(&self, request: TransportRequest) -> TransportFuture { - let state = Arc::clone(&self.state); - Box::pin(async move { - let mut state = state.lock().map_err(|_| { - ClientTransportError::Failed("mock transport lock was poisoned".to_owned()) - })?; - state.requests.push(request); - state - .responses - .pop_front() - .ok_or_else(|| { - ClientTransportError::Failed("mock transport has no queued response".to_owned()) - })? - .map(|body| TransportResponse { body }) - }) - } -} diff --git a/crates/clusterflux-client/src/types.rs b/crates/clusterflux-client/src/types.rs deleted file mode 100644 index eb17e6e..0000000 --- a/crates/clusterflux-client/src/types.rs +++ /dev/null @@ -1,468 +0,0 @@ -use std::collections::BTreeMap; - -use clusterflux_core::{ - AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement, - ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy, - TaskInstanceId, TenantId, UserId, VfsPath, -}; -use serde::{Deserialize, Serialize}; -use std::fmt; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AccountStatus { - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub authenticated: bool, - pub account_status: String, - pub suspended: bool, - pub disabled: bool, - pub deleted: bool, - pub manual_review: bool, - pub sanitized_reason: Option, - pub next_actions: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Project { - pub id: ProjectId, - pub tenant: TenantId, - pub name: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AgentPublicKey { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub agent: AgentId, - pub public_key: String, - pub public_key_fingerprint: Digest, - pub version: u64, - pub revoked: bool, - pub scopes: Vec, - pub human_account_creation_privilege: bool, - pub browser_interaction_required_each_run: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeEnrollmentGrant { - pub tenant: TenantId, - pub project: ProjectId, - pub grant: String, - pub scope: String, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeSummary { - pub id: NodeId, - pub display_name: String, - pub online: bool, - pub stale: bool, - pub last_seen_epoch_seconds: Option, - pub capabilities: NodeCapabilities, - pub direct_connectivity: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodePage { - pub nodes: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeRevocation { - pub node: NodeId, - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub descriptor_removed: bool, - pub queued_assignments_removed: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessLifecycleState { - Active, - RecentTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessActivityState { - Running, - WaitingForNode, - WaitingForTask, - AwaitingAction, - DebugEpochPartial, - Cancelling, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessFinalResult { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochSummary { - pub epoch: u64, - pub command: String, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessSummary { - pub process: ProcessId, - pub lifecycle: ProcessLifecycleState, - pub activity: ProcessActivityState, - pub main_wait_state: Option, - pub started_at_epoch_seconds: u64, - pub ended_at_epoch_seconds: Option, - pub final_result: Option, - pub connected_nodes: Vec, - pub current_debug_epoch: Option, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessPage { - pub processes: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskTerminalState { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskExecutor { - CoordinatorMain, - Node, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskCompletionEvent { - pub tenant: TenantId, - pub project: ProjectId, - pub process: ProcessId, - pub node: NodeId, - pub executor: TaskExecutor, - pub task_definition: TaskDefinitionId, - pub task: TaskInstanceId, - pub attempt_id: Option, - pub placement: Option, - pub terminal_state: TaskTerminalState, - pub status_code: Option, - pub stdout_bytes: u64, - pub stderr_bytes: u64, - pub stdout_tail: String, - pub stderr_tail: String, - pub stdout_truncated: bool, - pub stderr_truncated: bool, - pub artifact_path: Option, - pub artifact_digest: Option, - pub artifact_size_bytes: Option, - pub result: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskAttemptState { - Queued, - Running, - FailedAwaitingAction, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskAttemptSnapshot { - pub process: ProcessId, - pub task: TaskInstanceId, - pub attempt_id: String, - pub attempt_number: u32, - pub task_definition: TaskDefinitionId, - pub display_name: String, - pub state: TaskAttemptState, - pub current: bool, - pub node: Option, - pub environment_id: Option, - pub environment_digest: Option, - pub argument_summary: Vec, - pub handle_summary: Vec, - pub command_state: Option, - pub vfs_checkpoint: String, - pub probe_symbol: Option, - pub source_path: Option, - pub source_line: Option, - pub restart_compatible: bool, - pub failure_policy: TaskFailurePolicy, - pub artifact_path: Option, - pub artifact_digest: Option, - pub artifact_size_bytes: Option, - pub status_code: Option, - pub error: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskLogStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogEntry { - pub sequence: u64, - pub process: ProcessId, - pub task: TaskInstanceId, - pub stream: TaskLogStream, - pub text: String, - pub server_timestamp_epoch_seconds: u64, - pub truncated: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogPage { - pub entries: Vec, - pub next_sequence: Option, - pub history_truncated: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactAvailability { - Available, - NodeOffline, - Unavailable, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactRetentionState { - NodeRetained, - ExplicitStorage, - Lost, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactSummary { - pub id: ArtifactId, - pub display_path: String, - pub display_name: String, - pub process: ProcessId, - pub producer_task: TaskInstanceId, - pub safe_node: Option, - pub digest: Digest, - pub size_bytes: u64, - pub availability: ArtifactAvailability, - pub downloadable_now: bool, - pub retention_state: ArtifactRetentionState, - pub explicit_storage: bool, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactPage { - pub artifacts: Vec, - pub next_cursor: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct QuotaStatus { - pub tenant: TenantId, - pub project: ProjectId, - pub actor: UserId, - pub policy_label: Option, - pub limits: ResourceLimits, - pub window_seconds: BTreeMap, - pub usage: BTreeMap, - pub window_started_epoch_seconds: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskCancellationTarget { - pub process: ProcessId, - pub task: TaskInstanceId, - pub node: NodeId, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessCancellation { - pub process: ProcessId, - pub affected_tasks: Vec, - pub affected_nodes: Vec, - pub aborted: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugAuditEvent { - pub tenant: TenantId, - pub project: ProjectId, - pub process: ProcessId, - pub task: Option, - pub actor: UserId, - pub operation: String, - pub allowed: bool, - pub reason: String, - pub charged_debug_read_bytes: u64, - pub used_debug_read_bytes: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugAttach { - pub process: ProcessId, - pub actor: UserId, - pub authorization: Authorization, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochControl { - pub process: ProcessId, - pub actor: UserId, - pub epoch: u64, - pub command: String, - pub affected_tasks: Vec, - pub all_stop_requested: bool, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DebugAcknowledgementState { - Frozen, - Running, - Failed, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugParticipantAcknowledgement { - pub node: NodeId, - pub task_definition: TaskDefinitionId, - pub task: TaskInstanceId, - pub epoch: u64, - pub state: DebugAcknowledgementState, - pub stack_frames: Vec, - pub local_values: Vec<(String, String)>, - pub task_args: Vec<(String, String)>, - pub handles: Vec<(String, String)>, - pub command_status: Option, - pub recent_output: Vec, - pub message: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochStatus { - pub process: ProcessId, - pub actor: UserId, - pub epoch: u64, - pub command: String, - pub expected_tasks: Vec, - pub acknowledgements: Vec, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, - pub failure_messages: Vec, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskRestart { - pub process: ProcessId, - pub task: TaskInstanceId, - pub restarted_task_instance: Option, - pub restarted_attempt_id: Option, - pub actor: UserId, - pub accepted: bool, - pub clean_boundary_available: bool, - pub active_task: bool, - pub completed_event_observed: bool, - pub requires_whole_process_restart: bool, - pub message: String, - pub audit_event: DebugAuditEvent, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskFailureResolutionResult { - pub process: ProcessId, - pub task: TaskInstanceId, - pub attempt_id: String, - pub resolution: TaskFailureResolution, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskFailureResolution { - AcceptFailure, - Cancel, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct TaskReplacementBundle { - pub bundle_digest: Digest, - pub wasm_module_base64: String, - pub source_snapshot: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct BrowserLoginStart { - pub transaction_id: String, - pub authorization_url: String, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, PartialEq, Eq)] -pub struct SessionCredential(pub(crate) String); - -impl SessionCredential { - pub fn from_secret(secret: impl Into) -> Self { - Self(secret.into()) - } - - pub fn expose_secret(&self) -> &str { - &self.0 - } -} - -impl fmt::Debug for SessionCredential { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("SessionCredential([REDACTED])") - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct BrowserSession { - pub tenant: TenantId, - pub project: ProjectId, - pub user: UserId, - pub credential: SessionCredential, - pub expires_at_epoch_seconds: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ArtifactDownloadPoll { - Pending, - Chunk { - offset: u64, - bytes: Vec, - eof: bool, - }, -} diff --git a/crates/clusterflux-client/tests/client_contract.rs b/crates/clusterflux-client/tests/client_contract.rs deleted file mode 100644 index 8851a54..0000000 --- a/crates/clusterflux-client/tests/client_contract.rs +++ /dev/null @@ -1,254 +0,0 @@ -use std::net::TcpListener; -use std::time::Duration; - -use clusterflux_client::{ - ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError, - ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId, - UserId, CLIENT_API_VERSION, -}; -use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH}; -use clusterflux_coordinator::service::CoordinatorService; -use serde_json::{json, Value}; - -#[tokio::test] -async fn mock_transport_exercises_typed_envelope_and_session_plumbing() { - let transport = MockTransport::from_json_responses([json!({ - "type": "projects", - "projects": [{ - "id": "project-one", - "tenant": "tenant-one", - "name": "Project one" - }], - "actor": "user-one" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport.clone()) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let projects = client.list_projects().await.unwrap(); - assert_eq!(projects.len(), 1); - assert_eq!(projects[0].id, ProjectId::from("project-one")); - - let requests = transport.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].api_path, CONTROL_API_PATH); - let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION); - assert_eq!(envelope["request_id"], "client-1"); - assert_eq!(envelope["operation"], "authenticated"); - assert_eq!(envelope["payload"]["type"], "authenticated"); - assert_eq!(envelope["payload"]["request"]["type"], "list_projects"); - assert_eq!(envelope["payload"]["session_secret"], "test-session-secret"); -} - -#[tokio::test] -async fn structured_errors_retain_machine_fields_and_originating_request_id() { - let transport = MockTransport::from_json_responses([json!({ - "type": "error", - "code": "account_suspended", - "category": "authorization", - "message": "account access is suspended", - "retryable": false, - "request_id": "client-1" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let ClientError::Api(error) = client.account_status().await.unwrap_err() else { - panic!("expected typed API error"); - }; - assert_eq!(error.code, ApiErrorCode::AccountSuspended); - assert_eq!(error.category, ApiErrorCategory::Authorization); - assert_eq!(error.request_id, "client-1"); - assert!(!error.retryable); -} - -#[tokio::test] -async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() { - let transport = MockTransport::from_json_responses([ - json!({ "type": "web_browser_login_cancelled" }).to_string(), - ]); - let client = ClusterfluxClient::with_transport(transport.clone()); - - client - .cancel_browser_login("login-transaction") - .await - .unwrap(); - - let requests = transport.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].api_path, LOGIN_API_PATH); - let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert_eq!(envelope["operation"], "cancel_web_browser_login"); - assert_eq!( - envelope["payload"], - json!({ - "type": "cancel_web_browser_login", - "transaction_id": "login-transaction" - }) - ); -} - -#[tokio::test] -async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() { - let transport = MockTransport::from_json_responses([json!({ - "type": "error", - "code": "validation_error", - "category": "validation", - "message": "bad request", - "retryable": false, - "request_id": "another-request" - }) - .to_string()]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - - let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else { - panic!("expected protocol error"); - }; - assert!(message.contains("does not match client-1")); -} - -#[test] -fn session_credentials_are_redacted_from_debug_output() { - let credential = SessionCredential::from_secret("must-not-appear"); - assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])"); -} - -#[tokio::test] -async fn artifact_download_is_a_typed_bounded_chunk_stream() { - let link = json!({ - "artifact": "artifact-one", - "artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "artifact_size_bytes": 5, - "source": { "RetainedNode": "node-one" }, - "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", - "scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "expires_at_epoch_seconds": 1000, - "tenant": "tenant-one", - "project": "project-one", - "process": "process-one", - "actor": { "User": "user-one" }, - "max_bytes": 1024, - "policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - }); - let transport = MockTransport::from_json_responses([ - json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(), - json!({ - "type": "artifact_download_stream", - "link": link.clone(), - "streamed_bytes": 0, - "charged_download_bytes": 0, - "content_bytes_available": false, - "content_eof": false - }) - .to_string(), - json!({ - "type": "artifact_download_stream", - "link": link, - "streamed_bytes": 5, - "charged_download_bytes": 5, - "content_bytes_available": true, - "content_offset": 0, - "content_eof": true, - "content_base64": "aGVsbG8=" - }) - .to_string(), - ]); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("test-session-secret")); - let mut download = client - .begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64) - .await - .unwrap(); - assert_eq!( - download.next_chunk().await.unwrap(), - ArtifactDownloadPoll::Pending - ); - assert_eq!( - download.next_chunk().await.unwrap(), - ArtifactDownloadPoll::Chunk { - offset: 0, - bytes: b"hello".to_vec(), - eof: true, - } - ); -} - -#[tokio::test] -async fn typed_client_runs_against_the_real_strict_control_endpoint() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant-one"), - ProjectId::from("project-one"), - UserId::from("user-one"), - "real-endpoint-session", - None, - ) - .unwrap(); - let (stream, _) = listener.accept().unwrap(); - service.handle_stream(stream).unwrap(); - }); - - let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}")) - .unwrap() - .with_session_credential(&SessionCredential::from_secret("real-endpoint-session")); - let projects = client.list_projects().await.unwrap(); - assert_eq!(projects.len(), 1); - assert_eq!(projects[0].id, ProjectId::from("project-one")); - let status = client.account_status().await.unwrap(); - assert!(status.authenticated); - assert_eq!(status.actor, UserId::from("user-one")); - - drop(client); - server.join().unwrap(); -} - -#[tokio::test] -async fn concurrent_requests_expand_the_bounded_pool_without_serializing() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let handlers = (0..2) - .map(|_| { - let (stream, _) = listener.accept().unwrap(); - std::thread::spawn(move || { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant-one"), - ProjectId::from("project-one"), - UserId::from("user-one"), - "concurrent-session", - None, - ) - .unwrap(); - service.handle_stream(stream).unwrap(); - }) - }) - .collect::>(); - for handler in handlers { - handler.join().unwrap(); - } - }); - - let transport = ControlTransport::with_timeouts( - format!("clusterflux+tcp://{address}"), - Duration::from_secs(2), - Duration::from_secs(5), - ) - .unwrap(); - let client = ClusterfluxClient::with_transport(transport) - .with_session_credential(&SessionCredential::from_secret("concurrent-session")); - let (first, second) = tokio::join!(client.account_status(), client.account_status()); - assert!(first.unwrap().authenticated); - assert!(second.unwrap().authenticated); - - drop(client); - server.join().unwrap(); -} diff --git a/crates/clusterflux-client/tests/fixtures/web_operations.json b/crates/clusterflux-client/tests/fixtures/web_operations.json deleted file mode 100644 index 45b7b23..0000000 --- a/crates/clusterflux-client/tests/fixtures/web_operations.json +++ /dev/null @@ -1,200 +0,0 @@ -[ - { - "operation": "begin_web_browser_login", - "boundary": "login", - "request": { "type": "begin_web_browser_login" }, - "response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 } - }, - { - "operation": "cancel_web_browser_login", - "boundary": "login", - "request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" }, - "response": { "type": "web_browser_login_cancelled" } - }, - { - "operation": "exchange_web_login_handoff", - "boundary": "login", - "request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" }, - "response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } } - }, - { - "operation": "auth_status", - "boundary": "control", - "request": { "type": "auth_status" }, - "response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] } - }, - { - "operation": "revoke_cli_session", - "boundary": "control", - "request": { "type": "revoke_cli_session" }, - "response": { "type": "cli_session_revoked" } - }, - { - "operation": "create_project", - "boundary": "control", - "request": { "type": "create_project", "project": "project-new", "name": "New project" }, - "response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } } - }, - { - "operation": "select_project", - "boundary": "control", - "request": { "type": "select_project", "project": "project-selected" }, - "response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } } - }, - { - "operation": "list_projects", - "boundary": "control", - "request": { "type": "list_projects" }, - "response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] } - }, - { - "operation": "register_agent_public_key", - "boundary": "control", - "request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "list_agent_public_keys", - "boundary": "control", - "request": { "type": "list_agent_public_keys" }, - "response": { "type": "agent_public_keys", "records": [] } - }, - { - "operation": "rotate_agent_public_key", - "boundary": "control", - "request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "revoke_agent_public_key", - "boundary": "control", - "request": { "type": "revoke_agent_public_key", "agent": "agent-browser" }, - "response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } } - }, - { - "operation": "create_node_enrollment_grant", - "boundary": "control", - "request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 }, - "response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 } - }, - { - "operation": "list_node_summaries", - "boundary": "control", - "request": { "type": "list_node_summaries", "cursor": null, "limit": 200 }, - "response": { "type": "node_summaries", "nodes": [], "next_cursor": null } - }, - { - "operation": "revoke_node_credential", - "boundary": "control", - "request": { "type": "revoke_node_credential", "node": "node-one" }, - "response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 } - }, - { - "operation": "list_process_summaries", - "boundary": "control", - "request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 }, - "response": { "type": "process_summaries", "processes": [], "next_cursor": null } - }, - { - "operation": "cancel_process", - "boundary": "control", - "request": { "type": "cancel_process", "process": "process-one" }, - "response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] } - }, - { - "operation": "abort_process", - "boundary": "control", - "request": { "type": "abort_process", "process": "process-one", "launch_attempt": null }, - "response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] } - }, - { - "operation": "quota_status", - "boundary": "control", - "request": { "type": "quota_status" }, - "response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } } - }, - { - "operation": "list_task_events", - "boundary": "control", - "request": { "type": "list_task_events", "process": "process-one" }, - "response": { "type": "task_events", "events": [] } - }, - { - "operation": "list_task_snapshots", - "boundary": "control", - "request": { "type": "list_task_snapshots", "process": "process-one" }, - "response": { "type": "task_snapshots", "snapshots": [] } - }, - { - "operation": "list_recent_logs", - "boundary": "control", - "request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 }, - "response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false } - }, - { - "operation": "restart_task", - "boundary": "control", - "request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null }, - "response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "resolve_task_failure", - "boundary": "control", - "request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" }, - "response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" } - }, - { - "operation": "debug_attach", - "boundary": "control", - "request": { "type": "debug_attach", "process": "process-one" }, - "response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "create_debug_epoch", - "boundary": "control", - "request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" }, - "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "resume_debug_epoch", - "boundary": "control", - "request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 }, - "response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "inspect_debug_epoch", - "boundary": "control", - "request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 }, - "response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } } - }, - { - "operation": "list_artifacts", - "boundary": "control", - "request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 }, - "response": { "type": "artifacts", "artifacts": [], "next_cursor": null } - }, - { - "operation": "get_artifact", - "boundary": "control", - "request": { "type": "get_artifact", "artifact": "artifact-one" }, - "response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } } - }, - { - "operation": "create_artifact_download_link", - "boundary": "control", - "request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 }, - "response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } } - }, - { - "operation": "open_artifact_download_stream", - "boundary": "control", - "request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 }, - "response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" } - }, - { - "operation": "revoke_artifact_download_link", - "boundary": "control", - "request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, - "response": { "type": "artifact_download_link_revoked" } - } -] diff --git a/crates/clusterflux-control/src/lib.rs b/crates/clusterflux-control/src/lib.rs index 39a150d..a3b0800 100644 --- a/crates/clusterflux-control/src/lib.rs +++ b/crates/clusterflux-control/src/lib.rs @@ -3,8 +3,6 @@ use std::io::{BufRead, BufReader, Read, Write}; #[cfg(not(target_arch = "wasm32"))] use std::net::TcpStream; use std::net::{IpAddr, ToSocketAddrs}; -#[cfg(not(target_arch = "wasm32"))] -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::Value; @@ -61,34 +59,12 @@ impl ControlSession { endpoint: &str, api_path: &str, ) -> Result { - Self::connect_to_api_path_with_timeouts( - endpoint, - api_path, - Duration::from_secs(10), - Duration::from_secs(30), - ) - } - - pub fn connect_to_api_path_with_timeouts( - endpoint: &str, - api_path: &str, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result { - let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?; + let mut session = Self::connect(endpoint)?; #[cfg(not(target_arch = "wasm32"))] - { - let mut session = session; - if let ControlTransport::Https { url, .. } = &mut session.transport { - *url = endpoint_api_url(endpoint, api_path)?; - } - Ok(session) - } - #[cfg(target_arch = "wasm32")] - { - let _ = api_path; - Ok(session) + if let ControlTransport::Https { url, .. } = &mut session.transport { + *url = endpoint_api_url(endpoint, api_path)?; } + Ok(session) } pub fn connect_with_timeouts( @@ -151,7 +127,6 @@ impl ControlSession { if encoded.len() > MAX_CONTROL_FRAME_BYTES { return Err(ControlTransportError::FrameTooLarge); } - let inject_response_loss = should_inject_response_loss(value); let response = match &mut self.transport { #[cfg(not(target_arch = "wasm32"))] ControlTransport::Https { agent, url } => { @@ -161,9 +136,6 @@ impl ControlSession { .set("Accept", "application/json") .send_bytes(&encoded) .map_err(|error| ControlTransportError::Http(error.to_string()))?; - if inject_response_loss { - return Err(ControlTransportError::Closed); - } if response.status() != 200 { return Err(ControlTransportError::Http(format!( "coordinator returned HTTP {} {}", @@ -185,9 +157,6 @@ impl ControlSession { writer.write_all(&encoded)?; writer.write_all(b"\n")?; writer.flush()?; - if inject_response_loss { - return Err(ControlTransportError::Closed); - } let mut bytes = Vec::new(); reader .take((MAX_CONTROL_FRAME_BYTES + 1) as u64) @@ -211,24 +180,6 @@ impl ControlSession { } } -#[cfg(not(target_arch = "wasm32"))] -fn should_inject_response_loss(value: &Value) -> bool { - static INJECTED: AtomicBool = AtomicBool::new(false); - let Ok(expected_operation) = std::env::var("CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION") - else { - return false; - }; - let operation = value - .pointer("/payload/request/type") - .or_else(|| value.pointer("/payload/type")) - .or_else(|| value.get("type")) - .and_then(Value::as_str); - operation == Some(expected_operation.as_str()) - && INJECTED - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_ok() -} - pub fn control_api_url(endpoint: &str) -> Result { endpoint_api_url(endpoint, CONTROL_API_PATH) } diff --git a/crates/clusterflux-coordinator/src/durable.rs b/crates/clusterflux-coordinator/src/durable.rs index 0cc8a66..cd3e3e5 100644 --- a/crates/clusterflux-coordinator/src/durable.rs +++ b/crates/clusterflux-coordinator/src/durable.rs @@ -33,39 +33,6 @@ pub struct NodeIdentityRecord { pub enrollment_scope: String, } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub struct NodeScopeKey { - pub tenant: TenantId, - pub project: ProjectId, - pub node: NodeId, -} - -impl NodeScopeKey { - pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self { - Self { - tenant, - project, - node, - } - } - - pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self { - Self::new(tenant.clone(), project.clone(), node.clone()) - } - - pub fn credential_subject(&self) -> String { - format!( - "node:{}:{}:{}:{}:{}:{}", - self.tenant.as_str().len(), - self.tenant, - self.project.as_str().len(), - self.project, - self.node.as_str().len(), - self.node - ) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CredentialRecord { pub subject: String, @@ -140,7 +107,7 @@ pub struct DurableState { pub tenants: BTreeMap, pub users: BTreeMap, pub projects: BTreeMap, - pub node_identities: BTreeMap, + pub node_identities: BTreeMap, pub credentials: BTreeMap, pub cli_sessions: BTreeMap, pub source_provider_configs: diff --git a/crates/clusterflux-coordinator/src/lib.rs b/crates/clusterflux-coordinator/src/lib.rs index a110834..bf3f376 100644 --- a/crates/clusterflux-coordinator/src/lib.rs +++ b/crates/clusterflux-coordinator/src/lib.rs @@ -13,7 +13,7 @@ pub mod service; mod sessions; pub use durable::{ AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState, - DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey, + DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; @@ -33,7 +33,6 @@ pub use service::{ #[derive(Clone, Debug, PartialEq, Eq)] pub struct ActiveProcess { pub id: ProcessId, - pub launch_attempt: Option, pub tenant: TenantId, pub project: ProjectId, pub connected_nodes: BTreeSet, @@ -128,9 +127,8 @@ impl Coordinator { public_key: impl Into, enrollment_scope: impl Into, ) { - let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone()); self.durable.node_identities.insert( - key, + node.clone(), NodeIdentityRecord { id: node, tenant, @@ -182,13 +180,10 @@ impl Coordinator { public_key, credential.scope.clone(), ); - let subject = - NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node) - .credential_subject(); self.durable.credentials.insert( - subject.clone(), + format!("node:{node}"), CredentialRecord { - subject, + subject: format!("node:{node}"), tenant: credential.tenant.clone(), project: Some(credential.project.clone()), kind: credential.credential_kind.clone(), @@ -327,20 +322,9 @@ impl Coordinator { tenant: TenantId, project: ProjectId, id: ProcessId, - ) -> ActiveProcess { - self.start_process_for_launch_attempt(tenant, project, id, None) - } - - pub fn start_process_for_launch_attempt( - &mut self, - tenant: TenantId, - project: ProjectId, - id: ProcessId, - launch_attempt: Option, ) -> ActiveProcess { let process = ActiveProcess { id: id.clone(), - launch_attempt, tenant, project, connected_nodes: BTreeSet::new(), @@ -360,12 +344,15 @@ impl Coordinator { project: &ProjectId, process: &ProcessId, ) -> Result<(), CoordinatorError> { - if !self + let identity = self .durable .node_identities - .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) - { - return Err(CoordinatorError::UnknownNode); + .get(node) + .ok_or(CoordinatorError::UnknownNode)?; + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node identity is outside the requested tenant/project scope".to_owned(), + )); } if !self .active_processes @@ -380,18 +367,14 @@ impl Coordinator { pub fn reconnect_node( &mut self, - tenant: &TenantId, - project: &ProjectId, node: &NodeId, process: Option<(&ProcessId, u64)>, ) -> Result<(), CoordinatorError> { - if !self + let identity = self .durable .node_identities - .contains_key(&NodeScopeKey::from_refs(tenant, project, node)) - { - return Err(CoordinatorError::UnknownNode); - } + .get(node) + .ok_or(CoordinatorError::UnknownNode)?; if let Some((process_id, stale_epoch)) = process { if stale_epoch != self.coordinator_epoch { @@ -400,7 +383,11 @@ impl Coordinator { current_epoch: self.coordinator_epoch, }); } - let key = (tenant.clone(), project.clone(), process_id.clone()); + let key = ( + identity.tenant.clone(), + identity.project.clone(), + process_id.clone(), + ); if let Some(active) = self.active_processes.get_mut(&key) { active.connected_nodes.insert(node.clone()); } @@ -417,27 +404,23 @@ impl Coordinator { let identity = self .durable .node_identities - .get(&NodeScopeKey::from_refs( - &context.tenant, - &context.project, - node, - )) + .get(node) .ok_or(CoordinatorError::UnknownNode)? .clone(); + if identity.tenant != context.tenant || identity.project != context.project { + return Err(CoordinatorError::Unauthorized( + "node credential is outside the signed-in tenant/project scope".to_owned(), + )); + } if !matches!(context.actor, Actor::User(_)) { return Err(CoordinatorError::Unauthorized( "node credential revocation requires a user identity".to_owned(), )); } - let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node); - self.durable.node_identities.remove(&key); - self.durable.credentials.remove(&key.credential_subject()); - for active in self - .active_processes - .values_mut() - .filter(|active| active.tenant == context.tenant && active.project == context.project) - { - active.connected_nodes.remove(&key.node); + self.durable.node_identities.remove(node); + self.durable.credentials.remove(&format!("node:{node}")); + for active in self.active_processes.values_mut() { + active.connected_nodes.remove(node); } Ok(identity) } @@ -548,32 +531,6 @@ impl Coordinator { .expect("active process was checked immediately before removal")) } - pub fn abort_process_for_launch_attempt( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - launch_attempt: &clusterflux_core::LaunchAttemptId, - ) -> Result { - let key = (tenant.clone(), project.clone(), process.clone()); - let active = self.active_processes.get(&key).ok_or_else(|| { - CoordinatorError::Unauthorized( - "launch rollback requires an active virtual process".to_owned(), - ) - })?; - if active.launch_attempt.as_ref() != Some(launch_attempt) { - return Err(CoordinatorError::Unauthorized(format!( - "launch rollback denied: attempt {} does not own process {}", - launch_attempt.as_str(), - process.as_str() - ))); - } - Ok(self - .active_processes - .remove(&key) - .expect("active process was checked immediately before removal")) - } - pub fn active_process_count(&self) -> usize { self.active_processes.len() } @@ -585,31 +542,8 @@ impl Coordinator { .count() } - pub fn tenant_count(&self) -> usize { - self.durable.tenants.len() - } - - pub fn user_count(&self) -> usize { - self.durable.users.len() - } - - pub fn project_count(&self) -> usize { - self.durable.projects.len() - } - - pub fn node_identity_count(&self) -> usize { - self.durable.node_identities.len() - } - - pub fn node_identity( - &self, - tenant: &TenantId, - project: &ProjectId, - id: &NodeId, - ) -> Option<&NodeIdentityRecord> { - self.durable - .node_identities - .get(&NodeScopeKey::from_refs(tenant, project, id)) + pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> { + self.durable.node_identities.get(id) } pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize { @@ -711,24 +645,12 @@ mod tests { .contains_key(&TenantId::from("tenant"))); assert!(restarted.durable.users.contains_key(&UserId::from("user"))); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); - let node_subject = NodeScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - NodeId::from("node"), - ) - .credential_subject(); + assert!(restarted.node_identity(&NodeId::from("node")).is_some()); assert_eq!( restarted .durable .credentials - .get(&node_subject) + .get("node:node") .map(|credential| &credential.kind), Some(&CredentialKind::NodeCredential) ); @@ -752,12 +674,7 @@ mod tests { ); assert_eq!(rerun.coordinator_epoch, 2); restarted - .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - Some((&process, 2)), - ) + .reconnect_node(&NodeId::from("node"), Some((&process, 2))) .unwrap(); assert!(restarted .active_process( @@ -770,68 +687,6 @@ mod tests { .contains(&NodeId::from("node"))); } - #[test] - fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() { - let store = InMemoryDurableStore::default(); - let mut coordinator = Coordinator::boot(&store, 1); - let node = NodeId::from("shared-node"); - let scopes = [ - (TenantId::from("tenant-a"), ProjectId::from("project-a")), - (TenantId::from("tenant-b"), ProjectId::from("project-b")), - (TenantId::from("tenant-a"), ProjectId::from("project-c")), - ]; - for (index, (tenant, project)) in scopes.iter().enumerate() { - let mut grant = coordinator.create_node_enrollment_grant( - tenant.clone(), - project.clone(), - format!("grant-{index}"), - "node:attach", - 100, - ); - coordinator - .exchange_node_enrollment_grant( - &mut grant, - node.clone(), - &format!("public-key-{index}"), - "node:attach", - 99, - ) - .unwrap(); - } - - let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node); - let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node); - let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node); - assert_ne!(scope_a.credential_subject(), scope_b.credential_subject()); - assert_ne!(scope_a.credential_subject(), scope_c.credential_subject()); - assert_eq!( - coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node), - coordinator.durable.node_identities.get(&scope_a) - ); - assert_eq!( - coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node), - coordinator.durable.node_identities.get(&scope_b) - ); - assert_eq!( - coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node), - coordinator.durable.node_identities.get(&scope_c) - ); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_a.credential_subject())); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_b.credential_subject())); - assert!(coordinator - .durable - .credentials - .contains_key(&scope_c.credential_subject())); - assert_eq!(coordinator.durable.node_identities.len(), 3); - assert_eq!(coordinator.durable.credentials.len(), 3); - } - #[test] fn identical_process_ids_are_isolated_by_tenant_and_project() { let store = InMemoryDurableStore::default(); @@ -880,18 +735,11 @@ mod tests { let mut restarted = Coordinator::boot(&store, 2); restarted - .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - None, - ) + .reconnect_node(&NodeId::from("node"), None) .unwrap(); let error = restarted .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -923,13 +771,7 @@ mod tests { .unwrap(); assert_eq!(credential.credential_kind, CredentialKind::NodeCredential); - assert!(coordinator - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); + assert!(coordinator.node_identity(&NodeId::from("node")).is_some()); } #[test] @@ -943,16 +785,10 @@ mod tests { "public-key", "node:attach", ); - let node_subject = NodeScopeKey::new( - TenantId::from("tenant"), - ProjectId::from("project"), - NodeId::from("node"), - ) - .credential_subject(); coordinator.durable.credentials.insert( - node_subject.clone(), + "node:node".to_owned(), CredentialRecord { - subject: node_subject.clone(), + subject: "node:node".to_owned(), tenant: TenantId::from("tenant"), project: Some(ProjectId::from("project")), kind: CredentialKind::NodeCredential, @@ -966,8 +802,6 @@ mod tests { ); coordinator .reconnect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), &NodeId::from("node"), Some((&ProcessId::from("process"), 1)), ) @@ -983,7 +817,7 @@ mod tests { &NodeId::from("node"), ) .unwrap_err(); - assert!(matches!(foreign, CoordinatorError::UnknownNode)); + assert!(matches!(foreign, CoordinatorError::Unauthorized(_))); let revoked = coordinator .revoke_node_credential( @@ -996,14 +830,8 @@ mod tests { ) .unwrap(); assert_eq!(revoked.id, NodeId::from("node")); - assert!(coordinator - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_none()); - assert!(!coordinator.durable.credentials.contains_key(&node_subject)); + assert!(coordinator.node_identity(&NodeId::from("node")).is_none()); + assert!(!coordinator.durable.credentials.contains_key("node:node")); assert!(!coordinator .active_process( &TenantId::from("tenant"), @@ -1062,7 +890,7 @@ mod tests { } #[test] - fn account_policy_state_summarizes_sensitive_admin_records_safely() { + fn account_policy_state_summarizes_private_admin_records_safely() { let tenant = TenantId::from("tenant"); let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1); @@ -1218,7 +1046,7 @@ mod tests { ) .unwrap_err(); - assert!(matches!(error, CoordinatorError::UnknownNode)); + assert!(matches!(error, CoordinatorError::Unauthorized(_))); } #[test] diff --git a/crates/clusterflux-coordinator/src/main.rs b/crates/clusterflux-coordinator/src/main.rs index 8344f5f..8c9ce7d 100644 --- a/crates/clusterflux-coordinator/src/main.rs +++ b/crates/clusterflux-coordinator/src/main.rs @@ -5,40 +5,17 @@ use clusterflux_core::{ProjectId, TenantId, UserId}; use serde_json::json; fn main() -> Result<(), Box> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux coordinator.\n\n\ - Usage: clusterflux-coordinator [OPTIONS]\n\n\ - Options:\n \ - --listen
[default: 127.0.0.1:0]\n \ - --allow-local-trusted-loopback\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - _ => {} - } - let mut listen = "127.0.0.1:0".to_owned(); let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK") .ok() .as_deref() == Some("1"); - let mut args = raw_args.into_iter(); + let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { if arg == "--listen" { listen = args.next().ok_or("--listen requires an address")?; } else if arg == "--allow-local-trusted-loopback" { allow_local_trusted = true; - } else { - return Err(format!("unknown argument: {arg}").into()); } } diff --git a/crates/clusterflux-coordinator/src/postgres_store.rs b/crates/clusterflux-coordinator/src/postgres_store.rs index 18d5206..4ddfd02 100644 --- a/crates/clusterflux-coordinator/src/postgres_store.rs +++ b/crates/clusterflux-coordinator/src/postgres_store.rs @@ -4,8 +4,8 @@ use thiserror::Error; use crate::{ AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord, - DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord, - ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, + DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, + ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord, }; #[derive(Clone, Debug, PartialEq, Eq)] @@ -154,16 +154,9 @@ impl FallibleDurableStore for PostgresDurableStore { state.projects.insert(record.id.clone(), record); } for record in self.query_records::( - "SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id", + "SELECT record FROM clusterflux_node_identities ORDER BY node_id", )? { - state.node_identities.insert( - NodeScopeKey::new( - record.tenant.clone(), - record.project.clone(), - record.id.clone(), - ), - record, - ); + state.node_identities.insert(record.id.clone(), record); } for record in self.query_records::( "SELECT record FROM clusterflux_credentials ORDER BY subject", @@ -383,59 +376,12 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects ( ); CREATE TABLE IF NOT EXISTS clusterflux_node_identities ( + node_id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE, - node_id TEXT NOT NULL, - record JSONB NOT NULL, - PRIMARY KEY (tenant_id, project_id, node_id) + record JSONB NOT NULL ); -DO $clusterflux_node_scope_migration$ -DECLARE - current_primary_key TEXT; - current_primary_key_columns TEXT[]; -BEGIN - IF EXISTS ( - SELECT 1 - FROM clusterflux_node_identities - WHERE record->>'id' IS DISTINCT FROM node_id - OR record->>'tenant' IS DISTINCT FROM tenant_id - OR record->>'project' IS DISTINCT FROM project_id - ) THEN - RAISE EXCEPTION - 'clusterflux node identity migration refused an internally inconsistent legacy row'; - END IF; - - SELECT - constraint_record.conname, - array_agg(attribute.attname ORDER BY key_column.ordinality) - INTO current_primary_key, current_primary_key_columns - FROM pg_constraint AS constraint_record - CROSS JOIN LATERAL unnest(constraint_record.conkey) - WITH ORDINALITY AS key_column(attribute_number, ordinality) - JOIN pg_attribute AS attribute - ON attribute.attrelid = constraint_record.conrelid - AND attribute.attnum = key_column.attribute_number - WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass - AND constraint_record.contype = 'p' - GROUP BY constraint_record.conname; - - IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN - EXECUTE format( - 'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I', - current_primary_key - ); - ALTER TABLE clusterflux_node_identities - ADD PRIMARY KEY (tenant_id, project_id, node_id); - ELSIF current_primary_key_columns - IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[] - THEN - RAISE EXCEPTION - 'clusterflux node identity migration found an unexpected primary key shape'; - END IF; -END -$clusterflux_node_scope_migration$; - CREATE TABLE IF NOT EXISTS clusterflux_credentials ( subject TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -443,74 +389,6 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials ( record JSONB NOT NULL ); -DO $clusterflux_node_credential_migration$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM clusterflux_credentials - WHERE record->>'subject' IS DISTINCT FROM subject - OR record->>'tenant' IS DISTINCT FROM tenant_id - OR record->>'project' IS DISTINCT FROM project_id - ) THEN - RAISE EXCEPTION - 'clusterflux node credential migration refused an internally inconsistent legacy row'; - END IF; - - UPDATE clusterflux_credentials AS credential - SET subject = format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - ), - record = jsonb_set( - credential.record, - '{subject}', - to_jsonb(format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - )), - false - ) - FROM clusterflux_node_identities AS identity - WHERE credential.subject = 'node:' || identity.node_id - AND credential.tenant_id = identity.tenant_id - AND credential.project_id = identity.project_id; - - IF EXISTS ( - SELECT 1 - FROM clusterflux_credentials AS credential - WHERE credential.subject LIKE 'node:%' - AND NOT EXISTS ( - SELECT 1 - FROM clusterflux_node_identities AS identity - WHERE credential.tenant_id = identity.tenant_id - AND credential.project_id = identity.project_id - AND credential.subject = format( - 'node:%s:%s:%s:%s:%s:%s', - octet_length(identity.tenant_id), - identity.tenant_id, - octet_length(identity.project_id), - identity.project_id, - octet_length(identity.node_id), - identity.node_id - ) - ) - ) THEN - RAISE EXCEPTION - 'clusterflux node credential migration found an unscoped or orphaned node subject'; - END IF; -END -$clusterflux_node_credential_migration$; - CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions ( session_digest TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE, @@ -651,13 +529,7 @@ mod tests { let restarted = Coordinator::try_boot(&mut store, 2).unwrap(); assert!(restarted.project(&ProjectId::from("project")).is_some()); - assert!(restarted - .node_identity( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ) - .is_some()); + assert!(restarted.node_identity(&NodeId::from("node")).is_some()); assert_eq!(restarted.active_process_count(), 0); } diff --git a/crates/clusterflux-coordinator/src/service.rs b/crates/clusterflux-coordinator/src/service.rs index bab6c13..abef88d 100644 --- a/crates/clusterflux-coordinator/src/service.rs +++ b/crates/clusterflux-coordinator/src/service.rs @@ -7,14 +7,13 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::time::{SystemTime, UNIX_EPOCH}; use clusterflux_core::{ - Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry, - CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport, - NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit, - TenantId, TransportError, UserId, + Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError, + NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId, + RateLimit, TenantId, TransportError, UserId, }; use thiserror::Error; -use crate::{Coordinator, CoordinatorError, NodeScopeKey}; +use crate::{Coordinator, CoordinatorError}; mod admin; mod artifacts; @@ -35,7 +34,6 @@ mod quota; mod relay; mod routing; mod signed_nodes; -mod summaries; mod tcp; mod wire_protocol; use authorization::authorize_authenticated_user_operation; @@ -45,14 +43,12 @@ use keys::{ ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey, }; pub use protocol::{ - ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment, - AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, - DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement, - NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, - RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment, - TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent, - TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState, - VirtualProcessStatus, WorkflowActor, + ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest, + CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent, + DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, + TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, + TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle, + TaskTerminalState, VirtualProcessStatus, WorkflowActor, }; pub use quota::CoordinatorQuotaConfiguration; pub use relay::{ @@ -73,15 +69,9 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128; const MAX_TASK_EVENTS_TOTAL: usize = 8_192; const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192; const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096; -const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000; +const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096; const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256; const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024; -const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256; -const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024; -const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024; -const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024; -const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32; -const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192; const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30; fn bounded_ttl(requested: u64, maximum: u64) -> u64 { requested.clamp(1, maximum) @@ -121,24 +111,6 @@ pub struct CoordinatorAdmission { pub max_artifact_download_ttl_seconds: u64, } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CoordinatorOperationalMetrics { - pub tenants: usize, - pub users: usize, - pub projects: usize, - pub enrolled_nodes: usize, - pub reported_nodes: usize, - pub live_nodes: usize, - pub active_processes: usize, - pub active_coordinator_mains: usize, - pub max_active_coordinator_mains: usize, - pub active_tasks: usize, - pub queued_tasks: usize, - pub artifacts: usize, - pub retained_download_links: usize, - pub relay: ArtifactRelayUsage, -} - impl Default for CoordinatorAdmission { fn default() -> Self { Self { @@ -179,139 +151,16 @@ pub enum CoordinatorServiceError { Durable(String), } -impl CoordinatorServiceError { - pub fn api_error(&self, request_id: impl Into) -> ApiError { - let request_id = request_id.into(); - let message = self.to_string(); - let (code, category, retryable) = match self { - Self::Io(_) => ( - ApiErrorCode::TemporaryCapacity, - ApiErrorCategory::Availability, - true, - ), - Self::Json(_) - | Self::CapabilityReport(_) - | Self::InvalidArtifactPath(_) - | Self::InvalidTaskLogTail(_) => ( - ApiErrorCode::ValidationError, - ApiErrorCategory::Validation, - false, - ), - Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => { - return ApiError::from_message(request_id, message); - } - Self::Coordinator(CoordinatorError::UnknownNode) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Coordinator(CoordinatorError::Enrollment(_)) => ( - ApiErrorCode::Unauthenticated, - ApiErrorCategory::Authentication, - false, - ), - Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => { - (ApiErrorCode::Conflict, ApiErrorCategory::State, true) - } - Self::Download(DownloadError::NotFound) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Download(DownloadError::Unavailable) - | Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => ( - ApiErrorCode::ArtifactUnavailable, - ApiErrorCategory::Availability, - true, - ), - Self::Download(DownloadError::LimitExceeded { .. }) => ( - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCategory::Resource, - false, - ), - Self::Download(DownloadError::Unauthorized(_)) - | Self::Download(DownloadError::InvalidToken) - | Self::Download(DownloadError::Expired) - | Self::Download(DownloadError::Revoked) => ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ), - Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ), - Self::Scheduler(_) => ( - ApiErrorCode::NoCapableNode, - ApiErrorCategory::Availability, - true, - ), - Self::Transport(_) => ( - ApiErrorCode::NodeOffline, - ApiErrorCategory::Availability, - true, - ), - Self::Panel(PanelError::RateLimited) => ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ), - Self::Panel(PanelError::UnknownWidget(_)) => { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } - Self::Panel(_) => ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ), - Self::Durable(_) => ( - ApiErrorCode::InternalError, - ApiErrorCategory::Internal, - true, - ), - }; - ApiError::new(code, category, message, retryable, request_id) - } -} - pub struct CoordinatorService { coordinator: Coordinator, store: RuntimeDurableStore, - node_descriptors: BTreeMap, - node_last_seen_epoch_seconds: BTreeMap, + node_descriptors: BTreeMap, + node_last_seen_epoch_seconds: BTreeMap, node_stale_after_seconds: u64, debug_freeze_timeout: std::time::Duration, enrollment_grants: BTreeMap, task_events: VecDeque, process_scope_history: VecDeque, - process_summaries: BTreeMap, - process_summary_order: VecDeque, - next_process_summary_order: u64, - task_terminal_states: BTreeMap, - recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque>, - recent_log_dropped_through: BTreeMap, - recent_log_accounted_bytes: BTreeMap< - ( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - ), - u64, - >, - recent_log_truncated_streams: BTreeSet<( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - )>, - recent_log_quota_truncated_streams: BTreeSet<( - TenantId, - ProjectId, - ProcessId, - clusterflux_core::TaskInstanceId, - String, - )>, - next_recent_log_sequence: u64, debug_audit_events: VecDeque, debug_epochs: BTreeMap, debug_epoch_runtime: BTreeMap, @@ -331,7 +180,7 @@ pub struct CoordinatorService { process_cancellations: BTreeSet, process_aborts: BTreeSet, agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>, - node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>, + node_replay_nonces: BTreeMap<(NodeId, String), u64>, panel_snapshots: BTreeMap, stopped_panels: BTreeSet, panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>, @@ -366,29 +215,6 @@ impl CoordinatorService { self.artifact_relay.usage() } - pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics { - CoordinatorOperationalMetrics { - tenants: self.coordinator.tenant_count(), - users: self.coordinator.user_count(), - projects: self.coordinator.project_count(), - enrolled_nodes: self.coordinator.node_identity_count(), - reported_nodes: self.node_descriptors.len(), - live_nodes: self - .node_descriptors - .keys() - .filter(|scope| self.node_is_live(scope)) - .count(), - active_processes: self.coordinator.active_process_count(), - active_coordinator_mains: self.main_runtime.active_main_count(), - max_active_coordinator_mains: self.main_runtime.max_active_mains(), - active_tasks: self.active_tasks.len(), - queued_tasks: self.pending_task_launches.len(), - artifacts: self.artifact_registry.artifact_count(), - retained_download_links: self.artifact_registry.retained_download_link_count(), - relay: self.artifact_relay.usage(), - } - } - fn commit_artifact_relay( &mut self, candidate: relay::ArtifactRelayLedger, @@ -458,10 +284,15 @@ impl CoordinatorService { { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node process-control request is outside its enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } return Ok(()); } self.coordinator @@ -578,16 +409,6 @@ impl CoordinatorService { enrollment_grants: BTreeMap::new(), task_events: VecDeque::new(), process_scope_history: VecDeque::new(), - process_summaries: BTreeMap::new(), - process_summary_order: VecDeque::new(), - next_process_summary_order: 1, - task_terminal_states: BTreeMap::new(), - recent_logs: BTreeMap::new(), - recent_log_dropped_through: BTreeMap::new(), - recent_log_accounted_bytes: BTreeMap::new(), - recent_log_truncated_streams: BTreeSet::new(), - recent_log_quota_truncated_streams: BTreeSet::new(), - next_recent_log_sequence: 1, debug_audit_events: VecDeque::new(), debug_epochs: BTreeMap::new(), debug_epoch_runtime: BTreeMap::new(), diff --git a/crates/clusterflux-coordinator/src/service/artifacts.rs b/crates/clusterflux-coordinator/src/service/artifacts.rs index 1a94c7e..c6698f9 100644 --- a/crates/clusterflux-coordinator/src/service/artifacts.rs +++ b/crates/clusterflux-coordinator/src/service/artifacts.rs @@ -8,7 +8,7 @@ use clusterflux_core::{ }; use sha2::{Digest as _, Sha256}; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::relay::RelayFinishReason; use super::{ @@ -52,11 +52,7 @@ impl CoordinatorService { let action = self .artifact_registry .download_action(&context, &artifact, &policy)?; - self.ensure_download_source_connectivity( - &context.tenant, - &context.project, - &action.source, - )?; + self.ensure_download_source_connectivity(&action.source)?; let downloadable_size = self .artifact_registry .downloadable_size(&context, &artifact, &policy)?; @@ -167,11 +163,7 @@ impl CoordinatorService { }, &mut validation_meter, )?; - self.ensure_download_source_connectivity( - &stream.link.tenant, - &stream.link.project, - &stream.link.source, - )?; + self.ensure_download_source_connectivity(&stream.link.source)?; self.expire_artifact_reverse_transfers(now_epoch_seconds)?; if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() { @@ -310,7 +302,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&context.tenant, &context.project, &artifact) + .metadata(&artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let transfer_id = generate_opaque_token("artifact_transfer") .map_err(CoordinatorServiceError::Protocol)?; @@ -418,7 +410,7 @@ impl CoordinatorService { }; let metadata = self .artifact_registry - .metadata(&context.tenant, &context.project, &artifact) + .metadata(&artifact) .ok_or(clusterflux_core::DownloadError::NotFound)?; let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?; let destination = @@ -690,10 +682,15 @@ impl CoordinatorService { ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "artifact reverse transfer node is outside its enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } Ok(()) } @@ -728,12 +725,15 @@ impl CoordinatorService { ) -> Result { let identity = self .coordinator - .node_identity(tenant, project, node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(&identity.tenant, tenant); - debug_assert_eq!(&identity.project, project); - let node_scope = NodeScopeKey::from_refs(tenant, project, node); - let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { + if &identity.tenant != tenant || &identity.project != project { + return Err(CoordinatorError::Unauthorized( + "artifact export node is outside the tenant/project scope".to_owned(), + ) + .into()); + } + let descriptor = self.node_descriptors.get(node).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} has not reported export connectivity" )) @@ -744,7 +744,7 @@ impl CoordinatorService { ) .into()); } - if !self.node_is_live(&node_scope) { + if !self.node_is_live(node) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "node {node} is offline for artifact export" @@ -769,20 +769,17 @@ impl CoordinatorService { fn ensure_download_source_connectivity( &self, - tenant: &TenantId, - project: &ProjectId, source: &StorageLocation, ) -> Result<(), clusterflux_core::DownloadError> { let StorageLocation::RetainedNode(node) = source else { return Ok(()); }; - let node_scope = NodeScopeKey::from_refs(tenant, project, node); - let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| { + let _descriptor = self.node_descriptors.get(node).ok_or_else(|| { clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} has not reported online status for artifact download" )) })?; - if !self.node_is_live(&node_scope) { + if !self.node_is_live(node) { return Err( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!( "retaining node {node} is offline for artifact download" diff --git a/crates/clusterflux-coordinator/src/service/authorization.rs b/crates/clusterflux-coordinator/src/service/authorization.rs index f4d60fd..92c1e6d 100644 --- a/crates/clusterflux-coordinator/src/service/authorization.rs +++ b/crates/clusterflux-coordinator/src/service/authorization.rs @@ -17,7 +17,6 @@ pub(super) enum PublicUserOperation { RevokeAgentPublicKey, CreateNodeEnrollmentGrant, ListNodeDescriptors, - ListNodeSummaries, RevokeNodeCredential, StartProcess, ScheduleTask, @@ -25,7 +24,6 @@ pub(super) enum PublicUserOperation { CancelProcess, AbortProcess, ListProcesses, - ListProcessSummaries, QuotaStatus, RestartTask, ResolveTaskFailure, @@ -37,10 +35,7 @@ pub(super) enum PublicUserOperation { InspectDebugEpoch, ListTaskEvents, ListTaskSnapshots, - ListRecentLogs, JoinTask, - ListArtifacts, - GetArtifact, CreateArtifactDownloadLink, OpenArtifactDownloadStream, RevokeArtifactDownloadLink, @@ -61,7 +56,6 @@ impl PublicUserOperation { Self::RevokeAgentPublicKey => "revoke_agent_public_key", Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant", Self::ListNodeDescriptors => "list_node_descriptors", - Self::ListNodeSummaries => "list_node_summaries", Self::RevokeNodeCredential => "revoke_node_credential", Self::StartProcess => "start_process", Self::ScheduleTask => "schedule_task", @@ -69,7 +63,6 @@ impl PublicUserOperation { Self::CancelProcess => "cancel_process", Self::AbortProcess => "abort_process", Self::ListProcesses => "list_processes", - Self::ListProcessSummaries => "list_process_summaries", Self::QuotaStatus => "quota_status", Self::RestartTask => "restart_task", Self::ResolveTaskFailure => "resolve_task_failure", @@ -81,10 +74,7 @@ impl PublicUserOperation { Self::InspectDebugEpoch => "inspect_debug_epoch", Self::ListTaskEvents => "list_task_events", Self::ListTaskSnapshots => "list_task_snapshots", - Self::ListRecentLogs => "list_recent_logs", Self::JoinTask => "join_task", - Self::ListArtifacts => "list_artifacts", - Self::GetArtifact => "get_artifact", Self::CreateArtifactDownloadLink => "create_artifact_download_link", Self::OpenArtifactDownloadStream => "open_artifact_download_stream", Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link", @@ -115,7 +105,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { Self::CreateNodeEnrollmentGrant } AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors, - AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries, AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => { Self::RevokeNodeCredential } @@ -125,9 +114,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess, AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess, AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses, - AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => { - Self::ListProcessSummaries - } AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus, AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask, AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure, @@ -143,10 +129,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation { AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch, AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents, AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots, - AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs, AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask, - AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts, - AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact, AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => { Self::CreateArtifactDownloadLink } diff --git a/crates/clusterflux-coordinator/src/service/debug.rs b/crates/clusterflux-coordinator/src/service/debug.rs index d5af181..efa1406 100644 --- a/crates/clusterflux-coordinator/src/service/debug.rs +++ b/crates/clusterflux-coordinator/src/service/debug.rs @@ -35,7 +35,6 @@ pub(super) struct DebugEpochRuntime { #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct DebugBreakpointPlan { pub(super) actor: UserId, - pub(super) revision: u64, pub(super) probe_symbols: BTreeSet, pub(super) hit_epoch: Option, pub(super) hit_task: Option, @@ -86,7 +85,6 @@ impl CoordinatorService { project: String, actor_user: String, process: String, - revision: u64, probe_symbols: Vec, ) -> Result { let probe_symbols = validate_probe_symbols(probe_symbols)?; @@ -117,28 +115,10 @@ impl CoordinatorService { )) .into()); } - let key = process_control_key(&tenant, &project, &process); - if let Some(current) = self.debug_breakpoints.get(&key) { - if current.actor == actor && revision < current.revision { - return Ok(CoordinatorResponse::DebugBreakpoints { - process, - actor, - revision: current.revision, - probe_symbols: current.probe_symbols.iter().cloned().collect(), - hit_epoch: current.hit_epoch, - hit_task: current.hit_task.clone(), - hit_probe_symbol: current.hit_probe_symbol.clone(), - charged_debug_read_bytes: audit_event.charged_debug_read_bytes, - used_debug_read_bytes: audit_event.used_debug_read_bytes, - audit_event, - }); - } - } self.debug_breakpoints.insert( - key, + process_control_key(&tenant, &project, &process), DebugBreakpointPlan { actor: actor.clone(), - revision, probe_symbols: probe_symbols.iter().cloned().collect(), hit_epoch: None, hit_task: None, @@ -148,7 +128,6 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, - revision, probe_symbols, hit_epoch: None, hit_task: None, @@ -211,7 +190,6 @@ impl CoordinatorService { Ok(CoordinatorResponse::DebugBreakpoints { process, actor, - revision: plan.revision, probe_symbols: plan.probe_symbols.into_iter().collect(), hit_epoch: plan.hit_epoch, hit_task: plan.hit_task, diff --git a/crates/clusterflux-coordinator/src/service/debug_requests.rs b/crates/clusterflux-coordinator/src/service/debug_requests.rs index 343f177..b7ac02e 100644 --- a/crates/clusterflux-coordinator/src/service/debug_requests.rs +++ b/crates/clusterflux-coordinator/src/service/debug_requests.rs @@ -33,14 +33,12 @@ impl CoordinatorService { project, actor_user, process, - revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant, project, actor_user, process, - revision, probe_symbols, ), CoordinatorRequest::InspectDebugBreakpoints { @@ -98,14 +96,12 @@ impl CoordinatorService { ), AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, - revision, probe_symbols, } => self.handle_set_debug_breakpoints( tenant.as_str().to_owned(), project.as_str().to_owned(), actor.as_str().to_owned(), process, - revision, probe_symbols, ), AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self @@ -226,25 +222,24 @@ impl CoordinatorService { } else if !completed_event_observed { "selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned() } else if let Some(checkpoint) = checkpoint { - let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold( - true, - |available, (path, object)| { - let artifact = super::keys::artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?; - Ok::<_, CoordinatorServiceError>( - available - && self - .artifact_registry - .metadata(&tenant, &project, &artifact) - .is_some_and(|metadata| { - metadata.digest == object.digest - && metadata.size == object.size - && !metadata.retaining_nodes.is_empty() - }), - ) - }, - )?; + let vfs_available = + checkpoint + .checkpoint + .vfs_manifest + .objects + .iter() + .all(|(path, object)| { + let artifact = super::keys::artifact_id_from_path(path); + self.artifact_registry + .metadata(&artifact) + .is_some_and(|metadata| { + metadata.tenant == tenant + && metadata.project == project + && metadata.digest == object.digest + && metadata.size == object.size + && !metadata.retaining_nodes.is_empty() + }) + }); if !vfs_available { "selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned() } else { @@ -290,25 +285,6 @@ impl CoordinatorService { task.as_str() ); if let Some(replacement) = replacement { - if assignment.task_spec.source_snapshot.is_some() { - let replacement_source = replacement_bundle - .as_ref() - .and_then(|bundle| bundle.source_snapshot.clone()) - .ok_or_else(|| { - CoordinatorServiceError::Protocol( - "replacement task omitted the current SourceSnapshot for source-bound arguments" - .to_owned(), - ) - })?; - assignment - .task_spec - .rebind_source_snapshot(replacement_source) - .map_err(|error| { - CoordinatorServiceError::Protocol(format!( - "replacement task SourceSnapshot is invalid: {error}" - )) - })?; - } assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm { export: Some(replacement.export), abi: WasmExportAbi::TaskV1, @@ -480,7 +456,6 @@ impl CoordinatorService { } self.record_task_completion_event(event.clone()); self.notify_coordinator_main_waiters(&event); - self.maybe_retire_terminal_process(&tenant, &project, &process)?; Ok(CoordinatorResponse::TaskFailureResolved { process, task, diff --git a/crates/clusterflux-coordinator/src/service/keys.rs b/crates/clusterflux-coordinator/src/service/keys.rs index 2e14ba8..8cc82c5 100644 --- a/crates/clusterflux-coordinator/src/service/keys.rs +++ b/crates/clusterflux-coordinator/src/service/keys.rs @@ -1,7 +1,6 @@ use clusterflux_core::{ ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath, }; -use thiserror::Error; pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId); pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId); @@ -64,47 +63,11 @@ pub(super) fn enrollment_grant_key( (tenant.clone(), project.clone(), grant.to_owned()) } -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub(super) enum ArtifactPathError { - #[error("path must start with the exact /vfs/artifacts/ prefix")] - WrongPrefix, - #[error("path must name an artifact after /vfs/artifacts/")] - EmptyArtifact, - #[error("mapped artifact identifier is invalid: {0}")] - InvalidArtifactId(#[from] clusterflux_core::IdParseError), -} - -pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result { +pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId { let value = path .as_str() .strip_prefix("/vfs/artifacts/") - .ok_or(ArtifactPathError::WrongPrefix)?; - if value.is_empty() { - return Err(ArtifactPathError::EmptyArtifact); - } - ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn artifact_id_conversion_is_fallible_and_structural() { - assert_eq!( - artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(), - ArtifactId::from("build:output") - ); - for path in [ - "/vfs/other/output", - "/vfs/artifacts", - "/vfs/artifacts/bad artifact!", - ] { - let path = VfsPath::new(path).unwrap(); - assert!(artifact_id_from_path(&path).is_err(), "{path:?}"); - } - - let mapped = format!("/vfs/artifacts/{}", "x".repeat(256)); - assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err()); - } + .unwrap_or(path.as_str()) + .replace('/', ":"); + ArtifactId::new(value) } diff --git a/crates/clusterflux-coordinator/src/service/logs.rs b/crates/clusterflux-coordinator/src/service/logs.rs index 1fb575b..fec1d2b 100644 --- a/crates/clusterflux-coordinator/src/service/logs.rs +++ b/crates/clusterflux-coordinator/src/service/logs.rs @@ -1,5 +1,5 @@ use clusterflux_core::{ - ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, + ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath, }; @@ -9,10 +9,7 @@ use super::keys::{process_control_key, task_control_key, task_restart_key}; use super::protocol::TaskAttemptState; use super::{ artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError, - RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState, - MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES, - MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT, - MAX_TASK_LOG_TAIL_BYTES, + TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES, }; impl CoordinatorService { @@ -39,44 +36,23 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; validate_task_log_tail("stdout_tail", &stdout_tail)?; validate_task_log_tail("stderr_tail", &stderr_tail)?; + let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - let stdout_retained = self.accept_final_log_stream( - &tenant, - &project, - &process, - &task, - TaskLogStream::Stdout, - stdout_bytes, - &stdout_tail, - stdout_truncated, - now_epoch_seconds, - )?; - let stderr_retained = self.accept_final_log_stream( - &tenant, - &project, - &process, - &task, - TaskLogStream::Stderr, - stderr_bytes, - &stderr_tail, - stderr_truncated, - now_epoch_seconds, - )?; + self.quota + .can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; + self.quota + .charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?; Ok(CoordinatorResponse::TaskLogRecorded { process, task, stdout_bytes, stderr_bytes, - stdout_tail: if !stdout_retained { - "[log output truncated at project log quota]".to_owned() - } else if stdout_truncated { + stdout_tail: if stdout_truncated { format!("{stdout_tail}\n... truncated") } else { stdout_tail }, - stderr_tail: if !stderr_retained { - "[log output truncated at project log quota]".to_owned() - } else if stderr_truncated { + stderr_tail: if stderr_truncated { format!("{stderr_tail}\n... truncated") } else { stderr_tail @@ -85,158 +61,6 @@ impl CoordinatorService { }) } - #[allow(clippy::too_many_arguments)] - pub(super) fn handle_report_task_log_chunk( - &mut self, - tenant: String, - project: String, - process: String, - node: String, - task: String, - stream: TaskLogStream, - offset: u64, - source_bytes: u64, - text: String, - truncated: bool, - ) -> Result { - if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { - return Err(CoordinatorServiceError::InvalidTaskLogTail(format!( - "live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}", - text.len() - ))); - } - if source_bytes == 0 && !text.is_empty() && !truncated { - return Err(CoordinatorServiceError::Protocol( - "live log chunk source_bytes must describe non-empty text".to_owned(), - )); - } - if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) { - return Err(CoordinatorServiceError::Protocol( - "live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(), - )); - } - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let process = ProcessId::new(process); - let node = NodeId::new(node); - let task = TaskInstanceId::new(task); - self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; - let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream); - let expected = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let end = offset.checked_add(source_bytes).ok_or_else(|| { - CoordinatorServiceError::Protocol( - "live log chunk offset exceeds the supported range".to_owned(), - ) - })?; - let state_marker = source_bytes == 0 && truncated; - if end < expected || (end == expected && !state_marker) { - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence: None, - next_offset: expected, - }); - } - let now_epoch_seconds = self.current_epoch_seconds()?; - if self.recent_log_quota_truncated_streams.contains(&key) { - if end > expected { - self.recent_log_accounted_bytes.insert(key, end); - } - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence: None, - next_offset: end.max(expected), - }); - } - let newly_accounted = end.saturating_sub(expected); - if self - .quota - .charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds) - .is_err() - { - if end > expected { - self.recent_log_accounted_bytes.insert(key.clone(), end); - } - let sequence = self.mark_log_quota_truncated( - &tenant, - &project, - &process, - &task, - &stream, - now_epoch_seconds, - ); - return Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence, - next_offset: end.max(expected), - }); - } - if offset > expected { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!("[log output lost: {} bytes]", offset - expected), - true, - now_epoch_seconds, - ); - } else if offset < expected { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[log output overlap omitted: {} new source bytes]", - end - expected - ), - true, - now_epoch_seconds, - ); - } - let marker_is_new = if truncated { - self.recent_log_truncated_streams.insert(key.clone()) - } else { - false - }; - let text = if state_marker && text.is_empty() { - "[log output truncated at source]".to_owned() - } else { - text - }; - let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new)) - .then(|| { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream, - text, - truncated, - now_epoch_seconds, - ) - }); - if end > expected { - self.recent_log_accounted_bytes.insert(key, end); - } - Ok(CoordinatorResponse::TaskLogChunkRecorded { - process, - task, - sequence, - next_offset: end, - }) - } - pub(super) fn handle_report_vfs_metadata( &mut self, tenant: String, @@ -261,9 +85,7 @@ impl CoordinatorService { self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?; if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?, + id: artifact_id_from_path(path), tenant, project, process: process.clone(), @@ -355,37 +177,20 @@ impl CoordinatorService { artifact_size_bytes, result, }; + let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?; let now_epoch_seconds = self.current_epoch_seconds()?; - let stdout_retained = self.accept_final_log_stream( + self.quota.can_charge_log_bytes( &event.tenant, &event.project, - &event.process, - &event.task, - TaskLogStream::Stdout, - event.stdout_bytes, - &event.stdout_tail, - event.stdout_truncated, + reported_bytes, now_epoch_seconds, )?; - let stderr_retained = self.accept_final_log_stream( + self.quota.charge_log_bytes( &event.tenant, &event.project, - &event.process, - &event.task, - TaskLogStream::Stderr, - event.stderr_bytes, - &event.stderr_tail, - event.stderr_truncated, + reported_bytes, now_epoch_seconds, )?; - if !stdout_retained { - event.stdout_tail = "[log output truncated at project log quota]".to_owned(); - event.stdout_truncated = true; - } - if !stderr_retained { - event.stderr_tail = "[log output truncated at project log quota]".to_owned(); - event.stderr_truncated = true; - } let task_key = task_control_key( &event.tenant, &event.project, @@ -398,9 +203,7 @@ impl CoordinatorService { event.placement = self.task_placements.remove(&task_key); if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) { self.flush_artifact_metadata(ArtifactFlush { - id: artifact_id_from_path(path).map_err(|error| { - CoordinatorServiceError::InvalidArtifactPath(error.to_string()) - })?, + id: artifact_id_from_path(path), tenant: event.tenant.clone(), project: event.project.clone(), process: event.process.clone(), @@ -414,22 +217,26 @@ impl CoordinatorService { self.task_aborts.remove(&task_key); self.debug_commands.remove(&task_key); self.active_tasks.remove(&task_key); - self.clear_recent_log_offsets_for_task( - &event.tenant, - &event.project, - &event.process, - &event.task, - ); - self.task_assignments.retain(|_, assignments| { - assignments.retain(|assignment| { - assignment.tenant != event.tenant - || assignment.project != event.project - || assignment.process != event.process - || assignment.node != event.node - || assignment.task != event.task - }); - !assignments.is_empty() - }); + let no_active_tasks = + !self + .active_tasks + .iter() + .any(|(task_tenant, task_project, task_process, _, _)| { + task_tenant == &event.tenant + && task_project == &event.project + && task_process == &event.process + }); + if no_active_tasks { + self.process_aborts.remove(&process_key); + if self.process_cancellations.remove(&process_key) + && !self.main_runtime.controls.contains_key(&process_key) + { + self.coordinator + .abort_process(&event.tenant, &event.project, &event.process)?; + self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process); + self.clear_operator_panel_state(&event.tenant, &event.project, &event.process); + } + } if process_was_aborted { let checkpoint_key = super::keys::task_restart_key( &event.tenant, @@ -446,20 +253,10 @@ impl CoordinatorService { if !awaiting_operator { self.notify_coordinator_main_waiters(&event); } - self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?; - let events_recorded = self - .task_events - .iter() - .filter(|recorded| { - recorded.tenant == event.tenant - && recorded.project == event.project - && recorded.process == event.process - }) - .count(); Ok(CoordinatorResponse::TaskRecorded { process: event.process, task: event.task, - events_recorded, + events_recorded: self.task_events.len(), }) } @@ -519,7 +316,7 @@ impl CoordinatorService { Ok(CoordinatorResponse::TaskSnapshots { snapshots }) } - pub(super) fn authorize_task_event_process_scope( + fn authorize_task_event_process_scope( &self, tenant: &TenantId, project: &ProjectId, @@ -559,47 +356,6 @@ impl CoordinatorService { Ok(()) } - pub(super) fn handle_list_recent_logs( - &mut self, - tenant: String, - project: String, - actor_user: String, - process: String, - task: Option, - after_sequence: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let process = ProcessId::new(process); - let task = task.map(TaskInstanceId::new); - self.authorize_task_event_process_scope(&tenant, &project, &process)?; - let after_sequence = after_sequence.unwrap_or(0); - let retained = self.recent_logs.get(&(tenant.clone(), project.clone())); - let history_truncated = self - .recent_log_dropped_through - .get(&process_control_key(&tenant, &project, &process)) - .is_some_and(|dropped_through| *dropped_through > after_sequence); - let entries = retained - .into_iter() - .flatten() - .filter(|entry| { - entry.process == process - && entry.sequence > after_sequence - && task.as_ref().is_none_or(|task| &entry.task == task) - }) - .take(limit as usize) - .cloned() - .collect::>(); - let next_sequence = entries.last().map(|entry| entry.sequence); - Ok(CoordinatorResponse::RecentLogs { - entries, - next_sequence, - history_truncated, - }) - } - pub(super) fn handle_join_task( &mut self, tenant: String, @@ -718,22 +474,6 @@ impl CoordinatorService { pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) { event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated); event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated); - match event.executor { - super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state( - &event.tenant, - &event.project, - &event.process, - event.task_definition.clone(), - event.task.clone(), - event.terminal_state.clone(), - ), - super::TaskExecutor::Node => { - self.task_terminal_states.insert( - task_restart_key(&event.tenant, &event.project, &event.process, &event.task), - event.terminal_state.clone(), - ); - } - } let process_scope = ( event.tenant.clone(), event.project.clone(), @@ -771,321 +511,6 @@ impl CoordinatorService { self.task_events.push_back(event); } - #[allow(clippy::too_many_arguments)] - fn record_recent_log( - &mut self, - tenant: TenantId, - project: ProjectId, - process: ProcessId, - task: TaskInstanceId, - stream: TaskLogStream, - mut text: String, - mut truncated: bool, - server_timestamp_epoch_seconds: u64, - ) -> u64 { - if text.len() > MAX_RECENT_LOG_CHUNK_BYTES { - let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES; - while !text.is_char_boundary(boundary) { - boundary -= 1; - } - text.truncate(boundary); - truncated = true; - } - let sequence = self.next_recent_log_sequence; - self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1); - let logs = self - .recent_logs - .entry((tenant.clone(), project.clone())) - .or_default(); - let mut dropped = Vec::new(); - while logs.iter().filter(|entry| entry.process == process).count() - >= MAX_RECENT_LOG_ENTRIES_PER_PROCESS - { - let Some(index) = logs.iter().position(|entry| entry.process == process) else { - break; - }; - if let Some(entry) = logs.remove(index) { - dropped.push(entry); - } - } - while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT - || logs - .iter() - .map(|entry| entry.text.len()) - .sum::() - .saturating_add(text.len()) - > MAX_RECENT_LOG_BYTES_PER_PROJECT - { - match logs.pop_front() { - Some(entry) => dropped.push(entry), - None => break, - } - } - logs.push_back(RecentLogEntry { - sequence, - process, - task, - stream, - text, - server_timestamp_epoch_seconds, - truncated, - }); - for entry in dropped { - let key = process_control_key(&tenant, &project, &entry.process); - self.recent_log_dropped_through - .entry(key) - .and_modify(|dropped_through| { - *dropped_through = (*dropped_through).max(entry.sequence); - }) - .or_insert(entry.sequence); - } - sequence - } - - #[allow(clippy::too_many_arguments)] - fn accept_final_log_stream( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: TaskLogStream, - total_source_bytes: u64, - final_tail: &str, - source_truncated: bool, - now_epoch_seconds: u64, - ) -> Result { - let key = recent_log_offset_key(tenant, project, process, task, &stream); - let accounted = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| { - let stream_name = match stream { - TaskLogStream::Stdout => "stdout", - TaskLogStream::Stderr => "stderr", - }; - CoordinatorServiceError::Protocol(format!( - "final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted" - )) - })?; - if self.recent_log_quota_truncated_streams.contains(&key) { - self.recent_log_accounted_bytes - .insert(key, total_source_bytes); - return Ok(false); - } - if self - .quota - .charge_log_bytes(tenant, project, remaining, now_epoch_seconds) - .is_err() - { - self.recent_log_accounted_bytes - .insert(key, total_source_bytes); - self.mark_log_quota_truncated( - tenant, - project, - process, - task, - &stream, - now_epoch_seconds, - ); - return Ok(false); - } - self.reconcile_final_log_stream( - tenant, - project, - process, - task, - stream, - total_source_bytes, - final_tail, - source_truncated, - now_epoch_seconds, - ); - Ok(true) - } - - #[allow(clippy::too_many_arguments)] - fn mark_log_quota_truncated( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: &TaskLogStream, - now_epoch_seconds: u64, - ) -> Option { - let key = recent_log_offset_key(tenant, project, process, task, stream); - if !self.recent_log_quota_truncated_streams.insert(key.clone()) { - return None; - } - self.recent_log_truncated_streams.insert(key); - Some(self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - "[log output truncated at project log quota]".to_owned(), - true, - now_epoch_seconds, - )) - } - - #[allow(clippy::too_many_arguments)] - fn reconcile_final_log_stream( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: TaskLogStream, - total_source_bytes: u64, - final_tail: &str, - source_truncated: bool, - now_epoch_seconds: u64, - ) { - let key = recent_log_offset_key(tenant, project, process, task, &stream); - let accounted = self - .recent_log_accounted_bytes - .get(&key) - .copied() - .unwrap_or(0); - let mut visible_truncation = false; - if total_source_bytes > accounted { - let missing = total_source_bytes - accounted; - if final_tail.is_empty() { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!("[log output unavailable: {missing} source bytes]"), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } else if (final_tail.len() as u64) <= total_source_bytes { - let tail_source_start = total_source_bytes - final_tail.len() as u64; - if accounted < tail_source_start { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[log output lost before final tail: {} source bytes]", - tail_source_start - accounted - ), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } - let source_start = accounted.max(tail_source_start) - tail_source_start; - let mut byte_start = usize::try_from(source_start) - .unwrap_or(final_tail.len()) - .min(final_tail.len()); - while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) { - byte_start += 1; - } - let suffix = &final_tail[byte_start..]; - if !suffix.is_empty() { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - suffix.to_owned(), - source_truncated || visible_truncation, - now_epoch_seconds, - ); - visible_truncation |= source_truncated; - } - } else if accounted == 0 { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - final_tail.to_owned(), - source_truncated, - now_epoch_seconds, - ); - visible_truncation |= source_truncated; - } else { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream.clone(), - format!( - "[{missing} additional source bytes could not be merged without duplicating redacted output]" - ), - true, - now_epoch_seconds, - ); - visible_truncation = true; - } - self.recent_log_accounted_bytes - .insert(key.clone(), total_source_bytes); - } - - let marker_is_new = (source_truncated || visible_truncation) - && self.recent_log_truncated_streams.insert(key); - if marker_is_new && !visible_truncation { - self.record_recent_log( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - stream, - "[log output truncated at source]".to_owned(), - true, - now_epoch_seconds, - ); - } - } - - pub(super) fn clear_recent_log_offsets_for_task( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - ) { - self.recent_log_accounted_bytes.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _), _| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - self.recent_log_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _)| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - self.recent_log_quota_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, entry_task, _)| { - entry_tenant != tenant - || entry_project != project - || entry_process != process - || entry_task != task - }, - ); - } - fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool { let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task); let Some(attempt) = self @@ -1120,118 +545,6 @@ impl CoordinatorService { awaiting_operator } - pub(super) fn maybe_retire_terminal_process( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - ) -> Result { - let process_key = process_control_key(tenant, project, process); - if self.main_runtime.controls.contains_key(&process_key) { - return Ok(false); - } - - let has_runnable_remote_work = - self.active_tasks - .iter() - .any(|(task_tenant, task_project, task_process, _, _)| { - task_tenant == tenant && task_project == project && task_process == process - }) - || self.pending_task_launches.iter().any(|pending| { - &pending.tenant == tenant - && &pending.project == project - && &pending.process == process - }) - || self.task_assignments.values().any(|assignments| { - assignments.iter().any(|assignment| { - &assignment.tenant == tenant - && &assignment.project == project - && &assignment.process == process - }) - }) - || self.task_attempts.iter().any( - |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { - attempt_tenant == tenant - && attempt_project == project - && attempt_process == process - && attempts.iter().rev().any(|attempt| { - attempt.current - && matches!( - attempt.state, - TaskAttemptState::Queued - | TaskAttemptState::Running - | TaskAttemptState::FailedAwaitingAction - ) - }) - }, - ); - if has_runnable_remote_work { - return Ok(false); - } - - let main_completed = self - .process_summaries - .get(&process_key) - .and_then(|summary| summary.main_terminal_state.as_ref()) - .is_some_and(|state| matches!(state, TaskTerminalState::Completed)); - let cancellation_completed = self.process_cancellations.contains(&process_key); - if !main_completed && !cancellation_completed { - return Ok(false); - } - - self.process_aborts.remove(&process_key); - self.process_cancellations.remove(&process_key); - if self - .coordinator - .active_process(tenant, project, process) - .is_none() - { - return Ok(false); - } - let final_result = if cancellation_completed { - super::ProcessFinalResult::Cancelled - } else if self.task_terminal_states.iter().any( - |((task_tenant, task_project, task_process, _), terminal_state)| { - task_tenant == tenant - && task_project == project - && task_process == process - && terminal_state == &TaskTerminalState::Failed - }, - ) { - super::ProcessFinalResult::Failed - } else if self.task_terminal_states.iter().any( - |((task_tenant, task_project, task_process, _), terminal_state)| { - task_tenant == tenant - && task_project == project - && task_process == process - && terminal_state == &TaskTerminalState::Cancelled - }, - ) { - super::ProcessFinalResult::Cancelled - } else { - super::ProcessFinalResult::Completed - }; - self.record_process_terminal( - tenant, - project, - process, - final_result, - self.current_epoch_seconds()?, - ); - self.coordinator.abort_process(tenant, project, process)?; - self.clear_debug_state_for_process(tenant, project, process); - self.clear_operator_panel_state(tenant, project, process); - let (pinned, protected_processes) = - self.artifact_retention_guards_for_project(tenant, project); - self.artifact_registry.enforce_project_metadata_limit( - tenant, - project, - &pinned, - &protected_processes, - ); - Ok(true) - } - fn flush_artifact_metadata( &mut self, flush: ArtifactFlush, @@ -1239,59 +552,23 @@ impl CoordinatorService { let now_epoch_seconds = self.current_epoch_seconds()?; self.artifact_registry .expire_download_links(now_epoch_seconds); - let tenant = flush.tenant.clone(); - let project = flush.project.clone(); - let (pinned, protected_processes) = - self.artifact_retention_guards_for_project(&tenant, &project); + let pinned = self + .task_restart_checkpoints + .values() + .flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter()) + .chain( + self.pending_task_launches + .iter() + .flat_map(|pending| pending.task_spec.required_artifacts.iter()), + ) + .cloned() + .collect::>(); self.artifact_registry - .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes) + .flush_metadata_bounded(flush, &pinned) .map(|_| ()) .map_err(CoordinatorServiceError::Protocol) } - fn artifact_retention_guards_for_project( - &self, - tenant: &TenantId, - project: &ProjectId, - ) -> ( - std::collections::BTreeSet, - std::collections::BTreeSet, - ) { - let mut pinned = std::collections::BTreeSet::new(); - for checkpoint in self.task_restart_checkpoints.values() { - if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project - { - continue; - } - for artifact in &checkpoint.assignment.task_spec.required_artifacts { - pinned.insert(ArtifactScopeKey::from_refs( - &checkpoint.assignment.tenant, - &checkpoint.assignment.project, - artifact, - )); - } - } - for pending in &self.pending_task_launches { - if &pending.tenant != tenant || &pending.project != project { - continue; - } - for artifact in &pending.task_spec.required_artifacts { - pinned.insert(ArtifactScopeKey::from_refs( - &pending.tenant, - &pending.project, - artifact, - )); - } - } - let protected_processes = self - .coordinator - .active_processes_for_project(tenant, project) - .into_iter() - .map(|process| process.id) - .collect(); - (pinned, protected_processes) - } - fn task_is_known_or_active( &self, tenant: &TenantId, @@ -1324,24 +601,15 @@ impl CoordinatorService { } } -fn recent_log_offset_key( - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task: &TaskInstanceId, - stream: &TaskLogStream, -) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) { - ( - tenant.clone(), - project.clone(), - process.clone(), - task.clone(), - match stream { - TaskLogStream::Stdout => "stdout", - TaskLogStream::Stderr => "stderr", - } - .to_owned(), - ) +fn checked_reported_log_bytes( + stdout_bytes: u64, + stderr_bytes: u64, +) -> Result { + stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| { + CoordinatorServiceError::Protocol( + "reported task log byte counts exceed the supported range".to_owned(), + ) + }) } fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> { @@ -1358,11 +626,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String { if value.len() <= MAX_TASK_LOG_TAIL_BYTES { return value; } - let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES; - while boundary < value.len() && !value.is_char_boundary(boundary) { - boundary += 1; + let mut boundary = MAX_TASK_LOG_TAIL_BYTES; + while !value.is_char_boundary(boundary) { + boundary -= 1; } - value.drain(..boundary); + value.truncate(boundary); *truncated = true; value } diff --git a/crates/clusterflux-coordinator/src/service/main_runtime.rs b/crates/clusterflux-coordinator/src/service/main_runtime.rs index 30b81a8..dd2feb2 100644 --- a/crates/clusterflux-coordinator/src/service/main_runtime.rs +++ b/crates/clusterflux-coordinator/src/service/main_runtime.rs @@ -30,14 +30,14 @@ use super::{ }; #[derive(Clone)] -pub(super) struct MainScope { - pub(super) tenant: TenantId, - pub(super) project: ProjectId, - pub(super) process: ProcessId, - pub(super) task_definition: TaskDefinitionId, - pub(super) task_instance: TaskInstanceId, - pub(super) epoch: u64, - pub(super) launch_id: u64, +struct MainScope { + tenant: TenantId, + project: ProjectId, + process: ProcessId, + task_definition: TaskDefinitionId, + task_instance: TaskInstanceId, + epoch: u64, + launch_id: u64, } enum MainCommand { @@ -109,14 +109,6 @@ impl Default for CoordinatorMainRuntime { } impl CoordinatorMainRuntime { - pub(super) fn active_main_count(&self) -> usize { - self.controls.len() - } - - pub(super) fn max_active_mains(&self) -> usize { - self.max_active_mains - } - pub(super) fn configure( &mut self, configuration: super::CoordinatorMainRuntimeConfiguration, @@ -737,13 +729,6 @@ impl CoordinatorService { &process, "coordinator main launch failed admission or validation", ); - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Failed, - self.liveness_now_epoch_seconds(), - ); let _ = self.coordinator.abort_process(&tenant, &project, &process); } result @@ -904,7 +889,7 @@ impl CoordinatorService { } } - pub(super) fn record_coordinator_main_completion( + fn record_coordinator_main_completion( &mut self, scope: MainScope, result: Result, @@ -944,7 +929,6 @@ impl CoordinatorService { ), Err(error) => (TaskTerminalState::Failed, None, error), }; - let main_completed = matches!(terminal_state, TaskTerminalState::Completed); let main_state = match terminal_state { TaskTerminalState::Completed => "completed", TaskTerminalState::Failed => "failed", @@ -986,12 +970,6 @@ impl CoordinatorService { } } let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process); - self.main_runtime.controls.remove(&process_key); - if main_completed { - let _ = - self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process); - return; - } for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() { if task_tenant == scope.tenant && task_project == scope.project @@ -1007,16 +985,10 @@ impl CoordinatorService { } } self.process_aborts.insert(process_key.clone()); - self.record_process_terminal( - &scope.tenant, - &scope.project, - &scope.process, - super::ProcessFinalResult::Failed, - self.liveness_now_epoch_seconds(), - ); let _ = self .coordinator .abort_process(&scope.tenant, &scope.project, &scope.process); + self.main_runtime.controls.remove(&process_key); self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process); self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process); } @@ -1185,120 +1157,6 @@ mod tests { assert!(!service.main_runtime.controls.contains_key(&process_key)); assert!(!service.debug_epochs.contains_key(&process_key)); assert!(!service.debug_epoch_runtime.contains_key(&process_key)); - assert!(!service.process_aborts.contains(&process_key)); - } - - #[test] - fn completed_main_keeps_process_and_debug_state_for_active_children() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("vp-current"); - let main_task = TaskInstanceId::from("ti:vp-current:main"); - let child_task = TaskInstanceId::from("ti:vp-current:child:1"); - let child_node = NodeId::from("worker"); - let scope = MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main_task.clone(), - epoch: 7, - launch_id: 1, - }; - service - .coordinator - .start_process(tenant.clone(), project.clone(), process.clone()); - let process_key = process_control_key(&tenant, &project, &process); - service.main_runtime.controls.insert( - process_key.clone(), - CoordinatorMainControl { - task_definition: scope.task_definition.clone(), - task_instance: main_task.clone(), - abort: Arc::new(AtomicBool::new(false)), - debug: Arc::new(WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: Arc::new(Mutex::new(HashMap::new())), - launch_id: 1, - }, - ); - let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task); - service.active_tasks.insert(child_key.clone()); - service.debug_epochs.insert(process_key.clone(), 2); - - service.record_coordinator_main_completion( - scope, - Ok(WasmTaskResult::completed( - main_task, - TaskBoundaryValue::SmallJson(serde_json::Value::Null), - )), - ); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(!service.main_runtime.controls.contains_key(&process_key)); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(!service.process_aborts.contains(&process_key)); - assert!(service.active_tasks.contains(&child_key)); - assert!(!service.task_aborts.contains(&child_key)); - } - - #[test] - fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("vp-failed-main"); - let main_task = TaskInstanceId::from("ti:vp-failed-main:main"); - let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1"); - let child_node = NodeId::from("worker"); - let scope = MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main_task.clone(), - epoch: 7, - launch_id: 1, - }; - service - .coordinator - .start_process(tenant.clone(), project.clone(), process.clone()); - let process_key = process_control_key(&tenant, &project, &process); - service.main_runtime.controls.insert( - process_key.clone(), - CoordinatorMainControl { - task_definition: scope.task_definition.clone(), - task_instance: main_task, - abort: Arc::new(AtomicBool::new(false)), - debug: Arc::new(WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: Arc::new(Mutex::new(HashMap::new())), - launch_id: 1, - }, - ); - let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task); - service.active_tasks.insert(child_key.clone()); - service.debug_epochs.insert(process_key.clone(), 2); - - service.record_coordinator_main_completion(scope, Err("main crashed".to_owned())); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(service.active_tasks.contains(&child_key)); - assert!(service.task_aborts.contains(&child_key)); assert!(service.process_aborts.contains(&process_key)); - assert!(!service.debug_epochs.contains_key(&process_key)); - assert!(service.task_events.iter().any(|event| { - event.process == process - && event.executor == TaskExecutor::CoordinatorMain - && event.terminal_state == TaskTerminalState::Failed - })); } } diff --git a/crates/clusterflux-coordinator/src/service/nodes.rs b/crates/clusterflux-coordinator/src/service/nodes.rs index cedae30..9db62c8 100644 --- a/crates/clusterflux-coordinator/src/service/nodes.rs +++ b/crates/clusterflux-coordinator/src/service/nodes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ SourceProviderKind, TenantId, UserId, }; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::{ bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService, @@ -27,9 +27,9 @@ impl CoordinatorService { unix_timestamp_seconds() } - pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool { + pub(super) fn node_is_live(&self, node: &NodeId) -> bool { self.node_last_seen_epoch_seconds - .get(scope) + .get(node) .is_some_and(|last_seen| { self.liveness_now_epoch_seconds().saturating_sub(*last_seen) <= self.node_stale_after_seconds @@ -41,11 +41,7 @@ impl CoordinatorService { .values() .cloned() .map(|mut descriptor| { - descriptor.online = self.node_is_live(&NodeScopeKey::from_refs( - &descriptor.tenant, - &descriptor.project, - &descriptor.id, - )); + descriptor.online = self.node_is_live(&descriptor.id); descriptor }) .collect() @@ -169,11 +165,7 @@ impl CoordinatorService { !grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds }); self.coordinator.ensure_tenant_active(&tenant)?; - if self - .coordinator - .node_identity(&tenant, &project, &node) - .is_none() - { + if self.coordinator.node_identity(&node).is_none() { self.quota.ensure_node_admission( &tenant, self.coordinator.node_identity_count_for_tenant(&tenant), @@ -205,21 +197,12 @@ impl CoordinatorService { pub(super) fn handle_node_heartbeat( &mut self, - tenant: String, - project: String, node: String, node_signature: Option, payload_digest: &Digest, ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); let node = NodeId::new(node); - self.authenticate_node_request( - &NodeScopeKey::new(tenant, project, node.clone()), - node_signature, - "node_heartbeat", - payload_digest, - )?; + self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?; Ok(CoordinatorResponse::NodeHeartbeat { node, epoch: self.coordinator.coordinator_epoch(), @@ -242,13 +225,16 @@ impl CoordinatorService { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); let node = NodeId::new(node); - let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "node capability report is outside the enrolled tenant/project scope".to_owned(), + ) + .into()); + } capabilities.validate_public_report()?; for (kind, count) in [ ("cached environments", cached_environment_digests.len()), @@ -288,20 +274,16 @@ impl CoordinatorService { .into_iter() .map(ArtifactId::new) .collect::>(); - self.artifact_registry.reconcile_node_retention( - &tenant, - &project, - &node, - &artifact_locations, - ); + self.artifact_registry + .reconcile_node_retention(&node, &artifact_locations); - let online = self.node_is_live(&node_scope); + let online = self.node_is_live(&node); self.node_descriptors.insert( - node_scope, + node.clone(), NodeDescriptor { id: node.clone(), - tenant: tenant.clone(), - project: project.clone(), + tenant, + project, capabilities, cached_environments: cached_environment_digests.into_iter().collect(), dependency_caches: dependency_cache_digests.into_iter().collect(), @@ -311,14 +293,9 @@ impl CoordinatorService { online, }, ); - let node_descriptors = self - .node_descriptors - .values() - .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project) - .count(); Ok(CoordinatorResponse::NodeCapabilitiesRecorded { node, - node_descriptors, + node_descriptors: self.node_descriptors.len(), }) } @@ -356,13 +333,9 @@ impl CoordinatorService { actor: Actor::User(actor.clone()), }; self.coordinator.revoke_node_credential(&context, &node)?; - let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node); - let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some(); - self.node_last_seen_epoch_seconds.remove(&node_scope); - self.node_replay_nonces - .retain(|(retained_scope, _), _| retained_scope != &node_scope); - self.artifact_registry - .garbage_collect_node(&tenant, &project, &node); + let descriptor_removed = self.node_descriptors.remove(&node).is_some(); + self.node_last_seen_epoch_seconds.remove(&node); + self.artifact_registry.garbage_collect_node(&node); let queued_assignments_removed = self .task_assignments .remove(&(tenant.clone(), project.clone(), node.clone())) @@ -396,14 +369,14 @@ impl CoordinatorService { pub(super) fn authenticate_node_request( &mut self, - scope: &NodeScopeKey, + node: &NodeId, node_signature: Option, request_kind: &str, payload_digest: &Digest, ) -> Result<(), CoordinatorServiceError> { let identity = self .coordinator - .node_identity(&scope.tenant, &scope.project, &scope.node) + .node_identity(node) .ok_or(CoordinatorError::UnknownNode)?; let signature = node_signature.ok_or_else(|| { CoordinatorError::Unauthorized( @@ -428,7 +401,7 @@ impl CoordinatorService { ) .into()); } - let replay_key = (scope.clone(), signature.nonce.clone()); + let replay_key = (node.clone(), signature.nonce.clone()); self.node_replay_nonces.retain(|_, accepted_at| { now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS) }); @@ -440,7 +413,7 @@ impl CoordinatorService { } verify_node_request_signature( &identity.public_key, - &scope.node, + node, request_kind, payload_digest, &signature, @@ -449,7 +422,7 @@ impl CoordinatorService { if self .node_replay_nonces .keys() - .filter(|(retained_scope, _)| retained_scope == scope) + .filter(|(retained_node, _)| retained_node == node) .count() >= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY { @@ -463,8 +436,8 @@ impl CoordinatorService { .insert(replay_key, now_epoch_seconds); let seen_at = self.liveness_now_epoch_seconds(); self.node_last_seen_epoch_seconds - .insert(scope.clone(), seen_at); - if let Some(descriptor) = self.node_descriptors.get_mut(scope) { + .insert(node.clone(), seen_at); + if let Some(descriptor) = self.node_descriptors.get_mut(node) { descriptor.online = true; } Ok(()) diff --git a/crates/clusterflux-coordinator/src/service/panels.rs b/crates/clusterflux-coordinator/src/service/panels.rs index 9dd6a9a..4e66042 100644 --- a/crates/clusterflux-coordinator/src/service/panels.rs +++ b/crates/clusterflux-coordinator/src/service/panels.rs @@ -254,8 +254,7 @@ impl CoordinatorService { .rev() .find_map(|event| event.artifact_path.as_ref()) { - let artifact = artifact_id_from_path(path) - .map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?; + let artifact = artifact_id_from_path(path); let context = clusterflux_core::AuthContext { tenant, project, diff --git a/crates/clusterflux-coordinator/src/service/process_launch.rs b/crates/clusterflux-coordinator/src/service/process_launch.rs index 81737c3..b68d296 100644 --- a/crates/clusterflux-coordinator/src/service/process_launch.rs +++ b/crates/clusterflux-coordinator/src/service/process_launch.rs @@ -3,9 +3,9 @@ use std::collections::BTreeMap; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_core::{ AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest, - NodeDescriptor, NodeId, Placement, PlacementError, PlacementRequest, ProcessId, ProjectId, - Scheduler, TaskBoundaryValue, TaskCheckpoint, TaskDispatch, TaskInstanceId, TaskSpec, TenantId, - VfsManifest, VfsObject, VfsPath, WasmTaskInvocation, + NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskBoundaryValue, TaskCheckpoint, + TaskDispatch, TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath, + WasmTaskInvocation, }; use crate::CoordinatorError; @@ -18,76 +18,7 @@ use super::{ use super::processes::*; use super::protocol::{TaskAttemptSnapshot, TaskAttemptState}; -fn select_task_placement( - candidates: &[Placement], - active_by_node: &BTreeMap, -) -> Option { - candidates - .iter() - .min_by(|left, right| { - right - .score - .cmp(&left.score) - .then_with(|| { - active_by_node - .get(&left.node) - .copied() - .unwrap_or_default() - .cmp(&active_by_node.get(&right.node).copied().unwrap_or_default()) - }) - .then_with(|| left.node.cmp(&right.node)) - }) - .cloned() -} - impl CoordinatorService { - fn place_workflow_task( - &self, - nodes: &[NodeDescriptor], - request: &PlacementRequest, - ) -> Result { - let candidates = nodes - .iter() - .filter_map(|node| { - DefaultScheduler - .place(std::slice::from_ref(node), request) - .ok() - }) - .collect::>(); - if candidates.is_empty() { - return DefaultScheduler.place(nodes, request); - } - let active_by_node = self - .active_tasks - .iter() - .filter(|(tenant, project, _, _, _)| { - tenant == &request.tenant && project == &request.project - }) - .fold(BTreeMap::::new(), |mut counts, key| { - *counts.entry(key.3.clone()).or_default() += 1; - counts - }); - let mut selected = select_task_placement(&candidates, &active_by_node) - .expect("one or more compatible task-placement candidates should remain"); - let selected_load = active_by_node - .get(&selected.node) - .copied() - .unwrap_or_default(); - if candidates.iter().any(|candidate| { - candidate.score == selected.score - && active_by_node - .get(&candidate.node) - .copied() - .unwrap_or_default() - > selected_load - }) { - selected.reasons.push(format!( - "least active equal-locality node ({selected_load} active assignment(s))" - )); - } - Ok(selected) - } - pub(super) fn capture_task_restart_checkpoint( &mut self, assignment: &TaskAssignment, @@ -118,14 +49,14 @@ impl CoordinatorService { let mut objects = BTreeMap::new(); let mut missing_required_artifact = false; for artifact in &task_spec.required_artifacts { - let Some(metadata) = - self.artifact_registry - .metadata(&assignment.tenant, &assignment.project, artifact) - else { + let Some(metadata) = self.artifact_registry.metadata(artifact) else { missing_required_artifact = true; continue; }; - if metadata.retaining_nodes.is_empty() { + if metadata.tenant != assignment.tenant + || metadata.project != assignment.project + || metadata.retaining_nodes.is_empty() + { missing_required_artifact = true; continue; } @@ -483,14 +414,17 @@ impl CoordinatorService { .validate() .map_err(CoordinatorServiceError::Protocol)?; for artifact in &task_spec.required_artifacts { - let metadata = self - .artifact_registry - .metadata(&tenant, &project, artifact) - .ok_or_else(|| { - CoordinatorError::Unauthorized(format!( - "required artifact {artifact} is unavailable or has expired in this tenant/project scope" - )) - })?; + let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| { + CoordinatorError::Unauthorized(format!( + "required artifact {artifact} is unavailable or has expired" + )) + })?; + if metadata.tenant != tenant || metadata.project != project { + return Err(CoordinatorError::Unauthorized(format!( + "required artifact {artifact} is outside the task tenant/project scope" + )) + .into()); + } if metadata.retaining_nodes.is_empty() { return Err(CoordinatorError::Unauthorized(format!( "required artifact {artifact} has no retaining node" @@ -522,7 +456,7 @@ impl CoordinatorService { prefer_node: None, }; let nodes = self.live_node_descriptors(); - let placement = match self.place_workflow_task(&nodes, &request) { + let placement = match DefaultScheduler.place(&nodes, &request) { Ok(placement) => placement, Err(err) if wait_for_node => { let reason = if err.message.is_empty() { @@ -545,22 +479,13 @@ impl CoordinatorService { task_spec, wasm_module_base64, }); - let queued_tasks = self - .pending_task_launches - .iter() - .filter(|pending| { - pending.tenant == tenant - && pending.project == project - && pending.process == process - }) - .count(); return Ok(CoordinatorResponse::TaskQueued { process, task, actor, reason, charged_spawns, - queued_tasks, + queued_tasks: self.pending_task_launches.len(), }); } Err(err) => return Err(err.into()), @@ -671,9 +596,7 @@ impl CoordinatorService { )); }; self.task_attempts.remove(&removable); - self.task_terminal_states.remove(&removable); } - self.task_terminal_states.remove(&key); let attempts = self.task_attempts.entry(key).or_default(); for attempt in attempts.iter_mut() { attempt.current = false; @@ -771,42 +694,3 @@ fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option Placement { - Placement { - node: NodeId::from(node), - score, - reasons: Vec::new(), - } - } - - #[test] - fn equal_locality_prefers_the_least_active_node() { - let candidates = vec![placement("busy", 40), placement("idle", 40)]; - let active = BTreeMap::from([(NodeId::from("busy"), 1)]); - - assert_eq!( - select_task_placement(&candidates, &active) - .expect("one placement") - .node, - NodeId::from("idle") - ); - } - - #[test] - fn locality_score_remains_stronger_than_load_balancing() { - let candidates = vec![placement("warm", 50), placement("cold", 40)]; - let active = BTreeMap::from([(NodeId::from("warm"), 2)]); - - assert_eq!( - select_task_placement(&candidates, &active) - .expect("one placement") - .node, - NodeId::from("warm") - ); - } -} diff --git a/crates/clusterflux-coordinator/src/service/processes.rs b/crates/clusterflux-coordinator/src/service/processes.rs index 646c607..1382c3c 100644 --- a/crates/clusterflux-coordinator/src/service/processes.rs +++ b/crates/clusterflux-coordinator/src/service/processes.rs @@ -7,7 +7,7 @@ use clusterflux_core::{ TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId, }; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::keys::{process_control_key, task_control_key}; use super::{ @@ -47,10 +47,14 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "task assignment poll is outside the enrolled tenant/project scope".to_owned(), + ) + .into()); + } let assignment_key = (tenant.clone(), project.clone(), node.clone()); let assignment = self .task_assignments @@ -73,11 +77,7 @@ impl CoordinatorService { project: &ProjectId, node: &NodeId, ) -> Result, CoordinatorServiceError> { - let Some(descriptor) = self - .node_descriptors - .get(&NodeScopeKey::from_refs(tenant, project, node)) - .cloned() - else { + let Some(descriptor) = self.node_descriptors.get(node).cloned() else { return Ok(None); }; let mut remaining = VecDeque::new(); @@ -233,18 +233,20 @@ impl CoordinatorService { let node = NodeId::new(node); let identity = self .coordinator - .node_identity(&tenant, &project, &node) + .node_identity(&node) .ok_or(CoordinatorError::UnknownNode)?; - debug_assert_eq!(identity.tenant, tenant); - debug_assert_eq!(identity.project, project); - let descriptor = self - .node_descriptors - .get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node)) - .ok_or_else(|| { - CoordinatorError::Unauthorized( - "source preparation completion requires a node capability report".to_owned(), - ) - })?; + if identity.tenant != tenant || identity.project != project { + return Err(CoordinatorError::Unauthorized( + "source preparation completion is outside the enrolled tenant/project scope" + .to_owned(), + ) + .into()); + } + let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| { + CoordinatorError::Unauthorized( + "source preparation completion requires a node capability report".to_owned(), + ) + })?; if !descriptor.source_snapshots.contains(&source_snapshot) && descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND { @@ -271,7 +273,6 @@ impl CoordinatorService { agent_signature: Option, request_payload_digest: Option<&Digest>, process: String, - launch_attempt: Option, restart: bool, ) -> Result { let tenant = TenantId::new(tenant); @@ -382,10 +383,6 @@ impl CoordinatorService { || attempt_project != &project || attempt_process != &process }); - self.task_terminal_states - .retain(|(task_tenant, task_project, task_process, _), _| { - task_tenant != &tenant || task_project != &project || task_process != &process - }); self.restart_launches .retain(|(attempt_tenant, attempt_project, attempt_process, _)| { attempt_tenant != &tenant @@ -395,19 +392,10 @@ impl CoordinatorService { self.debug_audit_events.retain(|event| { event.tenant != tenant || event.project != project || event.process != process }); - let active = self.coordinator.start_process_for_launch_attempt( - tenant.clone(), - project.clone(), - process.clone(), - launch_attempt.map(clusterflux_core::LaunchAttemptId::new), - ); - self.record_process_started(&tenant, &project, &process, now_epoch_seconds); + self.coordinator + .start_process(tenant, project, process.clone()); Ok(CoordinatorResponse::ProcessStarted { process, - launch_attempt: active - .launch_attempt - .as_ref() - .map(|attempt| attempt.as_str().to_owned()), epoch: self.coordinator.coordinator_epoch(), actor, charged_spawns, @@ -416,18 +404,14 @@ impl CoordinatorService { pub(super) fn handle_reconnect_node( &mut self, - tenant: String, - project: String, node: String, process: String, epoch: u64, ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); let node = NodeId::new(node); let process = ProcessId::new(process); self.coordinator - .reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?; + .reconnect_node(&node, Some((&process, epoch)))?; Ok(CoordinatorResponse::NodeReconnected { node, process }) } @@ -518,13 +502,6 @@ impl CoordinatorService { } let process_key = process_control_key(&tenant, &project, &process); if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) { - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Cancelled, - self.current_epoch_seconds()?, - ); self.coordinator .abort_process(&tenant, &project, &process)?; self.clear_operator_panel_state(&tenant, &project, &process); @@ -542,7 +519,6 @@ impl CoordinatorService { project: String, actor_user: String, process: String, - launch_attempt: Option, ) -> Result { let tenant = TenantId::new(tenant); let project = ProjectId::new(project); @@ -558,17 +534,6 @@ impl CoordinatorService { })?; debug_assert_eq!(active.tenant, tenant); debug_assert_eq!(active.project, project); - let launch_attempt = launch_attempt.map(clusterflux_core::LaunchAttemptId::new); - if let Some(expected) = launch_attempt.as_ref() { - if active.launch_attempt.as_ref() != Some(expected) { - return Err(CoordinatorError::Unauthorized(format!( - "launch rollback denied: attempt {} does not own process {}", - expected.as_str(), - process.as_str() - )) - .into()); - } - } let process_key = process_control_key(&tenant, &project, &process); self.process_cancellations.remove(&process_key); @@ -610,24 +575,8 @@ impl CoordinatorService { } } - self.record_process_terminal( - &tenant, - &project, - &process, - super::ProcessFinalResult::Cancelled, - self.current_epoch_seconds()?, - ); - if let Some(launch_attempt) = launch_attempt.as_ref() { - self.coordinator.abort_process_for_launch_attempt( - &tenant, - &project, - &process, - launch_attempt, - )?; - } else { - self.coordinator - .abort_process(&tenant, &project, &process)?; - } + self.coordinator + .abort_process(&tenant, &project, &process)?; let active_restart_tasks = aborted_tasks .iter() .map(|target| target.task.clone()) @@ -675,18 +624,10 @@ impl CoordinatorService { .map(|active| { let process_key = process_control_key(&active.tenant, &active.project, &active.id); let main = self.main_runtime.controls.get(&process_key); - let stored = self.process_summaries.get(&process_key); - let stored_main_state = stored - .and_then(|summary| summary.main_terminal_state.as_ref()) - .map(|state| match state { - super::TaskTerminalState::Completed => "completed", - super::TaskTerminalState::Failed => "failed", - super::TaskTerminalState::Cancelled => "cancelled", - }); let state = if self.process_cancellations.contains(&process_key) { "cancelling" } else { - "running" + main.map_or("running", |main| main.state.as_str()) }; let main_wait_state = main.and_then(|main| { if main.state != "running" { @@ -711,15 +652,9 @@ impl CoordinatorService { VirtualProcessStatus { process: active.id, state: state.to_owned(), - main_task_definition: main.map(|main| main.task_definition.clone()).or_else( - || stored.and_then(|summary| summary.main_task_definition.clone()), - ), - main_task_instance: main - .map(|main| main.task_instance.clone()) - .or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())), - main_state: main - .map(|main| main.state.clone()) - .or_else(|| stored_main_state.map(str::to_owned)), + main_task_definition: main.map(|main| main.task_definition.clone()), + main_task_instance: main.map(|main| main.task_instance.clone()), + main_state: main.map(|main| main.state.clone()), main_wait_state, main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()), connected_nodes: active.connected_nodes.into_iter().collect(), diff --git a/crates/clusterflux-coordinator/src/service/protocol.rs b/crates/clusterflux-coordinator/src/service/protocol.rs index 61faf98..3a3204e 100644 --- a/crates/clusterflux-coordinator/src/service/protocol.rs +++ b/crates/clusterflux-coordinator/src/service/protocol.rs @@ -2,11 +2,10 @@ use std::collections::BTreeMap; use clusterflux_core::{ AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind, - DataPlaneObject, DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, - EnvironmentRequirements, LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, - NodeEndpoint, NodeId, NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, - ProjectId, ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryHandle, - TaskBoundaryValue, TaskDefinitionId, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, + DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements, + LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest, + PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation, + SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, UserId, VfsPath, }; use serde::{Deserialize, Serialize}; @@ -21,8 +20,6 @@ use crate::{AgentPublicKeyRecord, ProjectRecord}; pub struct TaskReplacementBundle { pub bundle_digest: Digest, pub wasm_module_base64: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_snapshot: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -121,8 +118,6 @@ pub enum CoordinatorRequest { enrollment_grant: String, }, NodeHeartbeat { - tenant: String, - project: String, node: String, #[serde(default)] node_signature: Option, @@ -150,15 +145,6 @@ pub enum CoordinatorRequest { project: String, actor_user: String, }, - ListNodeSummaries { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, RevokeNodeCredential { tenant: String, project: String, @@ -274,14 +260,10 @@ pub enum CoordinatorRequest { #[serde(default)] agent_signature: Option, process: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - launch_attempt: Option, #[serde(default)] restart: bool, }, ReconnectNode { - tenant: String, - project: String, node: String, process: String, epoch: u64, @@ -304,23 +286,12 @@ pub enum CoordinatorRequest { project: String, actor_user: String, process: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - launch_attempt: Option, }, ListProcesses { tenant: String, project: String, actor_user: String, }, - ListProcessSummaries { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, QuotaStatus { tenant: String, project: String, @@ -361,8 +332,6 @@ pub enum CoordinatorRequest { project: String, actor_user: String, process: String, - #[serde(default)] - revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { @@ -447,19 +416,6 @@ pub enum CoordinatorRequest { stderr_truncated: bool, backpressured: bool, }, - ReportTaskLogChunk { - tenant: String, - project: String, - process: String, - node: String, - task: String, - stream: TaskLogStream, - offset: u64, - source_bytes: u64, - text: String, - #[serde(default)] - truncated: bool, - }, ReportVfsMetadata { tenant: String, project: String, @@ -509,18 +465,6 @@ pub enum CoordinatorRequest { actor_user: String, process: String, }, - ListRecentLogs { - tenant: String, - project: String, - actor_user: String, - process: String, - #[serde(default)] - task: Option, - #[serde(default)] - after_sequence: Option, - #[serde(default = "default_log_page_limit")] - limit: u32, - }, JoinTask { tenant: String, project: String, @@ -553,23 +497,6 @@ pub enum CoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, - ListArtifacts { - tenant: String, - project: String, - actor_user: String, - #[serde(default)] - process: Option, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, - GetArtifact { - tenant: String, - project: String, - actor_user: String, - artifact: String, - }, OpenArtifactDownloadStream { tenant: String, project: String, @@ -598,10 +525,6 @@ pub enum CoordinatorRequest { } impl CoordinatorRequest { - pub fn validate_external_identifiers(&self) -> Result<(), String> { - validate_coordinator_request(self, "request") - } - pub fn operation(&self) -> Result { serde_json::to_value(self) .map_err(|err| format!("failed to encode coordinator request operation: {err}")) @@ -609,1225 +532,6 @@ impl CoordinatorRequest { } } -fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Result<(), String> { - match request { - CoordinatorRequest::Ping => Ok(()), - CoordinatorRequest::Authenticated { - session_secret, - request, - } => { - validate_external_token(session_secret, &format!("{path}.session_secret"), 512)?; - validate_authenticated_request(request, &format!("{path}.request")) - } - CoordinatorRequest::AuthStatus { - tenant, - project, - actor_user, - } - | CoordinatorRequest::CreateProject { - tenant, - project, - actor_user, - .. - } - | CoordinatorRequest::SelectProject { - tenant, - project, - actor_user, - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_project(project, &format!("{path}.project"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListProjects { tenant, actor_user } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::AdminStatus { - tenant, - actor_user, - admin_nonce, - .. - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) - } - CoordinatorRequest::SuspendTenant { - tenant, - actor_user, - target_tenant, - admin_nonce, - .. - } => { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_tenant(target_tenant, &format!("{path}.target_tenant"))?; - validate_external_token(admin_nonce, &format!("{path}.admin_nonce"), 256) - } - CoordinatorRequest::RegisterAgentPublicKey { - tenant, - project, - user, - agent, - public_key, - } - | CoordinatorRequest::RotateAgentPublicKey { - tenant, - project, - user, - agent, - public_key, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user"))?; - validate_agent(agent, &format!("{path}.agent"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - CoordinatorRequest::ListAgentPublicKeys { - tenant, - project, - user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user")) - } - CoordinatorRequest::RevokeAgentPublicKey { - tenant, - project, - user, - agent, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(user, &format!("{path}.user"))?; - validate_agent(agent, &format!("{path}.agent")) - } - CoordinatorRequest::AttachNode { - tenant, - project, - node, - public_key, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - CoordinatorRequest::CreateNodeEnrollmentGrant { - tenant, - project, - actor_user, - .. - } - | CoordinatorRequest::ListNodeDescriptors { - tenant, - project, - actor_user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListNodeSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::ExchangeNodeEnrollmentGrant { - tenant, - project, - node, - public_key, - enrollment_grant, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024)?; - validate_external_token(enrollment_grant, &format!("{path}.enrollment_grant"), 512) - } - CoordinatorRequest::NodeHeartbeat { - tenant, - project, - node, - node_signature, - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - if let Some(signature) = node_signature { - validate_node_signature(signature, &format!("{path}.node_signature"))?; - } - Ok(()) - } - CoordinatorRequest::SignedNode { - node, - node_signature, - request, - } => { - validate_node(node, &format!("{path}.node"))?; - validate_node_signature(node_signature, &format!("{path}.node_signature"))?; - validate_coordinator_request(request, &format!("{path}.request")) - } - CoordinatorRequest::ReportNodeCapabilities { - tenant, - project, - node, - artifact_locations, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_artifact_array(artifact_locations, &format!("{path}.artifact_locations")) - } - CoordinatorRequest::RevokeNodeCredential { - tenant, - project, - actor_user, - node, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_node(node, &format!("{path}.node")) - } - CoordinatorRequest::ScheduleTask { - tenant, - project, - required_artifacts, - prefer_node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; - validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) - } - CoordinatorRequest::LaunchTask { - tenant, - project, - actor_user, - actor_agent, - agent_signature, - task_spec, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; - validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; - if let Some(signature) = agent_signature { - validate_agent_signature(signature, &format!("{path}.agent_signature"))?; - } - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - CoordinatorRequest::LaunchChildTask { - tenant, - project, - process, - node, - parent_task, - task_spec, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(parent_task, &format!("{path}.parent_task"))?; - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - CoordinatorRequest::JoinChildTask { - tenant, - project, - process, - node, - parent_task, - task, - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(parent_task, &format!("{path}.parent_task"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::PollTaskAssignment { - tenant, - project, - node, - } - | CoordinatorRequest::PollArtifactTransfer { - tenant, - project, - node, - } - | CoordinatorRequest::CompleteSourcePreparation { - tenant, - project, - node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node")) - } - CoordinatorRequest::UploadArtifactTransferChunk { - tenant, - project, - node, - transfer_id, - artifact, - .. - } - | CoordinatorRequest::FailArtifactTransfer { - tenant, - project, - node, - transfer_id, - artifact, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_external_token(transfer_id, &format!("{path}.transfer_id"), 256)?; - validate_artifact(artifact, &format!("{path}.artifact")) - } - CoordinatorRequest::RequestRendezvous { - scope, - source, - destination, - .. - } => { - validate_data_plane_scope(scope, &format!("{path}.scope"))?; - validate_node_endpoint(source, &format!("{path}.source"))?; - validate_node_endpoint(destination, &format!("{path}.destination")) - } - CoordinatorRequest::RequestSourcePreparation { - tenant, project, .. - } => validate_tenant_project(tenant, project, path), - CoordinatorRequest::StartProcess { - tenant, - project, - actor_user, - actor_agent, - agent_signature, - process, - launch_attempt, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_optional_user(actor_user.as_deref(), &format!("{path}.actor_user"))?; - validate_optional_agent(actor_agent.as_deref(), &format!("{path}.actor_agent"))?; - if let Some(signature) = agent_signature { - validate_agent_signature(signature, &format!("{path}.agent_signature"))?; - } - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - CoordinatorRequest::ReconnectNode { - tenant, - project, - node, - process, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_node(node, &format!("{path}.node"))?; - validate_process(process, &format!("{path}.process")) - } - CoordinatorRequest::CancelTask { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::PollTaskControl { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::PollDebugCommand { - tenant, - project, - process, - node, - task, - } - | CoordinatorRequest::ReportDebugProbeHit { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportTaskLog { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportVfsMetadata { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::TaskCompleted { - tenant, - project, - process, - node, - task, - .. - } - | CoordinatorRequest::ReportDebugState { - tenant, - project, - process, - node, - task, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_node(node, &format!("{path}.node"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::CancelProcess { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::DebugAttach { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::SetDebugBreakpoints { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::InspectDebugBreakpoints { - tenant, - project, - actor_user, - process, - } - | CoordinatorRequest::ResumeDebugEpoch { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::InspectDebugEpoch { - tenant, - project, - actor_user, - process, - .. - } - | CoordinatorRequest::ListTaskSnapshots { - tenant, - project, - actor_user, - process, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process")) - } - CoordinatorRequest::ListRecentLogs { - tenant, - project, - actor_user, - process, - task, - limit, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - if let Some(task) = task { - validate_task_instance(task, &format!("{path}.task"))?; - } - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::AbortProcess { - tenant, - project, - actor_user, - process, - launch_attempt, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - CoordinatorRequest::ListProcesses { - tenant, - project, - actor_user, - } - | CoordinatorRequest::QuotaStatus { - tenant, - project, - actor_user, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::ListProcessSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 100) - } - CoordinatorRequest::RestartTask { - tenant, - project, - actor_user, - process, - task, - .. - } - | CoordinatorRequest::ResolveTaskFailure { - tenant, - project, - actor_user, - process, - task, - .. - } - | CoordinatorRequest::JoinTask { - tenant, - project, - actor_user, - process, - task, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(task, &format!("{path}.task")) - } - CoordinatorRequest::CreateDebugEpoch { - tenant, - project, - actor_user, - process, - stopped_task, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(stopped_task, &format!("{path}.stopped_task")) - } - CoordinatorRequest::ListTaskEvents { - tenant, - project, - actor_user, - process, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_process(process.as_deref(), &format!("{path}.process")) - } - CoordinatorRequest::RenderOperatorPanel { - tenant, - project, - process, - actor_user, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_user(actor_user, &format!("{path}.actor_user")) - } - CoordinatorRequest::SubmitPanelEvent { - tenant, - project, - process, - widget_id, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_process(process, &format!("{path}.process"))?; - validate_external_token(widget_id, &format!("{path}.widget_id"), 256) - } - CoordinatorRequest::ListArtifacts { - tenant, - project, - actor_user, - process, - cursor, - limit, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_optional_process(process.as_deref(), &format!("{path}.process"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - CoordinatorRequest::CreateArtifactDownloadLink { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::OpenArtifactDownloadStream { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::RevokeArtifactDownloadLink { - tenant, - project, - actor_user, - artifact, - .. - } - | CoordinatorRequest::GetArtifact { - tenant, - project, - actor_user, - artifact, - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_artifact(artifact, &format!("{path}.artifact")) - } - CoordinatorRequest::ExportArtifactToNode { - tenant, - project, - actor_user, - artifact, - receiver_node, - .. - } => { - validate_tenant_project(tenant, project, path)?; - validate_user(actor_user, &format!("{path}.actor_user"))?; - validate_artifact(artifact, &format!("{path}.artifact"))?; - validate_node(receiver_node, &format!("{path}.receiver_node")) - } - } -} - -fn validate_authenticated_request( - request: &AuthenticatedCoordinatorRequest, - path: &str, -) -> Result<(), String> { - match request { - AuthenticatedCoordinatorRequest::AuthStatus - | AuthenticatedCoordinatorRequest::RevokeCliSession - | AuthenticatedCoordinatorRequest::ListProjects - | AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } - | AuthenticatedCoordinatorRequest::ListNodeDescriptors - | AuthenticatedCoordinatorRequest::ListProcesses - | AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()), - AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => { - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => { - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 100) - } - AuthenticatedCoordinatorRequest::CreateProject { project, .. } - | AuthenticatedCoordinatorRequest::SelectProject { project } => { - validate_project(project, &format!("{path}.project")) - } - AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { - agent, public_key, .. - } - | AuthenticatedCoordinatorRequest::RotateAgentPublicKey { - agent, public_key, .. - } => { - validate_agent(agent, &format!("{path}.agent"))?; - validate_external_token(public_key, &format!("{path}.public_key"), 1024) - } - AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Ok(()), - AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => { - validate_agent(agent, &format!("{path}.agent")) - } - AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => { - validate_node(node, &format!("{path}.node")) - } - AuthenticatedCoordinatorRequest::StartProcess { - process, - launch_attempt, - .. - } - | AuthenticatedCoordinatorRequest::AbortProcess { - process, - launch_attempt, - } => { - validate_process(process, &format!("{path}.process"))?; - validate_optional_launch_attempt( - launch_attempt.as_deref(), - &format!("{path}.launch_attempt"), - ) - } - AuthenticatedCoordinatorRequest::ScheduleTask { - required_artifacts, - prefer_node, - .. - } => { - validate_artifact_array(required_artifacts, &format!("{path}.required_artifacts"))?; - validate_optional_node(prefer_node.as_deref(), &format!("{path}.prefer_node")) - } - AuthenticatedCoordinatorRequest::LaunchTask { task_spec, .. } => { - validate_task_spec(task_spec, &format!("{path}.task_spec")) - } - AuthenticatedCoordinatorRequest::CancelProcess { process } - | AuthenticatedCoordinatorRequest::DebugAttach { process } - | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { process, .. } - | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } - | AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, .. } - | AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, .. } - | AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => { - validate_process(process, &format!("{path}.process")) - } - AuthenticatedCoordinatorRequest::ListRecentLogs { - process, - task, - limit, - .. - } => { - validate_process(process, &format!("{path}.process"))?; - if let Some(task) = task { - validate_task_instance(task, &format!("{path}.task"))?; - } - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::RestartTask { process, task, .. } - | AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. } - | AuthenticatedCoordinatorRequest::JoinTask { process, task } => { - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(task, &format!("{path}.task")) - } - AuthenticatedCoordinatorRequest::CreateDebugEpoch { - process, - stopped_task, - .. - } => { - validate_process(process, &format!("{path}.process"))?; - validate_task_instance(stopped_task, &format!("{path}.stopped_task")) - } - AuthenticatedCoordinatorRequest::ListTaskEvents { process } => { - validate_optional_process(process.as_deref(), &format!("{path}.process")) - } - AuthenticatedCoordinatorRequest::ListArtifacts { - process, - cursor, - limit, - } => { - validate_optional_process(process.as_deref(), &format!("{path}.process"))?; - validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?; - validate_page_limit(*limit, &format!("{path}.limit"), 200) - } - AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. } - | AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. } - | AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } - | AuthenticatedCoordinatorRequest::GetArtifact { artifact } => { - validate_artifact(artifact, &format!("{path}.artifact")) - } - AuthenticatedCoordinatorRequest::ExportArtifactToNode { - artifact, - receiver_node, - .. - } => { - validate_artifact(artifact, &format!("{path}.artifact"))?; - validate_node(receiver_node, &format!("{path}.receiver_node")) - } - } -} - -fn validate_task_spec(task_spec: &TaskSpec, path: &str) -> Result<(), String> { - validate_existing_id( - &task_spec.tenant, - &format!("{path}.tenant"), - TenantId::try_new, - )?; - validate_existing_id( - &task_spec.project, - &format!("{path}.project"), - ProjectId::try_new, - )?; - validate_existing_id( - &task_spec.process, - &format!("{path}.process"), - ProcessId::try_new, - )?; - validate_existing_id( - &task_spec.task_definition, - &format!("{path}.task_definition"), - TaskDefinitionId::try_new, - )?; - validate_existing_id( - &task_spec.task_instance, - &format!("{path}.task_instance"), - TaskInstanceId::try_new, - )?; - if let Some(environment_id) = &task_spec.environment_id { - validate_external_token(environment_id, &format!("{path}.environment_id"), 128)?; - } - for (index, artifact) in task_spec.required_artifacts.iter().enumerate() { - validate_existing_id( - artifact, - &format!("{path}.required_artifacts[{index}]"), - ArtifactId::try_new, - )?; - } - for (argument_index, argument) in task_spec.args.iter().enumerate() { - validate_task_boundary_value(argument, &format!("{path}.args[{argument_index}]"))?; - } - Ok(()) -} - -fn validate_task_boundary_value(value: &TaskBoundaryValue, path: &str) -> Result<(), String> { - match value { - TaskBoundaryValue::Artifact(artifact) => validate_existing_id( - &artifact.id, - &format!("{path}.artifact.id"), - ArtifactId::try_new, - ), - TaskBoundaryValue::Structured(structured) => { - for (index, handle) in structured.handles.iter().enumerate() { - if let TaskBoundaryHandle::Artifact(artifact) = handle { - validate_existing_id( - &artifact.id, - &format!("{path}.handles[{index}].artifact.id"), - ArtifactId::try_new, - )?; - } - } - Ok(()) - } - TaskBoundaryValue::SmallJson(_) - | TaskBoundaryValue::SourceSnapshot(_) - | TaskBoundaryValue::Blob(_) - | TaskBoundaryValue::VfsManifest(_) => Ok(()), - } -} - -fn validate_data_plane_scope(scope: &DataPlaneScope, path: &str) -> Result<(), String> { - validate_existing_id(&scope.tenant, &format!("{path}.tenant"), TenantId::try_new)?; - validate_existing_id( - &scope.project, - &format!("{path}.project"), - ProjectId::try_new, - )?; - validate_existing_id( - &scope.process, - &format!("{path}.process"), - ProcessId::try_new, - )?; - if let DataPlaneObject::Artifact(artifact) = &scope.object { - validate_existing_id( - artifact, - &format!("{path}.object.artifact"), - ArtifactId::try_new, - )?; - } - validate_external_token( - &scope.authorization_subject, - &format!("{path}.authorization_subject"), - 512, - ) -} - -fn validate_node_endpoint(endpoint: &NodeEndpoint, path: &str) -> Result<(), String> { - validate_existing_id(&endpoint.node, &format!("{path}.node"), NodeId::try_new)?; - validate_external_token( - &endpoint.advertised_addr, - &format!("{path}.advertised_addr"), - 512, - ) -} - -fn validate_node_signature(signature: &NodeSignedRequest, path: &str) -> Result<(), String> { - validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; - validate_external_token(&signature.signature, &format!("{path}.signature"), 512) -} - -fn validate_agent_signature(signature: &AgentSignedRequest, path: &str) -> Result<(), String> { - validate_external_token(&signature.nonce, &format!("{path}.nonce"), 256)?; - validate_external_token(&signature.signature, &format!("{path}.signature"), 512) -} - -fn validate_tenant_project(tenant: &str, project: &str, path: &str) -> Result<(), String> { - validate_tenant(tenant, &format!("{path}.tenant"))?; - validate_project(project, &format!("{path}.project")) -} - -fn validate_artifact_array(values: &[String], path: &str) -> Result<(), String> { - for (index, value) in values.iter().enumerate() { - validate_artifact(value, &format!("{path}[{index}]"))?; - } - Ok(()) -} - -fn validate_optional_user(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_user(value, path)) -} - -fn validate_optional_agent(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_agent(value, path)) -} - -fn validate_optional_node(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_node(value, path)) -} - -fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_process(value, path)) -} - -fn validate_optional_cursor(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_external_token(value, path, 256)) -} - -fn validate_page_limit(value: u32, path: &str, maximum: u32) -> Result<(), String> { - if value == 0 || value > maximum { - return Err(format!( - "malformed pagination limit {path}: expected 1 through {maximum}, received {value}" - )); - } - Ok(()) -} - -fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> { - value.map_or(Ok(()), |value| validate_launch_attempt(value, path)) -} - -fn validate_tenant(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, TenantId::try_new) -} - -fn validate_project(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ProjectId::try_new) -} - -fn validate_user(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, UserId::try_new) -} - -fn validate_agent(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, AgentId::try_new) -} - -fn validate_node(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, NodeId::try_new) -} - -fn validate_process(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ProcessId::try_new) -} - -fn validate_task_instance(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, TaskInstanceId::try_new) -} - -fn validate_artifact(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, ArtifactId::try_new) -} - -fn validate_launch_attempt(value: &str, path: &str) -> Result<(), String> { - validate_external_id(value, path, LaunchAttemptId::try_new) -} - -fn validate_external_id( - value: &str, - path: &str, - parser: impl FnOnce(String) -> Result, -) -> Result<(), String> -where - E: std::fmt::Display, -{ - parser(value.to_owned()) - .map(|_| ()) - .map_err(|error| format!("malformed external identifier {path}: {error}")) -} - -fn validate_existing_id( - value: &T, - path: &str, - parser: impl FnOnce(String) -> Result, -) -> Result<(), String> -where - T: std::fmt::Display, - E: std::fmt::Display, -{ - validate_external_id(&value.to_string(), path, parser) -} - -fn validate_external_token(value: &str, path: &str, max_bytes: usize) -> Result<(), String> { - clusterflux_core::validate_opaque_token(value, max_bytes) - .map_err(|error| format!("malformed external token {path}: {error}")) -} - -#[cfg(test)] -mod external_identifier_tests { - use super::*; - - fn valid_task_spec() -> TaskSpec { - TaskSpec { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - task_definition: TaskDefinitionId::from("compile"), - task_instance: TaskInstanceId::from("compile-1"), - dispatch: clusterflux_core::TaskDispatch::CoordinatorNodeWasm { - export: Some("compile".to_owned()), - abi: clusterflux_core::WasmExportAbi::TaskV1, - }, - environment_id: Some("linux-rootless".to_owned()), - environment: None, - environment_digest: None, - required_capabilities: Default::default(), - dependency_cache: None, - source_snapshot: None, - required_artifacts: vec![ArtifactId::from("input-artifact")], - args: Vec::new(), - vfs_epoch: 1, - failure_policy: Default::default(), - bundle_digest: None, - } - } - - fn validation_error(value: serde_json::Value) -> String { - match serde_json::from_value::(value) { - Ok(request) => request - .validate_external_identifiers() - .expect_err("request should contain one malformed external identifier"), - Err(error) => error.to_string(), - } - } - - #[test] - fn nested_authenticated_identifiers_are_validated() { - let request = CoordinatorRequest::Authenticated { - session_secret: "cli_session_valid".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "bad process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - }, - }; - let error = request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("request.request.process")); - } - - #[test] - fn opaque_secrets_are_bounded_as_tokens_instead_of_object_ids() { - let request = CoordinatorRequest::Authenticated { - session_secret: "opaque secret/+==".to_owned(), - request: AuthenticatedCoordinatorRequest::AuthStatus, - }; - request.validate_external_identifiers().unwrap(); - - let mut request = request; - let CoordinatorRequest::Authenticated { session_secret, .. } = &mut request else { - unreachable!() - }; - *session_secret = "bad\0secret".to_owned(); - let error = request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.session_secret")); - } - - #[test] - fn real_protocol_variants_reject_exactly_one_malformed_nested_identifier() { - let launch = CoordinatorRequest::LaunchTask { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: Some("user".to_owned()), - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - task_spec: valid_task_spec(), - wait_for_node: false, - artifact_path: "/vfs/artifacts/output".to_owned(), - wasm_module_base64: "AGFzbQEAAAA=".to_owned(), - }; - - let mut malformed_definition = serde_json::to_value(&launch).unwrap(); - malformed_definition["task_spec"]["task_definition"] = - serde_json::Value::String("bad task definition!".to_owned()); - let error = validation_error(malformed_definition); - assert!(error.contains("TaskDefinitionId is invalid")); - - let mut malformed_artifact = serde_json::to_value(&launch).unwrap(); - malformed_artifact["task_spec"]["required_artifacts"][0] = - serde_json::Value::String("bad artifact!".to_owned()); - let error = validation_error(malformed_artifact); - assert!(error.contains("ArtifactId is invalid")); - - let rendezvous = CoordinatorRequest::RequestRendezvous { - scope: DataPlaneScope { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - object: DataPlaneObject::Artifact(ArtifactId::from("artifact")), - authorization_subject: "node-a-to-node-b".to_owned(), - }, - source: NodeEndpoint { - node: NodeId::from("node-a"), - advertised_addr: "node-a.invalid:4433".to_owned(), - public_key_fingerprint: Digest::sha256("node-a"), - }, - destination: NodeEndpoint { - node: NodeId::from("node-b"), - advertised_addr: "node-b.invalid:4433".to_owned(), - public_key_fingerprint: Digest::sha256("node-b"), - }, - direct_connectivity: true, - failure_reason: String::new(), - }; - let mut malformed_endpoint = serde_json::to_value(rendezvous).unwrap(); - malformed_endpoint["destination"]["node"] = - serde_json::Value::String("bad node!".to_owned()); - let error = validation_error(malformed_endpoint); - assert!(error.contains("NodeId is invalid")); - } - - #[test] - fn signed_and_authenticated_real_variants_validate_scoped_ids_and_tokens() { - let cases = [ - CoordinatorRequest::Authenticated { - session_secret: "session-secret".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "bad process!".to_owned(), - launch_attempt: Some("attempt".to_owned()), - }, - }, - CoordinatorRequest::StartProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: Some("bad agent!".to_owned()), - agent_public_key_fingerprint: Some(Digest::sha256("agent")), - agent_signature: Some(AgentSignedRequest { - nonce: "agent-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - process: "process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - restart: false, - }, - CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "bad node!".to_owned(), - node_signature: Some(NodeSignedRequest { - nonce: "node-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - }, - CoordinatorRequest::SignedNode { - node: "node".to_owned(), - node_signature: NodeSignedRequest { - nonce: "node-nonce".to_owned(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }, - request: Box::new(CoordinatorRequest::PollTaskAssignment { - tenant: "tenant".to_owned(), - project: "bad project!".to_owned(), - node: "node".to_owned(), - }), - }, - CoordinatorRequest::AbortProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - process: "process".to_owned(), - launch_attempt: Some("bad attempt!".to_owned()), - }, - ]; - - for request in cases { - let error = request.validate_external_identifiers().unwrap_err(); - assert!( - error.contains("malformed external identifier"), - "unexpected validation error for {request:?}: {error}" - ); - } - } - - #[test] - fn signed_request_nonces_are_validated_as_opaque_tokens() { - let agent_request = CoordinatorRequest::StartProcess { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: Some("agent".to_owned()), - agent_public_key_fingerprint: Some(Digest::sha256("agent")), - agent_signature: Some(AgentSignedRequest { - nonce: String::new(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - process: "process".to_owned(), - launch_attempt: Some("attempt".to_owned()), - restart: false, - }; - let error = agent_request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.agent_signature.nonce")); - - let node_request = CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - node_signature: Some(NodeSignedRequest { - nonce: String::new(), - issued_at_epoch_seconds: 1, - signature: "ed25519:syntactically-bounded".to_owned(), - }), - }; - let error = node_request.validate_external_identifiers().unwrap_err(); - assert!(error.contains("malformed external token request.node_signature.nonce")); - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum AuthenticatedCoordinatorRequest { @@ -1858,19 +562,11 @@ pub enum AuthenticatedCoordinatorRequest { ttl_seconds: u64, }, ListNodeDescriptors, - ListNodeSummaries { - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, RevokeNodeCredential { node: String, }, StartProcess { process: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - launch_attempt: Option, #[serde(default)] restart: bool, }, @@ -1896,16 +592,8 @@ pub enum AuthenticatedCoordinatorRequest { }, AbortProcess { process: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - launch_attempt: Option, }, ListProcesses, - ListProcessSummaries { - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, QuotaStatus, RestartTask { process: String, @@ -1923,8 +611,6 @@ pub enum AuthenticatedCoordinatorRequest { }, SetDebugBreakpoints { process: String, - #[serde(default)] - revision: u64, probe_symbols: Vec, }, InspectDebugBreakpoints { @@ -1950,15 +636,6 @@ pub enum AuthenticatedCoordinatorRequest { ListTaskSnapshots { process: String, }, - ListRecentLogs { - process: String, - #[serde(default)] - task: Option, - #[serde(default)] - after_sequence: Option, - #[serde(default = "default_log_page_limit")] - limit: u32, - }, JoinTask { process: String, task: String, @@ -1969,17 +646,6 @@ pub enum AuthenticatedCoordinatorRequest { #[serde(default = "default_download_ttl_seconds")] ttl_seconds: u64, }, - ListArtifacts { - #[serde(default)] - process: Option, - #[serde(default)] - cursor: Option, - #[serde(default = "default_page_limit")] - limit: u32, - }, - GetArtifact { - artifact: String, - }, OpenArtifactDownloadStream { artifact: String, max_bytes: u64, @@ -2002,14 +668,6 @@ fn default_download_ttl_seconds() -> u64 { 900 } -fn default_page_limit() -> u32 { - 50 -} - -fn default_log_page_limit() -> u32 { - 100 -} - fn default_node_enrollment_ttl_seconds() -> u64 { 900 } diff --git a/crates/clusterflux-coordinator/src/service/protocol/responses.rs b/crates/clusterflux-coordinator/src/service/protocol/responses.rs index 8245536..3fba996 100644 --- a/crates/clusterflux-coordinator/src/service/protocol/responses.rs +++ b/crates/clusterflux-coordinator/src/service/protocol/responses.rs @@ -183,121 +183,6 @@ pub struct VirtualProcessStatus { pub coordinator_epoch: u64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct NodeSummary { - pub id: NodeId, - pub display_name: String, - pub online: bool, - pub stale: bool, - pub last_seen_epoch_seconds: Option, - pub capabilities: NodeCapabilities, - pub direct_connectivity: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessLifecycleState { - Active, - RecentTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessActivityState { - Running, - WaitingForNode, - WaitingForTask, - AwaitingAction, - DebugEpochPartial, - Cancelling, - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProcessFinalResult { - Completed, - Failed, - Cancelled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct DebugEpochSummary { - pub epoch: u64, - pub command: String, - pub fully_frozen: bool, - pub partially_frozen: bool, - pub fully_resumed: bool, - pub failed: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessSummary { - pub process: ProcessId, - pub lifecycle: ProcessLifecycleState, - pub activity: ProcessActivityState, - pub main_wait_state: Option, - pub started_at_epoch_seconds: u64, - pub ended_at_epoch_seconds: Option, - pub final_result: Option, - pub connected_nodes: Vec, - pub current_debug_epoch: Option, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactAvailability { - Available, - NodeOffline, - Unavailable, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ArtifactRetentionState { - NodeRetained, - ExplicitStorage, - Lost, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ArtifactSummary { - pub id: ArtifactId, - pub display_path: String, - pub display_name: String, - pub process: ProcessId, - pub producer_task: TaskInstanceId, - pub safe_node: Option, - pub digest: Digest, - pub size_bytes: u64, - pub availability: ArtifactAvailability, - pub downloadable_now: bool, - pub retention_state: ArtifactRetentionState, - pub explicit_storage: bool, - pub order_cursor: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TaskLogStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RecentLogEntry { - pub sequence: u64, - pub process: ProcessId, - pub task: TaskInstanceId, - pub stream: TaskLogStream, - pub text: String, - pub server_timestamp_epoch_seconds: u64, - pub truncated: bool, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum SourcePreparationDisposition { Pending { reason: String }, @@ -328,7 +213,7 @@ pub enum CoordinatorResponse { manual_review: bool, sanitized_reason: Option, next_actions: Vec, - sensitive_moderation_details_exposed: bool, + private_moderation_details_exposed: bool, signup_failure_details_exposed: bool, }, AdminStatus { @@ -397,11 +282,6 @@ pub enum CoordinatorResponse { descriptors: Vec, actor: UserId, }, - NodeSummaries { - nodes: Vec, - next_cursor: Option, - actor: UserId, - }, NodeCredentialRevoked { node: NodeId, tenant: TenantId, @@ -464,8 +344,6 @@ pub enum CoordinatorResponse { }, ProcessStarted { process: ProcessId, - #[serde(default, skip_serializing_if = "Option::is_none")] - launch_attempt: Option, epoch: u64, actor: WorkflowActor, charged_spawns: u64, @@ -493,11 +371,6 @@ pub enum CoordinatorResponse { processes: Vec, actor: UserId, }, - ProcessSummaries { - processes: Vec, - next_cursor: Option, - actor: UserId, - }, QuotaStatus { tenant: TenantId, project: ProjectId, @@ -555,7 +428,6 @@ pub enum CoordinatorResponse { DebugBreakpoints { process: ProcessId, actor: UserId, - revision: u64, probe_symbols: Vec, hit_epoch: Option, hit_task: Option, @@ -608,17 +480,6 @@ pub enum CoordinatorResponse { stderr_tail: String, backpressured: bool, }, - TaskLogChunkRecorded { - process: ProcessId, - task: TaskInstanceId, - sequence: Option, - next_offset: u64, - }, - RecentLogs { - entries: Vec, - next_sequence: Option, - history_truncated: bool, - }, VfsMetadataRecorded { process: ProcessId, task: TaskInstanceId, @@ -655,13 +516,6 @@ pub enum CoordinatorResponse { ArtifactDownloadLink { link: DownloadLink, }, - Artifacts { - artifacts: Vec, - next_cursor: Option, - }, - Artifact { - artifact: ArtifactSummary, - }, ArtifactDownloadLinkRevoked { link: DownloadLink, }, @@ -686,24 +540,6 @@ pub enum CoordinatorResponse { artifact_size_bytes: u64, }, Error { - #[serde(flatten)] - error: clusterflux_core::ApiError, + message: String, }, } - -impl CoordinatorResponse { - pub fn error(request_id: impl Into, message: impl Into) -> Self { - Self::Error { - error: clusterflux_core::ApiError::from_message(request_id, message), - } - } - - pub fn service_error( - request_id: impl Into, - error: &crate::service::CoordinatorServiceError, - ) -> Self { - Self::Error { - error: error.api_error(request_id), - } - } -} diff --git a/crates/clusterflux-coordinator/src/service/quota.rs b/crates/clusterflux-coordinator/src/service/quota.rs index 78b65ae..6fcd90f 100644 --- a/crates/clusterflux-coordinator/src/service/quota.rs +++ b/crates/clusterflux-coordinator/src/service/quota.rs @@ -260,6 +260,22 @@ impl CoordinatorQuota { self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds) } + pub(super) fn can_charge_log_bytes( + &self, + tenant: &TenantId, + project: &ProjectId, + bytes: u64, + now_epoch_seconds: u64, + ) -> Result<(), LimitError> { + self.can_charge( + tenant, + project, + LimitKind::LogBytes, + bytes, + now_epoch_seconds, + ) + } + pub(super) fn charge_log_bytes( &mut self, tenant: &TenantId, diff --git a/crates/clusterflux-coordinator/src/service/relay.rs b/crates/clusterflux-coordinator/src/service/relay.rs index b79db3d..cbe92dd 100644 --- a/crates/clusterflux-coordinator/src/service/relay.rs +++ b/crates/clusterflux-coordinator/src/service/relay.rs @@ -65,16 +65,6 @@ pub struct ArtifactRelayUsage { pub egress_bytes: u64, pub abandoned_or_failed_bytes: u64, pub reserved_bytes: u64, - pub lifetime_ingress_bytes: u64, - pub lifetime_egress_bytes: u64, - pub lifetime_abandoned_or_failed_bytes: u64, - pub completed_transfers: u64, - pub failed_transfers: u64, - pub cancelled_transfers: u64, - pub expired_transfers: u64, - pub tracked_scopes: usize, - pub max_active_global: usize, - pub max_tracked_scopes: usize, } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -87,20 +77,6 @@ pub struct ArtifactRelayDurableState { pub ingress_used: u64, pub egress_used: u64, pub abandoned_or_failed_used: u64, - #[serde(default)] - pub lifetime_ingress_bytes: u64, - #[serde(default)] - pub lifetime_egress_bytes: u64, - #[serde(default)] - pub lifetime_abandoned_or_failed_bytes: u64, - #[serde(default)] - pub completed_transfers: u64, - #[serde(default)] - pub failed_transfers: u64, - #[serde(default)] - pub cancelled_transfers: u64, - #[serde(default)] - pub expired_transfers: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -169,13 +145,6 @@ pub(super) struct ArtifactRelayLedger { ingress_used: u64, egress_used: u64, abandoned_or_failed_used: u64, - lifetime_ingress_bytes: u64, - lifetime_egress_bytes: u64, - lifetime_abandoned_or_failed_bytes: u64, - completed_transfers: u64, - failed_transfers: u64, - cancelled_transfers: u64, - expired_transfers: u64, } impl Default for ArtifactRelayLedger { @@ -196,13 +165,6 @@ impl ArtifactRelayLedger { ingress_used: 0, egress_used: 0, abandoned_or_failed_used: 0, - lifetime_ingress_bytes: 0, - lifetime_egress_bytes: 0, - lifetime_abandoned_or_failed_bytes: 0, - completed_transfers: 0, - failed_transfers: 0, - cancelled_transfers: 0, - expired_transfers: 0, } } @@ -256,13 +218,6 @@ impl ArtifactRelayLedger { ingress_used: state.ingress_used, egress_used: state.egress_used, abandoned_or_failed_used: state.abandoned_or_failed_used, - lifetime_ingress_bytes: state.lifetime_ingress_bytes, - lifetime_egress_bytes: state.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes, - completed_transfers: state.completed_transfers, - failed_transfers: state.failed_transfers, - cancelled_transfers: state.cancelled_transfers, - expired_transfers: state.expired_transfers, } } @@ -305,13 +260,6 @@ impl ArtifactRelayLedger { ingress_used: self.ingress_used, egress_used: self.egress_used, abandoned_or_failed_used: self.abandoned_or_failed_used, - lifetime_ingress_bytes: self.lifetime_ingress_bytes, - lifetime_egress_bytes: self.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, - completed_transfers: self.completed_transfers, - failed_transfers: self.failed_transfers, - cancelled_transfers: self.cancelled_transfers, - expired_transfers: self.expired_transfers, } } @@ -543,11 +491,9 @@ impl ArtifactRelayLedger { if ingress { reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes); self.ingress_used = self.ingress_used.saturating_add(bytes); - self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes); } else { reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes); self.egress_used = self.egress_used.saturating_add(bytes); - self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes); } let project_key = ( reservation.scope.tenant.clone(), @@ -608,29 +554,10 @@ impl ArtifactRelayLedger { pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) { if let Some(reservation) = self.reservations.remove(key) { if reason != RelayFinishReason::Completed { - let abandoned_or_failed_bytes = reservation - .ingress_bytes - .saturating_add(reservation.egress_bytes); self.abandoned_or_failed_used = self .abandoned_or_failed_used - .saturating_add(abandoned_or_failed_bytes); - self.lifetime_abandoned_or_failed_bytes = self - .lifetime_abandoned_or_failed_bytes - .saturating_add(abandoned_or_failed_bytes); - } - match reason { - RelayFinishReason::Completed => { - self.completed_transfers = self.completed_transfers.saturating_add(1); - } - RelayFinishReason::Failed => { - self.failed_transfers = self.failed_transfers.saturating_add(1); - } - RelayFinishReason::Cancelled => { - self.cancelled_transfers = self.cancelled_transfers.saturating_add(1); - } - RelayFinishReason::Expired => { - self.expired_transfers = self.expired_transfers.saturating_add(1); - } + .saturating_add(reservation.ingress_bytes) + .saturating_add(reservation.egress_bytes); } } } @@ -653,18 +580,6 @@ impl ArtifactRelayLedger { ingress_bytes: self.ingress_used, egress_bytes: self.egress_used, abandoned_or_failed_bytes: self.abandoned_or_failed_used, - lifetime_ingress_bytes: self.lifetime_ingress_bytes, - lifetime_egress_bytes: self.lifetime_egress_bytes, - lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes, - completed_transfers: self.completed_transfers, - failed_transfers: self.failed_transfers, - cancelled_transfers: self.cancelled_transfers, - expired_transfers: self.expired_transfers, - tracked_scopes: self.project_used.len() - + self.tenant_used.len() - + self.account_used.len(), - max_active_global: self.configuration.max_active_global, - max_tracked_scopes: self.configuration.max_tracked_scopes, reserved_bytes: self .reservations .values() @@ -673,68 +588,3 @@ impl ArtifactRelayLedger { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() { - let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited()); - ledger - .reserve( - "transfer".to_owned(), - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - 16, - 16, - 100, - 1, - ) - .unwrap(); - ledger.charge_ingress("transfer", 24, 1).unwrap(); - ledger.charge_egress("transfer", 24, 1).unwrap(); - ledger.finish("transfer", RelayFinishReason::Completed); - - let usage = ledger.usage(); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - assert_eq!(usage.completed_transfers, 1); - - ledger.prepare_period(u64::MAX); - let usage = ledger.usage(); - assert_eq!(usage.ingress_bytes, 0); - assert_eq!(usage.egress_bytes, 0); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - - let restored = ArtifactRelayLedger::from_durable( - ArtifactRelayConfiguration::unlimited(), - ledger.durable_state(), - ); - let usage = restored.usage(); - assert_eq!(usage.lifetime_ingress_bytes, 24); - assert_eq!(usage.lifetime_egress_bytes, 24); - assert_eq!(usage.completed_transfers, 1); - } - - #[test] - fn older_durable_relay_state_defaults_new_metrics() { - let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({ - "reservations": {}, - "period": 1, - "project_used": [], - "tenant_used": [], - "account_used": [], - "ingress_used": 3, - "egress_used": 4, - "abandoned_or_failed_used": 2 - })) - .unwrap(); - - assert_eq!(state.lifetime_ingress_bytes, 0); - assert_eq!(state.lifetime_egress_bytes, 0); - assert_eq!(state.completed_transfers, 0); - } -} diff --git a/crates/clusterflux-coordinator/src/service/routing.rs b/crates/clusterflux-coordinator/src/service/routing.rs index e05a670..00bef6a 100644 --- a/crates/clusterflux-coordinator/src/service/routing.rs +++ b/crates/clusterflux-coordinator/src/service/routing.rs @@ -5,9 +5,6 @@ impl CoordinatorService { &mut self, request: CoordinatorRequest, ) -> Result { - request - .validate_external_identifiers() - .map_err(CoordinatorServiceError::Protocol)?; self.pump_main_runtime_commands(); let request_payload = serde_json::to_value(&request).map_err(|error| { CoordinatorServiceError::Protocol(format!( @@ -45,7 +42,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - sensitive_moderation_details_exposed: false, + private_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -274,17 +271,9 @@ impl CoordinatorService { enrollment_grant, ), CoordinatorRequest::NodeHeartbeat { - tenant, - project, node, node_signature, - } => self.handle_node_heartbeat( - tenant, - project, - node, - node_signature, - &request_payload_digest, - ), + } => self.handle_node_heartbeat(node, node_signature, &request_payload_digest), CoordinatorRequest::SignedNode { node, node_signature, @@ -298,13 +287,6 @@ impl CoordinatorService { project, actor_user, } => self.handle_list_node_descriptors(tenant, project, actor_user), - CoordinatorRequest::ListNodeSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::RevokeNodeCredential { tenant, project, @@ -388,7 +370,6 @@ impl CoordinatorService { agent_public_key_fingerprint, agent_signature, process, - launch_attempt, restart, } => self.handle_start_process( tenant, @@ -399,7 +380,6 @@ impl CoordinatorService { agent_signature, Some(&request_payload_digest), process, - launch_attempt, restart, ), CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(), @@ -421,20 +401,12 @@ impl CoordinatorService { project, actor_user, process, - launch_attempt, - } => self.handle_abort_process(tenant, project, actor_user, process, launch_attempt), + } => self.handle_abort_process(tenant, project, actor_user, process), CoordinatorRequest::ListProcesses { tenant, project, actor_user, } => self.handle_list_processes(tenant, project, actor_user), - CoordinatorRequest::ListProcessSummaries { - tenant, - project, - actor_user, - cursor, - limit, - } => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit), CoordinatorRequest::QuotaStatus { tenant, project, @@ -480,8 +452,7 @@ impl CoordinatorService { CoordinatorRequest::PollDebugCommand { .. } | CoordinatorRequest::ReportDebugState { .. } | CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(), - CoordinatorRequest::ReportTaskLog { .. } - | CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(), + CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(), CoordinatorRequest::ListTaskEvents { @@ -496,23 +467,6 @@ impl CoordinatorService { actor_user, process, } => self.handle_list_task_snapshots(tenant, project, actor_user, process), - CoordinatorRequest::ListRecentLogs { - tenant, - project, - actor_user, - process, - task, - after_sequence, - limit, - } => self.handle_list_recent_logs( - tenant, - project, - actor_user, - process, - task, - after_sequence, - limit, - ), CoordinatorRequest::JoinTask { tenant, project, @@ -559,20 +513,6 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), - CoordinatorRequest::ListArtifacts { - tenant, - project, - actor_user, - process, - cursor, - limit, - } => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit), - CoordinatorRequest::GetArtifact { - tenant, - project, - actor_user, - artifact, - } => self.handle_get_artifact(tenant, project, actor_user, artifact), CoordinatorRequest::OpenArtifactDownloadStream { tenant, project, @@ -655,7 +595,7 @@ impl CoordinatorService { manual_review: account_state.manual_review, sanitized_reason: account_state.sanitized_reason, next_actions: account_state.next_actions, - sensitive_moderation_details_exposed: false, + private_moderation_details_exposed: false, signup_failure_details_exposed: false, }) } @@ -742,14 +682,6 @@ impl CoordinatorService { context.project.as_str().to_owned(), actor.as_str().to_owned(), ), - AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self - .handle_list_node_summaries( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - cursor, - limit, - ), AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self .handle_revoke_node_credential( context.tenant.as_str().to_owned(), @@ -757,22 +689,18 @@ impl CoordinatorService { actor.as_str().to_owned(), node, ), - AuthenticatedCoordinatorRequest::StartProcess { - process, - launch_attempt, - restart, - } => self.handle_start_process( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - Some(actor.as_str().to_owned()), - None, - None, - None, - None, - process, - launch_attempt, - restart, - ), + AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self + .handle_start_process( + context.tenant.as_str().to_owned(), + context.project.as_str().to_owned(), + Some(actor.as_str().to_owned()), + None, + None, + None, + None, + process, + restart, + ), request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => { self.handle_authenticated_schedule_task(&context, request) } @@ -786,29 +714,17 @@ impl CoordinatorService { actor.as_str().to_owned(), process, ), - AuthenticatedCoordinatorRequest::AbortProcess { - process, - launch_attempt, - } => self.handle_abort_process( + AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), actor.as_str().to_owned(), process, - launch_attempt, ), AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), actor.as_str().to_owned(), ), - AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self - .handle_list_process_summaries( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - cursor, - limit, - ), AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -864,20 +780,6 @@ impl CoordinatorService { actor.as_str().to_owned(), process, ), - AuthenticatedCoordinatorRequest::ListRecentLogs { - process, - task, - after_sequence, - limit, - } => self.handle_list_recent_logs( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - process, - task, - after_sequence, - limit, - ), AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task( context.tenant.as_str().to_owned(), context.project.as_str().to_owned(), @@ -897,24 +799,6 @@ impl CoordinatorService { max_bytes, ttl_seconds, ), - AuthenticatedCoordinatorRequest::ListArtifacts { - process, - cursor, - limit, - } => self.handle_list_artifacts( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - process, - cursor, - limit, - ), - AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact( - context.tenant.as_str().to_owned(), - context.project.as_str().to_owned(), - actor.as_str().to_owned(), - artifact, - ), AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, max_bytes, diff --git a/crates/clusterflux-coordinator/src/service/signed_nodes.rs b/crates/clusterflux-coordinator/src/service/signed_nodes.rs index 40f633a..e6e3878 100644 --- a/crates/clusterflux-coordinator/src/service/signed_nodes.rs +++ b/crates/clusterflux-coordinator/src/service/signed_nodes.rs @@ -1,6 +1,6 @@ -use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId}; +use clusterflux_core::{NodeId, NodeSignedRequest}; -use crate::{CoordinatorError, NodeScopeKey}; +use crate::CoordinatorError; use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError}; @@ -12,24 +12,22 @@ impl CoordinatorService { request: CoordinatorRequest, ) -> Result { let request_kind = signed_node_request_kind(&request)?; - let request_scope = signed_node_request_scope(&request)?; + let request_node = signed_node_request_node(&request)?; let request_payload = serde_json::to_value(&request).map_err(|error| { CoordinatorServiceError::Protocol(format!( "failed to canonicalize signed node request: {error}" )) })?; let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload); - let signed_node = NodeId::try_new(signed_node).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}")) - })?; - if request_scope.node != signed_node { + let signed_node = NodeId::new(signed_node); + if request_node != signed_node { return Err(CoordinatorError::Unauthorized( "signed node request node does not match the wrapped request node".to_owned(), ) .into()); } self.authenticate_node_request( - &request_scope, + &signed_node, Some(node_signature), request_kind, &payload_digest, @@ -146,26 +144,11 @@ impl CoordinatorService { provider, source_snapshot, ), - CoordinatorRequest::RequestRendezvous { - scope, - source, - destination, - direct_connectivity, - failure_reason, - } => self.handle_request_rendezvous( - scope, - source, - destination, - direct_connectivity, - failure_reason, - ), CoordinatorRequest::ReconnectNode { - tenant, - project, node, process, epoch, - } => self.handle_reconnect_node(tenant, project, node, process, epoch), + } => self.handle_reconnect_node(node, process, epoch), CoordinatorRequest::PollTaskControl { tenant, project, @@ -253,29 +236,6 @@ impl CoordinatorService { stderr_truncated, backpressured, ), - CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - process, - node, - task, - stream, - offset, - source_bytes, - text, - truncated, - } => self.handle_report_task_log_chunk( - tenant, - project, - process, - node, - task, - stream, - offset, - source_bytes, - text, - truncated, - ), CoordinatorRequest::ReportVfsMetadata { tenant, project, @@ -362,14 +322,12 @@ fn signed_node_request_kind( CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"), CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"), CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"), - CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"), CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"), CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"), CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"), CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"), CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"), CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"), - CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"), CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"), CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"), _ => Err(CoordinatorError::Unauthorized( @@ -379,135 +337,29 @@ fn signed_node_request_kind( } } -fn signed_node_request_scope( +fn signed_node_request_node( request: &CoordinatorRequest, -) -> Result { +) -> Result { match request { - CoordinatorRequest::ReportNodeCapabilities { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollTaskAssignment { - tenant, - project, - node, - } - | CoordinatorRequest::PollArtifactTransfer { - tenant, - project, - node, - } - | CoordinatorRequest::UploadArtifactTransferChunk { - tenant, - project, - node, - .. - } - | CoordinatorRequest::FailArtifactTransfer { - tenant, - project, - node, - .. - } - | CoordinatorRequest::LaunchChildTask { - tenant, - project, - node, - .. - } - | CoordinatorRequest::JoinChildTask { - tenant, - project, - node, - .. - } - | CoordinatorRequest::CompleteSourcePreparation { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReconnectNode { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollTaskControl { - tenant, - project, - node, - .. - } - | CoordinatorRequest::PollDebugCommand { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportDebugState { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportDebugProbeHit { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportTaskLog { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportTaskLogChunk { - tenant, - project, - node, - .. - } - | CoordinatorRequest::ReportVfsMetadata { - tenant, - project, - node, - .. - } - | CoordinatorRequest::TaskCompleted { - tenant, - project, - node, - .. - } => node_scope_from_strings(tenant, project, node), - CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new( - scope.tenant.clone(), - scope.project.clone(), - source.node.clone(), - )), + CoordinatorRequest::ReportNodeCapabilities { node, .. } + | CoordinatorRequest::PollTaskAssignment { node, .. } + | CoordinatorRequest::PollArtifactTransfer { node, .. } + | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } + | CoordinatorRequest::FailArtifactTransfer { node, .. } + | CoordinatorRequest::LaunchChildTask { node, .. } + | CoordinatorRequest::JoinChildTask { node, .. } + | CoordinatorRequest::CompleteSourcePreparation { node, .. } + | CoordinatorRequest::ReconnectNode { node, .. } + | CoordinatorRequest::PollTaskControl { node, .. } + | CoordinatorRequest::PollDebugCommand { node, .. } + | CoordinatorRequest::ReportDebugState { node, .. } + | CoordinatorRequest::ReportDebugProbeHit { node, .. } + | CoordinatorRequest::ReportTaskLog { node, .. } + | CoordinatorRequest::ReportVfsMetadata { node, .. } + | CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())), _ => Err(CoordinatorError::Unauthorized( "signed_node envelope only accepts node-originated coordinator requests".to_owned(), ) .into()), } } - -fn node_scope_from_strings( - tenant: &str, - project: &str, - node: &str, -) -> Result { - let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}")) - })?; - let project = ProjectId::try_new(project.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}")) - })?; - let node = NodeId::try_new(node.to_owned()).map_err(|error| { - CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}")) - })?; - Ok(NodeScopeKey::new(tenant, project, node)) -} diff --git a/crates/clusterflux-coordinator/src/service/summaries.rs b/crates/clusterflux-coordinator/src/service/summaries.rs deleted file mode 100644 index 1e5ad92..0000000 --- a/crates/clusterflux-coordinator/src/service/summaries.rs +++ /dev/null @@ -1,565 +0,0 @@ -use std::collections::BTreeSet; -use std::time::Instant; - -use clusterflux_core::{ - ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, - TenantId, UserId, -}; - -use super::keys::{process_control_key, ProcessControlKey}; -use super::{ - ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse, - CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary, - NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary, - TaskAttemptState, -}; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(super) struct StoredProcessSummary { - pub(super) started_at_epoch_seconds: u64, - pub(super) ended_at_epoch_seconds: Option, - pub(super) final_result: Option, - pub(super) connected_nodes: Vec, - pub(super) main_task_definition: Option, - pub(super) main_task_instance: Option, - pub(super) main_terminal_state: Option, - pub(super) order: u64, -} - -impl CoordinatorService { - pub(super) fn handle_list_node_summaries( - &mut self, - tenant: String, - project: String, - actor_user: String, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let actor = UserId::new(actor_user); - let cursor = cursor.as_deref(); - let mut nodes = self - .node_descriptors - .iter() - .filter(|(scope, descriptor)| { - scope.tenant == tenant - && scope.project == project - && cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor) - }) - .map(|(scope, descriptor)| { - let online = self.node_is_live(scope); - NodeSummary { - id: descriptor.id.clone(), - display_name: descriptor.id.as_str().to_owned(), - online, - stale: !online, - last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(), - capabilities: descriptor.capabilities.clone(), - direct_connectivity: descriptor.direct_connectivity, - } - }) - .collect::>(); - nodes.sort_by(|left, right| left.id.cmp(&right.id)); - let has_more = nodes.len() > limit as usize; - nodes.truncate(limit as usize); - let next_cursor = has_more - .then(|| nodes.last().map(|node| node.id.as_str().to_owned())) - .flatten(); - Ok(CoordinatorResponse::NodeSummaries { - nodes, - next_cursor, - actor, - }) - } - - pub(super) fn record_process_started( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - now_epoch_seconds: u64, - ) { - let key = process_control_key(tenant, project, process); - if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) { - logs.retain(|entry| &entry.process != process); - if logs.is_empty() { - self.recent_logs.remove(&(tenant.clone(), project.clone())); - } - } - self.recent_log_dropped_through.remove(&key); - self.recent_log_accounted_bytes.retain( - |(entry_tenant, entry_project, entry_process, _, _), _| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.recent_log_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, _, _)| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.recent_log_quota_truncated_streams.retain( - |(entry_tenant, entry_project, entry_process, _, _)| { - entry_tenant != tenant || entry_project != project || entry_process != process - }, - ); - self.process_summary_order - .retain(|retained| retained != &key); - self.evict_process_summaries_for_project(tenant, project); - self.evict_process_summaries_total(); - let order = self.next_process_summary_order; - self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); - self.process_summaries.insert( - key.clone(), - StoredProcessSummary { - started_at_epoch_seconds: now_epoch_seconds, - ended_at_epoch_seconds: None, - final_result: None, - connected_nodes: Vec::new(), - main_task_definition: None, - main_task_instance: None, - main_terminal_state: None, - order, - }, - ); - self.process_summary_order.push_back(key); - } - - pub(super) fn record_process_terminal( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - final_result: ProcessFinalResult, - now_epoch_seconds: u64, - ) { - let key = process_control_key(tenant, project, process); - let connected_nodes = self - .coordinator - .active_process(tenant, project, process) - .map(|active| active.connected_nodes.iter().cloned().collect()) - .unwrap_or_default(); - let order = self.next_process_summary_order; - let entry = self - .process_summaries - .entry(key.clone()) - .or_insert_with(|| { - self.next_process_summary_order = self.next_process_summary_order.saturating_add(1); - self.process_summary_order.push_back(key.clone()); - StoredProcessSummary { - started_at_epoch_seconds: now_epoch_seconds, - ended_at_epoch_seconds: None, - final_result: None, - connected_nodes: Vec::new(), - main_task_definition: None, - main_task_instance: None, - main_terminal_state: None, - order, - } - }); - entry.ended_at_epoch_seconds = Some(now_epoch_seconds); - entry.final_result = Some(final_result); - entry.connected_nodes = connected_nodes; - } - - pub(super) fn record_main_terminal_state( - &mut self, - tenant: &TenantId, - project: &ProjectId, - process: &ProcessId, - task_definition: TaskDefinitionId, - task_instance: TaskInstanceId, - terminal_state: super::TaskTerminalState, - ) { - let key = process_control_key(tenant, project, process); - if !self.process_summaries.contains_key(&key) { - self.record_process_started( - tenant, - project, - process, - self.liveness_now_epoch_seconds(), - ); - } - if let Some(summary) = self.process_summaries.get_mut(&key) { - summary.main_task_definition = Some(task_definition); - summary.main_task_instance = Some(task_instance); - summary.main_terminal_state = Some(terminal_state); - } - } - - pub(super) fn handle_list_process_summaries( - &mut self, - tenant: String, - project: String, - actor_user: String, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let actor = UserId::new(actor_user); - let cursor = parse_order_cursor(cursor.as_deref(), "process")?; - let mut stored = self - .process_summaries - .iter() - .filter(|((entry_tenant, entry_project, _), summary)| { - entry_tenant == &tenant - && entry_project == &project - && cursor.is_none_or(|cursor| summary.order < cursor) - }) - .map(|(key, summary)| (key.clone(), summary.clone())) - .collect::>(); - stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order)); - let has_more = stored.len() > limit as usize; - stored.truncate(limit as usize); - let processes = stored - .into_iter() - .map(|(key, stored)| self.process_summary_from_stored(&key, stored)) - .collect::>(); - let next_cursor = has_more - .then(|| processes.last().map(|process| process.order_cursor.clone())) - .flatten(); - Ok(CoordinatorResponse::ProcessSummaries { - processes, - next_cursor, - actor, - }) - } - - fn process_summary_from_stored( - &self, - key: &ProcessControlKey, - stored: StoredProcessSummary, - ) -> ProcessSummary { - let (tenant, project, process) = key; - let active = self.coordinator.active_process(tenant, project, process); - let process_key = process_control_key(tenant, project, process); - let main_wait_state = active.and_then(|_| { - if self.pending_task_launches.iter().any(|pending| { - &pending.tenant == tenant - && &pending.project == project - && &pending.process == process - }) { - Some("waiting_for_node".to_owned()) - } else if self - .main_runtime - .is_waiting_for_task(tenant, project, process) - { - Some("waiting_for_task".to_owned()) - } else { - None - } - }); - let current_debug_epoch = self.debug_epoch_summary(&process_key); - let awaiting_action = self.task_attempts.iter().any( - |((attempt_tenant, attempt_project, attempt_process, _), attempts)| { - attempt_tenant == tenant - && attempt_project == project - && attempt_process == process - && attempts.iter().any(|attempt| { - attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction - }) - }, - ); - let activity = if let Some(result) = &stored.final_result { - match result { - ProcessFinalResult::Completed => ProcessActivityState::Completed, - ProcessFinalResult::Failed => ProcessActivityState::Failed, - ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled, - } - } else if self.process_cancellations.contains(&process_key) { - ProcessActivityState::Cancelling - } else if current_debug_epoch - .as_ref() - .is_some_and(|epoch| epoch.partially_frozen) - { - ProcessActivityState::DebugEpochPartial - } else if awaiting_action { - ProcessActivityState::AwaitingAction - } else { - match main_wait_state.as_deref() { - Some("waiting_for_node") => ProcessActivityState::WaitingForNode, - Some("waiting_for_task") => ProcessActivityState::WaitingForTask, - _ => ProcessActivityState::Running, - } - }; - let connected_nodes = active - .map(|active| active.connected_nodes.iter().cloned().collect()) - .unwrap_or(stored.connected_nodes); - ProcessSummary { - process: process.clone(), - lifecycle: if active.is_some() { - ProcessLifecycleState::Active - } else { - ProcessLifecycleState::RecentTerminal - }, - activity, - main_wait_state, - started_at_epoch_seconds: stored.started_at_epoch_seconds, - ended_at_epoch_seconds: stored.ended_at_epoch_seconds, - final_result: stored.final_result, - connected_nodes, - current_debug_epoch, - order_cursor: format!("process:{}", stored.order), - } - } - - fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option { - let runtime = self.debug_epoch_runtime.get(key)?; - let acknowledgements = runtime.acknowledgements.values().collect::>(); - let all_acknowledged = !runtime.expected.is_empty() - && runtime - .expected - .iter() - .all(|participant| runtime.acknowledgements.contains_key(participant)); - let fully_frozen = runtime.command == "freeze" - && all_acknowledged - && acknowledgements - .iter() - .all(|ack| ack.state == DebugAcknowledgementState::Frozen); - let freeze_deadline_elapsed = - runtime.command == "freeze" && Instant::now() >= runtime.deadline; - let frozen_count = acknowledgements - .iter() - .filter(|ack| ack.state == DebugAcknowledgementState::Frozen) - .count(); - let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen; - let fully_resumed = runtime.command == "resume" - && all_acknowledged - && acknowledgements - .iter() - .all(|ack| ack.state == DebugAcknowledgementState::Running); - let failed = acknowledgements - .iter() - .any(|ack| ack.state == DebugAcknowledgementState::Failed) - || (freeze_deadline_elapsed - && runtime - .expected - .iter() - .any(|participant| !runtime.acknowledgements.contains_key(participant))); - Some(DebugEpochSummary { - epoch: runtime.epoch, - command: runtime.command.clone(), - fully_frozen, - partially_frozen, - fully_resumed, - failed, - }) - } - - pub(super) fn handle_list_artifacts( - &mut self, - tenant: String, - project: String, - actor_user: String, - process: Option, - cursor: Option, - limit: u32, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let process = process.map(ProcessId::new); - if let Some(process) = &process { - self.authorize_task_event_process_scope(&tenant, &project, process)?; - } - let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?; - let mut metadata = self - .artifact_registry - .metadata_for_project(&tenant, &project) - .filter(|metadata| { - process - .as_ref() - .is_none_or(|process| &metadata.process == process) - && cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor) - }) - .cloned() - .collect::>(); - metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch)); - let has_more = metadata.len() > limit as usize; - metadata.truncate(limit as usize); - let artifacts = metadata - .into_iter() - .map(|metadata| self.artifact_summary(metadata)) - .collect::>(); - let next_cursor = has_more - .then(|| { - artifacts - .last() - .map(|artifact| artifact.order_cursor.clone()) - }) - .flatten(); - Ok(CoordinatorResponse::Artifacts { - artifacts, - next_cursor, - }) - } - - pub(super) fn handle_get_artifact( - &mut self, - tenant: String, - project: String, - actor_user: String, - artifact: String, - ) -> Result { - let tenant = TenantId::new(tenant); - let project = ProjectId::new(project); - let _actor = UserId::new(actor_user); - let artifact = ArtifactId::new(artifact); - let metadata = self - .artifact_registry - .metadata(&tenant, &project, &artifact) - .cloned() - .ok_or(clusterflux_core::DownloadError::NotFound)?; - Ok(CoordinatorResponse::Artifact { - artifact: self.artifact_summary(metadata), - }) - } - - fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary { - let live_retaining_nodes = metadata - .retaining_nodes - .iter() - .filter(|node| { - self.node_is_live(&crate::NodeScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - node, - )) - }) - .cloned() - .collect::>(); - let safe_node = live_retaining_nodes - .iter() - .next() - .cloned() - .or_else(|| metadata.retaining_nodes.iter().next().cloned()); - let explicit_storage = !metadata.explicit_locations.is_empty(); - let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage; - let availability = if downloadable_now { - ArtifactAvailability::Available - } else if !metadata.retaining_nodes.is_empty() { - ArtifactAvailability::NodeOffline - } else { - ArtifactAvailability::Unavailable - }; - let retention_state = if explicit_storage { - ArtifactRetentionState::ExplicitStorage - } else if !metadata.retaining_nodes.is_empty() { - ArtifactRetentionState::NodeRetained - } else { - ArtifactRetentionState::Lost - }; - let display_suffix = metadata.id.as_str().replace(':', "/"); - let display_path = format!("/vfs/artifacts/{display_suffix}"); - let display_name = display_suffix - .rsplit('/') - .next() - .unwrap_or(metadata.id.as_str()) - .to_owned(); - ArtifactSummary { - id: metadata.id, - display_path, - display_name, - process: metadata.process, - producer_task: metadata.producer_task, - safe_node, - digest: metadata.digest, - size_bytes: metadata.size, - availability, - downloadable_now, - retention_state, - explicit_storage, - order_cursor: format!("artifact:{}", metadata.flushed_epoch), - } - } - - fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) { - while self - .process_summaries - .keys() - .filter(|(entry_tenant, entry_project, _)| { - entry_tenant == tenant && entry_project == project - }) - .count() - >= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT - { - let candidate = self.process_summary_order.iter().find(|key| { - &key.0 == tenant - && &key.1 == project - && self - .process_summaries - .get(*key) - .is_some_and(|summary| summary.final_result.is_some()) - }); - let Some(candidate) = candidate.cloned() else { - break; - }; - self.remove_process_summary_state(&candidate); - } - } - - fn evict_process_summaries_total(&mut self) { - while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL { - let candidate = self.process_summary_order.iter().find(|key| { - self.process_summaries - .get(*key) - .is_some_and(|summary| summary.final_result.is_some()) - }); - let Some(candidate) = candidate.cloned() else { - break; - }; - self.remove_process_summary_state(&candidate); - } - } - - fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) { - self.process_summaries.remove(key); - self.recent_log_dropped_through.remove(key); - if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) { - logs.retain(|entry| entry.process != key.2); - if logs.is_empty() { - self.recent_logs.remove(&(key.0.clone(), key.1.clone())); - } - } - self.recent_log_accounted_bytes - .retain(|(tenant, project, process, _, _), _| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.recent_log_truncated_streams - .retain(|(tenant, project, process, _, _)| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.recent_log_quota_truncated_streams - .retain(|(tenant, project, process, _, _)| { - tenant != &key.0 || project != &key.1 || process != &key.2 - }); - self.process_summary_order - .retain(|retained| retained != key); - } -} - -fn parse_order_cursor( - cursor: Option<&str>, - expected_kind: &str, -) -> Result, CoordinatorServiceError> { - cursor - .map(|cursor| { - let (kind, order) = cursor.split_once(':').ok_or_else(|| { - CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - )) - })?; - if kind != expected_kind { - return Err(CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - ))); - } - order.parse::().map_err(|_| { - CoordinatorServiceError::Protocol(format!( - "invalid {expected_kind} pagination cursor" - )) - }) - }) - .transpose() -} diff --git a/crates/clusterflux-coordinator/src/service/tcp.rs b/crates/clusterflux-coordinator/src/service/tcp.rs index 2b6e9c9..d519f34 100644 --- a/crates/clusterflux-coordinator/src/service/tcp.rs +++ b/crates/clusterflux-coordinator/src/service/tcp.rs @@ -79,15 +79,17 @@ impl CoordinatorService { continue; } let response = match decode_wire_request(&line) { - Ok((request_id, request)) => { - match authorize_client_request(&request, authority_mode) - .and_then(|()| self.handle_request(request)) - { - Ok(response) => response, - Err(err) => CoordinatorResponse::service_error(request_id, &err), - } - } - Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), + Ok(request) => match authorize_client_request(&request, authority_mode) + .and_then(|()| self.handle_request(request)) + { + Ok(response) => response, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -112,19 +114,25 @@ fn handle_shared_stream( continue; } let response = match decode_wire_request(&line) { - Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) { + Ok(request) => match authorize_client_request(&request, authority_mode) { Ok(()) => match service.lock() { Ok(mut service) => match service.handle_request(request) { Ok(response) => response, - Err(err) => CoordinatorResponse::service_error(request_id, &err), + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(_) => CoordinatorResponse::Error { + message: "coordinator service lock poisoned".to_owned(), }, - Err(_) => { - CoordinatorResponse::error(request_id, "coordinator service lock poisoned") - } }, - Err(err) => CoordinatorResponse::service_error(request_id, &err), + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), + }, + }, + Err(err) => CoordinatorResponse::Error { + message: err.to_string(), }, - Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err), }; serde_json::to_writer(&mut writer, &response)?; writer.write_all(b"\n")?; @@ -138,27 +146,12 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato Ok((listener, addr)) } -fn decode_wire_request( - line: &str, -) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> { +fn decode_wire_request(line: &str) -> Result { serde_json::from_str::(line)? - .into_parts() + .into_request() .map_err(CoordinatorServiceError::Protocol) } -fn wire_request_id_hint(line: &str) -> String { - serde_json::from_str::(line) - .ok() - .and_then(|value| { - value - .get("request_id") - .and_then(|value| value.as_str()) - .map(str::to_owned) - }) - .filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok()) - .unwrap_or_else(|| "unavailable".to_owned()) -} - fn authorize_client_request( request: &CoordinatorRequest, authority_mode: ClientAuthorityMode, @@ -187,21 +180,15 @@ fn authorize_client_request( } | CoordinatorRequest::AdminStatus { .. } | CoordinatorRequest::SuspendTenant { .. } => Ok(()), - _ => Err(crate::CoordinatorError::Unauthorized( + _ => Err(CoordinatorServiceError::Protocol( "strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority" .to_owned(), - ) - .into()), + )), } } #[cfg(test)] mod transport_boundary_tests { - use std::io::{BufRead as _, BufReader}; - - use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId}; - use serde_json::json; - use super::*; #[test] @@ -210,99 +197,4 @@ mod transport_boundary_tests { let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err(); assert!(error.to_string().contains("restricted to loopback")); } - - #[test] - fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() { - let (listener, addr) = bind_listener("127.0.0.1:0").unwrap(); - let mut coordinator = CoordinatorService::new(11); - coordinator - .issue_cli_session( - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - "healthy-session", - None, - ) - .unwrap(); - let shared = Arc::new(Mutex::new(coordinator)); - let server_shared = Arc::clone(&shared); - let server = std::thread::spawn(move || { - let (stream, _) = listener.accept().unwrap(); - handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap(); - }); - - let mut stream = TcpStream::connect(addr).unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - for (index, malformed_process) in [ - String::new(), - " ".to_owned(), - "bad\0process".to_owned(), - "bad process!".to_owned(), - "x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1), - ] - .into_iter() - .enumerate() - { - let malformed = coordinator_wire_request( - format!("malformed-{index}"), - json!({ - "type": "authenticated", - "session_secret": "healthy-session", - "request": { - "type": "abort_process", - "process": malformed_process, - "launch_attempt": "valid-attempt" - } - }), - ); - serde_json::to_writer(&mut stream, &malformed).unwrap(); - stream.write_all(b"\n").unwrap(); - stream.flush().unwrap(); - - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { error } = - serde_json::from_str::(&line).unwrap() - else { - panic!("malformed identifier request unexpectedly succeeded"); - }; - assert!( - error.message.contains("malformed external identifier") - && error.message.contains("request.request.process"), - "unexpected malformed identifier response: {}", - error.message - ); - assert_eq!(error.request_id, format!("malformed-{index}")); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); - - let valid = coordinator_wire_request( - format!("healthy-{index}"), - json!({ - "type": "authenticated", - "session_secret": "healthy-session", - "request": { "type": "auth_status" } - }), - ); - serde_json::to_writer(&mut stream, &valid).unwrap(); - stream.write_all(b"\n").unwrap(); - stream.flush().unwrap(); - - line.clear(); - reader.read_line(&mut line).unwrap(); - assert!( - matches!( - serde_json::from_str::(&line).unwrap(), - CoordinatorResponse::AuthStatus { - authenticated: true, - .. - } - ), - "valid authenticated traffic failed after malformed request {index}" - ); - } - - stream.shutdown(std::net::Shutdown::Both).unwrap(); - server.join().unwrap(); - assert!(!shared.is_poisoned()); - } } diff --git a/crates/clusterflux-coordinator/src/service/tests.rs b/crates/clusterflux-coordinator/src/service/tests.rs index 339860d..eefbc8b 100644 --- a/crates/clusterflux-coordinator/src/service/tests.rs +++ b/crates/clusterflux-coordinator/src/service/tests.rs @@ -12,8 +12,8 @@ use clusterflux_core::{ AgentSignedRequest, AgentWorkflowScope, ArtifactFlush, ArtifactHandle, ArtifactId, Capability, DataPlaneObject, DataPlaneScope, Digest, EnvironmentBackend, EnvironmentRequirements, LimitKind, NodeCapabilities, NodeEndpoint, NodeSignedRequest, Os, ResourceLimits, - SourceProviderKind, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, - TaskJoinState, TaskSpec, VfsPath, WasmExportAbi, WasmTaskResult, + SourceProviderKind, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec, + VfsPath, WasmExportAbi, }; use serde_json::json; @@ -49,35 +49,6 @@ fn test_admin_request( ) } -fn enroll_test_node( - service: &mut CoordinatorService, - tenant: &str, - project: &str, - node: &str, - public_key: &str, -) { - let response = service - .handle_request(CoordinatorRequest::CreateNodeEnrollmentGrant { - tenant: tenant.to_owned(), - project: project.to_owned(), - actor_user: "test-user".to_owned(), - ttl_seconds: 900, - }) - .unwrap(); - let CoordinatorResponse::NodeEnrollmentGrantCreated { grant, .. } = response else { - panic!("expected node enrollment grant"); - }; - service - .handle_request(CoordinatorRequest::ExchangeNodeEnrollmentGrant { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - public_key: public_key.to_owned(), - enrollment_grant: grant, - }) - .unwrap(); -} - #[test] fn runtime_service_uses_memory_only_when_database_url_is_absent() { let service = CoordinatorService::new_with_database_url(1, None).unwrap(); @@ -92,7 +63,7 @@ fn runtime_service_uses_memory_only_when_database_url_is_absent() { } #[test] -fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() { +fn postgres_backed_runtime_service_survives_restart_without_live_process_state() { let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else { return; }; @@ -113,26 +84,18 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() None, ) .unwrap(); - enroll_test_node( - &mut first, - "tenant-pg", - "project-pg", - "node-pg", - &test_node_public_key("node-pg"), - ); - let duplicate_private_key = test_node_private_key("node-pg-other-scope"); - enroll_test_node( - &mut first, - "tenant-pg-other", - "project-pg-other", - "node-pg", - &node_ed25519_public_key_from_private_key(&duplicate_private_key).unwrap(), - ); + first + .handle_request(CoordinatorRequest::AttachNode { + tenant: "tenant-pg".to_owned(), + project: "project-pg".to_owned(), + node: "node-pg".to_owned(), + public_key: test_node_public_key("node-pg"), + }) + .unwrap(); first .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, process: "process-ephemeral".to_owned(), restart: false, }, @@ -157,12 +120,8 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() )); let heartbeat = restarted .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg".to_owned(), - project: "project-pg".to_owned(), node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope( - "tenant-pg", - "project-pg", + node_signature: Some(signed_node_heartbeat( "node-pg", "postgres-restart-heartbeat", )), @@ -172,24 +131,6 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() heartbeat, CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } )); - let duplicate_heartbeat = restarted - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg-other".to_owned(), - project: "project-pg-other".to_owned(), - node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-pg-other", - "project-pg-other", - "node-pg", - &duplicate_private_key, - "postgres-restart-heartbeat", - )), - }) - .unwrap(); - assert!(matches!( - duplicate_heartbeat, - CoordinatorResponse::NodeHeartbeat { epoch: 42, .. } - )); let CoordinatorResponse::ProcessStatuses { processes, .. } = restarted .handle_request(CoordinatorRequest::Authenticated { session_secret: session_secret.to_owned(), @@ -200,49 +141,6 @@ fn postgres_persists_duplicate_scoped_node_ids_and_credentials_across_restart() panic!("expected process list"); }; assert!(processes.is_empty()); - - restarted - .handle_request(CoordinatorRequest::RevokeNodeCredential { - tenant: "tenant-pg".to_owned(), - project: "project-pg".to_owned(), - actor_user: "user-pg".to_owned(), - node: "node-pg".to_owned(), - }) - .unwrap(); - drop(restarted); - - let mut after_revocation = - CoordinatorService::new_with_database_url(43, Some(&database_url)).unwrap(); - assert!(after_revocation - .coordinator - .node_identity( - &TenantId::from("tenant-pg"), - &ProjectId::from("project-pg"), - &NodeId::from("node-pg"), - ) - .is_none()); - assert!(after_revocation - .coordinator - .node_identity( - &TenantId::from("tenant-pg-other"), - &ProjectId::from("project-pg-other"), - &NodeId::from("node-pg"), - ) - .is_some()); - after_revocation - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-pg-other".to_owned(), - project: "project-pg-other".to_owned(), - node: "node-pg".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-pg-other", - "project-pg-other", - "node-pg", - &duplicate_private_key, - "post-revocation-restart", - )), - }) - .expect("the duplicate node in the other scope must remain authenticated"); } fn linux_capabilities() -> NodeCapabilities { @@ -494,191 +392,6 @@ fn register_test_task_assignment( )); } -fn service_with_completed_main_and_final_child( - failure_policy: clusterflux_core::TaskFailurePolicy, -) -> CoordinatorService { - let mut service = CoordinatorService::new(83); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let node = NodeId::from("worker"); - let task = TaskInstanceId::from("final-child"); - - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - public_key: test_node_public_key(node.as_str()), - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: true, - online: true, - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: process.to_string(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: node.to_string(), - process: process.to_string(), - epoch: 83, - }) - .unwrap(); - - let mut task_spec = test_task_spec_instance( - tenant.as_str(), - project.as_str(), - process.as_str(), - "child-definition", - task.as_str(), - 83, - [], - ); - task_spec.failure_policy = failure_policy; - let assignment = TaskAssignment { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task: task.clone(), - node: node.clone(), - epoch: 83, - artifact_path: "/vfs/artifacts/final-child.bin".to_owned(), - task_spec: task_spec.clone(), - wasm_module_base64: test_wasm_module_base64(), - }; - service - .capture_task_restart_checkpoint(&assignment) - .unwrap(); - service - .begin_task_attempt( - &task_spec, - Some(node.clone()), - Some(&assignment.artifact_path), - false, - ) - .unwrap(); - service - .active_tasks - .insert(task_control_key(&tenant, &project, &process, &node, &task)); - service - .task_assignments - .entry((tenant.clone(), project.clone(), node.clone())) - .or_default() - .push_back(assignment); - service - .debug_epochs - .insert(process_control_key(&tenant, &project, &process), 11); - service.debug_breakpoints.insert( - process_control_key(&tenant, &project, &process), - super::debug::DebugBreakpointPlan { - actor: UserId::from("user"), - revision: 1, - probe_symbols: BTreeSet::from(["child-probe".to_owned()]), - hit_epoch: None, - hit_task: None, - hit_probe_symbol: None, - }, - ); - service.debug_commands.insert( - task_control_key(&tenant, &project, &process, &node, &task), - super::debug::DebugPendingCommand { - epoch: 11, - command: "continue".to_owned(), - }, - ); - let panel_key = super::keys::panel_stop_key(&tenant, &project, &process); - service.panel_snapshots.insert( - panel_key.clone(), - PanelState { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - widgets: BTreeMap::new(), - program_ui_events_enabled: false, - control_plane_actions: Vec::new(), - }, - ); - service.stopped_panels.insert(panel_key); - service.record_task_completion_event(TaskCompletionEvent { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - node: NodeId::from("coordinator-main"), - executor: TaskExecutor::CoordinatorMain, - task_definition: TaskDefinitionId::from("build"), - task: TaskInstanceId::from("main"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - service - .coordinator - .grant_project_debug(tenant, project, UserId::from("user")); - service -} - -fn complete_terminal_matrix_child( - service: &mut CoordinatorService, - terminal_state: TaskTerminalState, -) { - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "terminal-matrix".to_owned(), - node: "worker".to_owned(), - task: "final-child".to_owned(), - terminal_state: Some(terminal_state), - status_code: Some(1), - stdout_bytes: 0, - stderr_bytes: 4, - stdout_tail: String::new(), - stderr_tail: "boom".to_owned(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }) - .unwrap(); -} - fn test_agent_public_key() -> String { agent_ed25519_public_key_from_private_key(TEST_AGENT_PRIVATE_KEY).unwrap() } @@ -720,9 +433,7 @@ fn with_signed_agent_workflow( let signature = signed_agent_workflow_request(&request, request_kind, process, task, nonce); match &mut request { CoordinatorRequest::StartProcess { - launch_attempt: None, - agent_signature, - .. + agent_signature, .. } | CoordinatorRequest::LaunchTask { agent_signature, .. @@ -741,21 +452,7 @@ fn test_node_public_key(node: &str) -> String { } fn signed_node_heartbeat(node: &str, nonce: &str) -> NodeSignedRequest { - signed_node_heartbeat_in_scope("tenant", "project", node, nonce) -} - -fn signed_node_heartbeat_in_scope( - tenant: &str, - project: &str, - node: &str, - nonce: &str, -) -> NodeSignedRequest { - let payload = json!({ - "type": "node_heartbeat", - "tenant": tenant, - "project": project, - "node": node - }); + let payload = json!({"type": "node_heartbeat", "node": node}); signed_node_request_with_private_key( node, &test_node_private_key(node), @@ -770,22 +467,7 @@ fn signed_node_heartbeat_with_private_key( private_key: &str, nonce: &str, ) -> NodeSignedRequest { - signed_node_heartbeat_in_scope_with_private_key("tenant", "project", node, private_key, nonce) -} - -fn signed_node_heartbeat_in_scope_with_private_key( - tenant: &str, - project: &str, - node: &str, - private_key: &str, - nonce: &str, -) -> NodeSignedRequest { - let payload = json!({ - "type": "node_heartbeat", - "tenant": tenant, - "project": project, - "node": node - }); + let payload = json!({"type": "node_heartbeat", "node": node}); signed_node_request_with_private_key( node, private_key, @@ -814,34 +496,6 @@ fn signed_node_request_with_private_key( } fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest { - let node = match &request { - CoordinatorRequest::ReportNodeCapabilities { node, .. } - | CoordinatorRequest::PollTaskAssignment { node, .. } - | CoordinatorRequest::PollArtifactTransfer { node, .. } - | CoordinatorRequest::UploadArtifactTransferChunk { node, .. } - | CoordinatorRequest::FailArtifactTransfer { node, .. } - | CoordinatorRequest::LaunchChildTask { node, .. } - | CoordinatorRequest::JoinChildTask { node, .. } - | CoordinatorRequest::CompleteSourcePreparation { node, .. } - | CoordinatorRequest::ReconnectNode { node, .. } - | CoordinatorRequest::PollTaskControl { node, .. } - | CoordinatorRequest::PollDebugCommand { node, .. } - | CoordinatorRequest::ReportDebugState { node, .. } - | CoordinatorRequest::ReportDebugProbeHit { node, .. } - | CoordinatorRequest::ReportTaskLog { node, .. } - | CoordinatorRequest::ReportTaskLogChunk { node, .. } - | CoordinatorRequest::ReportVfsMetadata { node, .. } - | CoordinatorRequest::TaskCompleted { node, .. } => node.clone(), - CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(), - _ => panic!("test helper only signs node-originated requests"), - }; - signed_node_request_auto_with_private_key(request, &test_node_private_key(&node)) -} - -fn signed_node_request_auto_with_private_key( - request: CoordinatorRequest, - private_key: &str, -) -> CoordinatorRequest { let payload_digest = signed_request_payload_digest(&serde_json::to_value(&request).unwrap()); let (node, request_kind) = match &request { CoordinatorRequest::ReportNodeCapabilities { node, .. } => { @@ -864,9 +518,6 @@ fn signed_node_request_auto_with_private_key( CoordinatorRequest::CompleteSourcePreparation { node, .. } => { (node.clone(), "complete_source_preparation") } - CoordinatorRequest::RequestRendezvous { source, .. } => { - (source.node.to_string(), "request_rendezvous") - } CoordinatorRequest::ReconnectNode { node, .. } => (node.clone(), "reconnect_node"), CoordinatorRequest::PollTaskControl { node, .. } => (node.clone(), "poll_task_control"), CoordinatorRequest::PollDebugCommand { node, .. } => (node.clone(), "poll_debug_command"), @@ -875,9 +526,6 @@ fn signed_node_request_auto_with_private_key( (node.clone(), "report_debug_probe_hit") } CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"), - CoordinatorRequest::ReportTaskLogChunk { node, .. } => { - (node.clone(), "report_task_log_chunk") - } CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"), CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"), _ => panic!("test helper only signs node-originated requests"), @@ -889,7 +537,7 @@ fn signed_node_request_auto_with_private_key( node: node.clone(), node_signature: signed_node_request_with_private_key( &node, - private_key, + &test_node_private_key(&node), request_kind, &payload_digest, &format!("node-request-{nonce}"), @@ -1214,16 +862,10 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() { }; assert_eq!(placement.node, NodeId::from("session-node")); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - process, - actor, - .. - } = service + let CoordinatorResponse::ProcessStarted { process, actor, .. } = service .handle_request(CoordinatorRequest::Authenticated { session_secret: "cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, process: "vp-session".to_owned(), restart: false, }, @@ -1295,8 +937,6 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), node: "session-node".to_owned(), process: "vp-session".to_owned(), epoch: 7, @@ -1353,7 +993,6 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() { .handle_request(CoordinatorRequest::Authenticated { session_secret: "victim-cli-session-secret".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, process: "vp-victim".to_owned(), restart: false, }, @@ -1503,7 +1142,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { suspended, disabled, sanitized_reason, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1524,7 +1163,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(!suspended); assert!(!disabled); assert!(sanitized_reason.is_none()); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); for (tenant, policy_name, expected_status, expected_reason) in [ @@ -1560,7 +1199,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { manual_review, sanitized_reason, next_actions, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1582,7 +1221,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); } @@ -1721,7 +1360,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { disabled, sanitized_reason, next_actions, - sensitive_moderation_details_exposed, + private_moderation_details_exposed, signup_failure_details_exposed, .. } = service @@ -1744,7 +1383,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { assert!(next_actions .iter() .any(|action| action.contains("hosted operator"))); - assert!(!sensitive_moderation_details_exposed); + assert!(!private_moderation_details_exposed); assert!(!signup_failure_details_exposed); let create = service @@ -1759,7 +1398,6 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() { let start = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: Some("attempt-a".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -1898,7 +1536,6 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() { let fingerprint_only = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -1914,7 +1551,6 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() { let wrong_fingerprint = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -1956,13 +1592,10 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() { .unwrap(); let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - actor: start_actor, - .. + actor: start_actor, .. } = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -1986,7 +1619,6 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() { let replay = service .handle_request(with_signed_agent_workflow( CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2096,7 +1728,6 @@ fn signed_node_and_agent_requests_reject_body_modification() { }) .unwrap(); let original_agent_request = CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2115,7 +1746,6 @@ fn signed_node_and_agent_requests_reject_body_modification() { ); let mut modified_agent_request = original_agent_request; let CoordinatorRequest::StartProcess { - launch_attempt: None, restart, agent_signature: request_signature, .. @@ -2159,13 +1789,8 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() { }) .unwrap(); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - charged_spawns, - .. - } = service + let CoordinatorResponse::ProcessStarted { charged_spawns, .. } = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2262,7 +1887,6 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() { let other_project_process = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "other-project".to_owned(), actor_user: None, @@ -2275,10 +1899,7 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() { .unwrap(); assert!(matches!( other_project_process, - CoordinatorResponse::ProcessStarted { - launch_attempt: None, - .. - } + CoordinatorResponse::ProcessStarted { .. } )); assert_eq!( service.quota.used_workflow_spawns( @@ -2314,7 +1935,6 @@ fn project_quota_resets_at_the_configured_window_boundary() { service.set_server_time(59); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: Some("attempt-b".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2327,7 +1947,6 @@ fn project_quota_resets_at_the_configured_window_boundary() { .unwrap(); service .handle_request(CoordinatorRequest::AbortProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), @@ -2337,7 +1956,6 @@ fn project_quota_resets_at_the_configured_window_boundary() { let exhausted = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: Some("attempt-b".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2353,7 +1971,6 @@ fn project_quota_resets_at_the_configured_window_boundary() { service.set_server_time(60); let started = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2367,7 +1984,6 @@ fn project_quota_resets_at_the_configured_window_boundary() { assert!(matches!( started, CoordinatorResponse::ProcessStarted { - launch_attempt: None, charged_spawns: 1, .. } @@ -2435,7 +2051,7 @@ fn authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch() } #[test] -fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() { +fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() { let mut limits = ResourceLimits::unlimited(); limits.limits.insert(LimitKind::LogBytes, 4); let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); @@ -2457,7 +2073,6 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2470,8 +2085,6 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2494,37 +2107,23 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() backpressured: false, }) .unwrap(); - let truncated = service + let denied = service .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLog { tenant: "tenant".to_owned(), project: "project".to_owned(), process: "process".to_owned(), node: "node".to_owned(), task: "task".to_owned(), - stdout_bytes: 4, - stderr_bytes: 1, - stdout_tail: "outx".to_owned(), - stderr_tail: "e".to_owned(), + stdout_bytes: 1, + stderr_bytes: 0, + stdout_tail: "x".to_owned(), + stderr_tail: String::new(), stdout_truncated: false, stderr_truncated: false, backpressured: true, }) - .unwrap(); - let CoordinatorResponse::TaskLogRecorded { - stdout_tail, - stdout_bytes: 4, - .. - } = truncated - else { - panic!("expected a successful truncated task-log report"); - }; - assert_eq!(stdout_tail, "[log output truncated at project log quota]"); - assert!(service - .recent_logs - .get(&(TenantId::from("tenant"), ProjectId::from("project"))) - .unwrap() - .iter() - .any(|entry| entry.text.contains("project log quota") && entry.truncated)); + .unwrap_err(); + assert!(denied.to_string().contains("LogBytes")); assert_eq!( service .quota @@ -2533,140 +2132,6 @@ fn signed_node_log_ingestion_truncates_at_scoped_quota_without_failing_reports() ); } -#[test] -fn log_quota_exhaustion_cannot_strand_task_completion_or_artifact_publication() { - let mut limits = ResourceLimits::unlimited(); - limits.limits.insert(LimitKind::LogBytes, 4); - let quota = CoordinatorQuotaConfiguration::new(limits, [(LimitKind::LogBytes, 60)]).unwrap(); - let mut service = CoordinatorService::new_with_admin_token_database_url_and_quota( - 7, - "test-admin-token", - None, - quota, - ) - .unwrap(); - service.set_server_time(30); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - public_key: test_node_public_key("node"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process".to_owned(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - process: "process".to_owned(), - epoch: 7, - }) - .unwrap(); - register_test_task_assignment( - &mut service, - "tenant", - "project", - "process", - "node", - "compile", - "compile-one", - 7, - ); - service.record_task_completion_event(TaskCompletionEvent { - tenant: TenantId::from("tenant"), - project: ProjectId::from("project"), - process: ProcessId::from("process"), - node: NodeId::from("coordinator-main"), - executor: TaskExecutor::CoordinatorMain, - task_definition: TaskDefinitionId::from("build"), - task: TaskInstanceId::from("main"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "process".to_owned(), - node: "node".to_owned(), - task: "compile-one".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 64, - stderr_bytes: 0, - stdout_tail: "the-real-final-tail".to_owned(), - stderr_tail: String::new(), - stdout_truncated: true, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/result.bin".to_owned()), - artifact_digest: Some(Digest::sha256("artifact bytes")), - artifact_size_bytes: Some(14), - result: None, - }) - .unwrap(); - - let event = service - .task_events - .iter() - .find(|event| event.task == TaskInstanceId::from("compile-one")) - .unwrap(); - assert_eq!( - event.stdout_tail, - "[log output truncated at project log quota]" - ); - assert!(event.stdout_truncated); - assert!(!service - .active_tasks - .iter() - .any(|key| key.4 == TaskInstanceId::from("compile-one"))); - assert!(service - .coordinator - .active_process( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ProcessId::from("process"), - ) - .is_none()); - assert!(matches!( - service - .handle_request(CoordinatorRequest::GetArtifact { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - artifact: "result.bin".to_owned(), - }) - .unwrap(), - CoordinatorResponse::Artifact { .. } - )); -} - #[test] fn service_attaches_node_starts_process_and_records_scoped_task_event() { let mut service = CoordinatorService::new(7); @@ -2683,7 +2148,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { let started = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -2697,7 +2161,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { assert_eq!( started, CoordinatorResponse::ProcessStarted { - launch_attempt: None, process: ProcessId::from("process"), epoch: 7, actor: WorkflowActor { @@ -2715,8 +2178,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "task-event-heartbeat")), }) @@ -2731,8 +2192,6 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -2917,8 +2376,6 @@ fn service_revokes_node_credentials_and_live_descriptors() { let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "revoked-heartbeat")), }) @@ -2938,201 +2395,6 @@ fn service_revokes_node_credentials_and_live_descriptors() { assert!(descriptors.is_empty()); } -#[test] -fn duplicate_node_ids_are_isolated_across_identity_replay_liveness_and_revocation() { - let mut service = CoordinatorService::new(19); - let node = "shared-node"; - let private_a = test_node_private_key("tenant-a-shared-node"); - let private_b = test_node_private_key("tenant-b-shared-node"); - let private_c = test_node_private_key("tenant-a-other-project-shared-node"); - let public_a = node_ed25519_public_key_from_private_key(&private_a).unwrap(); - let public_b = node_ed25519_public_key_from_private_key(&private_b).unwrap(); - let public_c = node_ed25519_public_key_from_private_key(&private_c).unwrap(); - - for (tenant, project, public_key) in [ - ("tenant-a", "project-a", public_a), - ("tenant-b", "project-b", public_b), - ("tenant-a", "project-c", public_c), - ] { - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - public_key, - }) - .unwrap(); - } - assert_eq!( - service - .coordinator - .node_identity_count_for_tenant(&TenantId::from("tenant-a")), - 2 - ); - assert_eq!( - service - .coordinator - .node_identity_count_for_tenant(&TenantId::from("tenant-b")), - 1 - ); - - for (index, (tenant, project, private_key)) in [ - ("tenant-a", "project-a", private_a.as_str()), - ("tenant-b", "project-b", private_b.as_str()), - ("tenant-a", "project-c", private_c.as_str()), - ] - .into_iter() - .enumerate() - { - service.set_server_time(100 + index as u64); - service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - tenant, - project, - node, - private_key, - "same-nonce", - )), - }) - .unwrap(); - service - .handle_request(signed_node_request_auto_with_private_key( - CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: vec![Digest::sha256(format!( - "cache-{tenant}-{project}" - ))], - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: index != 1, - online: true, - }, - private_key, - )) - .unwrap(); - } - - let replay = service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-a", - "project-a", - node, - &private_a, - "same-nonce", - )), - }) - .unwrap_err(); - assert!(replay.to_string().contains("nonce")); - - let forged = service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-b".to_owned(), - project: "project-b".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-b", - "project-b", - node, - &private_a, - "forged-cross-scope", - )), - }) - .unwrap_err(); - assert!(forged.to_string().contains("signature")); - - let scope_a = crate::NodeScopeKey::new( - TenantId::from("tenant-a"), - ProjectId::from("project-a"), - NodeId::from(node), - ); - let scope_b = crate::NodeScopeKey::new( - TenantId::from("tenant-b"), - ProjectId::from("project-b"), - NodeId::from(node), - ); - let scope_c = crate::NodeScopeKey::new( - TenantId::from("tenant-a"), - ProjectId::from("project-c"), - NodeId::from(node), - ); - assert!(service.node_descriptors.contains_key(&scope_a)); - assert!(service.node_descriptors.contains_key(&scope_b)); - assert!(service.node_descriptors.contains_key(&scope_c)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_a)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_b)); - assert!(service.node_last_seen_epoch_seconds.contains_key(&scope_c)); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_a], 100); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_b], 101); - assert_eq!(service.node_last_seen_epoch_seconds[&scope_c], 102); - assert!(service.node_descriptors[&scope_a].direct_connectivity); - assert!(!service.node_descriptors[&scope_b].direct_connectivity); - assert!(service.node_descriptors[&scope_c].direct_connectivity); - assert!(service - .node_replay_nonces - .contains_key(&(scope_a.clone(), "same-nonce".to_owned()))); - assert!(service - .node_replay_nonces - .contains_key(&(scope_b.clone(), "same-nonce".to_owned()))); - assert!(service - .node_replay_nonces - .contains_key(&(scope_c.clone(), "same-nonce".to_owned()))); - - service - .handle_request(CoordinatorRequest::RevokeNodeCredential { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - actor_user: "user-a".to_owned(), - node: node.to_owned(), - }) - .unwrap(); - assert!(service - .coordinator - .node_identity( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - &NodeId::from(node), - ) - .is_none()); - assert!(service - .coordinator - .node_identity( - &TenantId::from("tenant-b"), - &ProjectId::from("project-b"), - &NodeId::from(node), - ) - .is_some()); - assert!(!service.node_descriptors.contains_key(&scope_a)); - assert!(service.node_descriptors.contains_key(&scope_b)); - assert!(service.node_descriptors.contains_key(&scope_c)); - - service - .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-b".to_owned(), - project: "project-b".to_owned(), - node: node.to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope_with_private_key( - "tenant-b", - "project-b", - node, - &private_b, - "post-other-scope-revocation", - )), - }) - .expect("revoking tenant A's duplicate node must not affect tenant B"); -} - #[test] fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() { let mut service = CoordinatorService::new(7); @@ -3147,7 +2409,6 @@ fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -3160,8 +2421,6 @@ fn service_delivers_cancellation_to_connected_node_and_records_terminal_state() .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -3283,7 +2542,6 @@ fn service_authorizes_debug_attach_through_public_api() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -3406,7 +2664,6 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), @@ -3456,7 +2713,6 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { }, ); let CoordinatorResponse::DebugBreakpoints { - revision, probe_symbols, hit_epoch, .. @@ -3466,37 +2722,15 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() { project: "project".to_owned(), actor_user: "user".to_owned(), process: "process".to_owned(), - revision: 1, probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()], }) .unwrap() else { panic!("expected debug breakpoints response"); }; - assert_eq!(revision, 1); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); assert_eq!(hit_epoch, None); - let CoordinatorResponse::DebugBreakpoints { - revision, - probe_symbols, - .. - } = service - .handle_request(CoordinatorRequest::SetDebugBreakpoints { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - process: "process".to_owned(), - revision: 0, - probe_symbols: vec!["clusterflux.probe.stale".to_owned()], - }) - .unwrap() - else { - panic!("expected stale breakpoint response"); - }; - assert_eq!(revision, 1); - assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]); - let CoordinatorResponse::DebugProbeHit { breakpoint_matched, debug_epoch, @@ -3757,7 +2991,6 @@ fn service_reports_task_restart_boundary_through_public_api() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4030,7 +3263,6 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() { } service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4044,8 +3276,6 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() { for node in ["node-a", "node-b"] { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: node.to_owned(), process: "process".to_owned(), epoch: 7, @@ -4152,7 +3382,6 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() { service .handle_request(CoordinatorRequest::AbortProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), @@ -4184,7 +3413,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { let mut service = CoordinatorService::new(7); let started = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4197,15 +3425,11 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { .unwrap(); assert!(matches!( started, - CoordinatorResponse::ProcessStarted { - launch_attempt: None, - .. - } + CoordinatorResponse::ProcessStarted { .. } )); let same_without_restart = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4222,7 +3446,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { let other_process = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4237,27 +3460,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { .to_string() .contains("already has active virtual process")); - let wrong_attempt_abort = service - .handle_request(CoordinatorRequest::AbortProcess { - launch_attempt: Some("attempt-b".to_owned()), - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: "user".to_owned(), - process: "process-a".to_owned(), - }) - .unwrap_err(); - assert!(wrong_attempt_abort - .to_string() - .contains("does not own process process-a")); - assert!(service - .coordinator - .active_process( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ProcessId::from("process-a") - ) - .is_some()); - let retired_main_abort = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let process_key = process_control_key( &TenantId::from("tenant"), @@ -4279,7 +3481,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { ); let restarted = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: Some("attempt-a-restart".to_owned()), tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -4293,7 +3494,6 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { assert_eq!( restarted, CoordinatorResponse::ProcessStarted { - launch_attempt: Some("attempt-a-restart".to_owned()), process: ProcessId::from("process-a"), epoch: 7, actor: WorkflowActor { @@ -4312,628 +3512,11 @@ fn service_rejects_second_active_process_unless_restarting_same_process() { assert!(!service.main_runtime.controls.contains_key(&process_key)); } -#[test] -fn completed_main_waits_for_final_child_then_preserves_history_and_releases_the_slot() { - let mut service = CoordinatorService::new(31); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("process-main-before-child"); - let child = TaskInstanceId::from("child-active"); - let process_key = process_control_key(&tenant, &project, &process); - - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: "worker".to_owned(), - public_key: test_node_public_key("worker"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: process.to_string(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_string(), - project: project.to_string(), - node: "worker".to_owned(), - process: process.to_string(), - epoch: 31, - }) - .unwrap(); - register_test_task_assignment( - &mut service, - tenant.as_str(), - project.as_str(), - process.as_str(), - "worker", - "child-definition", - child.as_str(), - 31, - ); - - let main = TaskInstanceId::from("main-instance"); - service.main_runtime.controls.insert( - process_key.clone(), - super::main_runtime::CoordinatorMainControl { - task_definition: TaskDefinitionId::from("build"), - task_instance: main.clone(), - abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), - debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()), - state: "running".to_owned(), - stopped_probe_symbol: None, - handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), - launch_id: 1, - }, - ); - service.debug_epochs.insert(process_key.clone(), 9); - - service.record_coordinator_main_completion( - super::main_runtime::MainScope { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - task_definition: TaskDefinitionId::from("build"), - task_instance: main, - epoch: 31, - launch_id: 1, - }, - Ok(WasmTaskResult::completed( - TaskInstanceId::from("main-instance"), - TaskBoundaryValue::SmallJson(json!("main completed")), - )), - ); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(service - .active_tasks - .iter() - .any(|(_, _, retained_process, _, task)| { - retained_process == &process && task == &child - })); - let blocked_next = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process-too-early".to_owned(), - restart: false, - }) - .unwrap_err(); - assert!(blocked_next - .to_string() - .contains("already has active virtual process")); - - let artifact_bytes = b"child artifact survives terminal cleanup"; - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: child.to_string(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: artifact_bytes.len() as u64, - stderr_bytes: 0, - stdout_tail: "child completed".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/child-output".to_owned()), - artifact_digest: Some(Digest::sha256(artifact_bytes)), - artifact_size_bytes: Some(artifact_bytes.len() as u64), - result: Some(TaskBoundaryValue::SmallJson(json!("child completed"))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.debug_epochs.contains_key(&process_key)); - let CoordinatorResponse::TaskEvents { events } = service - .handle_request(CoordinatorRequest::ListTaskEvents { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: Some(process.to_string()), - }) - .unwrap() - else { - panic!("expected retained task events"); - }; - assert_eq!(events.len(), 2); - assert!(events.iter().any(|event| { - event.executor == TaskExecutor::CoordinatorMain - && event.terminal_state == TaskTerminalState::Completed - })); - assert!(events.iter().any(|event| { - event.task == child && event.artifact_digest == Some(Digest::sha256(artifact_bytes)) - })); - let metadata = service - .artifact_registry - .metadata(&tenant, &project, &ArtifactId::from("child-output")) - .expect("artifact metadata must survive terminal cleanup"); - assert_eq!(metadata.process, process); - assert_eq!(metadata.digest, Digest::sha256(artifact_bytes)); - - let next = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "process-after-cleanup".to_owned(), - restart: false, - }) - .unwrap(); - assert!(matches!(next, CoordinatorResponse::ProcessStarted { .. })); -} - -#[test] -fn completed_main_terminal_matrix_retires_after_failed_or_cancelled_final_child() { - for terminal_state in [TaskTerminalState::Failed, TaskTerminalState::Cancelled] { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::FailFast, - ); - complete_terminal_matrix_child(&mut service, terminal_state.clone()); - - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - assert!( - service - .coordinator - .active_process(&tenant, &project, &process) - .is_none(), - "{terminal_state:?} final child left the process slot active" - ); - assert!(!service - .debug_epochs - .contains_key(&process_control_key(&tenant, &project, &process))); - assert!(!service - .debug_breakpoints - .contains_key(&process_control_key(&tenant, &project, &process))); - assert!(service.debug_commands.keys().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - assert!(!service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - assert!(service.active_tasks.iter().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - assert!(!service - .main_runtime - .controls - .contains_key(&process_control_key(&tenant, &project, &process))); - let join = service.task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ); - assert_eq!( - join.state, - match terminal_state { - TaskTerminalState::Failed => TaskJoinState::Failed, - TaskTerminalState::Cancelled => TaskJoinState::Cancelled, - TaskTerminalState::Completed => unreachable!(), - } - ); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-terminal".to_owned(), - restart: false, - }) - .expect("the terminal outcome must release the one-process project slot"); - } -} - -#[test] -fn completed_main_unpolled_final_assignment_completion_retires_process() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let node = NodeId::from("worker"); - let assignment_key = (tenant.clone(), project.clone(), node); - assert_eq!( - service - .task_assignments - .get(&assignment_key) - .map_or(0, VecDeque::len), - 1 - ); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: "final-child".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 2, - stderr_bytes: 0, - stdout_tail: "42".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: Some(TaskBoundaryValue::SmallJson(json!(42))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.task_assignments.contains_key(&assignment_key)); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-unpolled-completion".to_owned(), - restart: false, - }) - .expect("unpolled terminal completion must release the one-process slot"); -} - -#[test] -fn completed_main_retires_from_authoritative_state_after_event_history_rotates() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - - for index in 0..=MAX_TASK_EVENTS_PER_PROCESS { - service.record_task_completion_event(TaskCompletionEvent { - tenant: tenant.clone(), - project: project.clone(), - process: process.clone(), - node: NodeId::from("worker"), - executor: TaskExecutor::Node, - task_definition: TaskDefinitionId::from("historical"), - task: TaskInstanceId::new(format!("historical-{index}")), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); - } - assert!( - service.task_events.iter().all(|event| { - event.process != process || event.executor != TaskExecutor::CoordinatorMain - }), - "the regression requires bounded history to have rotated the main event" - ); - - complete_terminal_matrix_child(&mut service, TaskTerminalState::Completed); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - let summary = service - .process_summaries - .get(&process_control_key(&tenant, &project, &process)) - .expect("the terminal process summary must remain authoritative"); - assert_eq!(summary.final_result, Some(ProcessFinalResult::Completed)); - assert_eq!( - summary.main_terminal_state, - Some(TaskTerminalState::Completed) - ); -} - -#[test] -fn completed_main_await_operator_blocks_retirement_until_each_resolution() { - for resolution in [ - TaskFailureResolution::AcceptFailure, - TaskFailureResolution::Cancel, - ] { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::AwaitOperator, - ); - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let process_key = process_control_key(&tenant, &project, &process); - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.debug_epochs.contains_key(&process_key)); - assert!(service.debug_breakpoints.contains_key(&process_key)); - assert!(service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - let attempt = service - .task_attempts - .get(&super::keys::task_restart_key( - &tenant, - &project, - &process, - &TaskInstanceId::from("final-child"), - )) - .and_then(|attempts| attempts.iter().rev().find(|attempt| attempt.current)) - .unwrap(); - assert_eq!(attempt.state, TaskAttemptState::FailedAwaitingAction); - - service - .handle_request(CoordinatorRequest::ResolveTaskFailure { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: process.to_string(), - task: "final-child".to_owned(), - resolution, - }) - .unwrap(); - - assert!( - service - .coordinator - .active_process(&tenant, &project, &process) - .is_none(), - "{resolution:?} left the process slot active" - ); - assert!(!service.debug_epochs.contains_key(&process_key)); - assert!(!service.debug_breakpoints.contains_key(&process_key)); - assert!(!service - .panel_snapshots - .contains_key(&super::keys::panel_stop_key(&tenant, &project, &process))); - assert!(service.debug_commands.keys().all( - |(task_tenant, task_project, task_process, _, _)| { - task_tenant != &tenant || task_project != &project || task_process != &process - } - )); - let join = service.task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ); - assert_eq!( - join.state, - match resolution { - TaskFailureResolution::AcceptFailure => TaskJoinState::Failed, - TaskFailureResolution::Cancel => TaskJoinState::Cancelled, - } - ); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "next-after-resolution".to_owned(), - restart: false, - }) - .expect("operator resolution must release the one-process project slot"); - } -} - -#[test] -fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() { - let mut service = service_with_completed_main_and_final_child( - clusterflux_core::TaskFailurePolicy::AwaitOperator, - ); - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let task = TaskInstanceId::from("final-child"); - - let CoordinatorResponse::TaskRestart { accepted, .. } = service - .handle_request(CoordinatorRequest::RestartTask { - tenant: tenant.to_string(), - project: project.to_string(), - actor_user: "user".to_owned(), - process: process.to_string(), - task: task.to_string(), - replacement_bundle: None, - }) - .unwrap() - else { - panic!("expected task restart"); - }; - assert!(accepted); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: task.to_string(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 2, - stderr_bytes: 0, - stdout_tail: "ok".to_owned(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: Some(TaskBoundaryValue::SmallJson(json!("ok"))), - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - let attempts = service - .task_attempts - .get(&super::keys::task_restart_key( - &tenant, &project, &process, &task, - )) - .unwrap(); - assert!( - attempts - .iter() - .any(|attempt| !attempt.current - && attempt.state == TaskAttemptState::FailedAwaitingAction) - ); - assert!(attempts - .iter() - .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed)); - assert_eq!( - service - .process_summaries - .get(&process_control_key(&tenant, &project, &process)) - .and_then(|summary| summary.final_result.clone()), - Some(ProcessFinalResult::Completed), - "a successful current retry must override the superseded failed attempt" - ); - assert_eq!( - service - .task_join_result(tenant, project, process, task) - .state, - TaskJoinState::Completed - ); -} - -#[test] -fn completed_main_failed_child_does_not_abort_another_active_child() { - let mut service = - service_with_completed_main_and_final_child(clusterflux_core::TaskFailurePolicy::FailFast); - register_test_task_assignment( - &mut service, - "tenant", - "project", - "terminal-matrix", - "worker", - "other-child-definition", - "other-child", - 83, - ); - - complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let process = ProcessId::from("terminal-matrix"); - let other_key = task_control_key( - &tenant, - &project, - &process, - &NodeId::from("worker"), - &TaskInstanceId::from("other-child"), - ); - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_some()); - assert!(service.active_tasks.contains(&other_key)); - assert!(!service.task_aborts.contains(&other_key)); - assert_eq!( - service - .task_join_result( - tenant.clone(), - project.clone(), - process.clone(), - TaskInstanceId::from("final-child"), - ) - .state, - TaskJoinState::Failed - ); - - service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: tenant.to_string(), - project: project.to_string(), - process: process.to_string(), - node: "worker".to_owned(), - task: "other-child".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }) - .unwrap(); - - assert!(service - .coordinator - .active_process(&tenant, &project, &process) - .is_none()); - assert!(!service.active_tasks.contains(&other_key)); -} - #[test] fn quiescent_cooperative_cancel_releases_slot_immediately() { let mut service = CoordinatorService::new(17); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5003,7 +3586,6 @@ fn quiescent_cooperative_cancel_releases_slot_immediately() { let started = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5016,10 +3598,7 @@ fn quiescent_cooperative_cancel_releases_slot_immediately() { .unwrap(); assert!(matches!( started, - CoordinatorResponse::ProcessStarted { - launch_attempt: None, - .. - } + CoordinatorResponse::ProcessStarted { .. } )); assert!(service.task_events.iter().all(|event| { event.tenant != TenantId::from("tenant") @@ -5046,7 +3625,6 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5059,8 +3637,6 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 17, @@ -5079,7 +3655,6 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() { let CoordinatorResponse::ProcessAborted { aborted_tasks, .. } = service .handle_request(CoordinatorRequest::AbortProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: "user".to_owned(), @@ -5156,7 +3731,6 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5370,7 +3944,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_tenant.to_string().contains("does not exist")); + assert!(cross_tenant.to_string().contains("tenant mismatch")); let cross_project = service .handle_request(CoordinatorRequest::CreateArtifactDownloadLink { @@ -5382,7 +3956,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { ttl_seconds: 60, }) .unwrap_err(); - assert!(cross_project.to_string().contains("does not exist")); + assert!(cross_project.to_string().contains("project mismatch")); let cross_tenant_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5395,7 +3969,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_open.to_string().contains("does not exist")); + assert!(cross_tenant_open.to_string().contains("tenant mismatch")); let cross_project_open = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5408,7 +3982,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_open.to_string().contains("does not exist")); + assert!(cross_project_open.to_string().contains("project mismatch")); let guessed = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5553,7 +4127,6 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5664,8 +4237,6 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), process: "process".to_owned(), epoch: 7, @@ -5710,7 +4281,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_tenant_retry.to_string().contains("does not exist")); + assert!(cross_tenant_retry.to_string().contains("tenant mismatch")); let cross_project_retry = service .handle_request(CoordinatorRequest::OpenArtifactDownloadStream { @@ -5723,7 +4294,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() { chunk_bytes: 1, }) .unwrap_err(); - assert!(cross_project_retry.to_string().contains("does not exist")); + assert!(cross_project_retry.to_string().contains("project mismatch")); } #[test] @@ -5754,7 +4325,6 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5876,29 +4446,6 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() { #[test] fn windows_task_events_share_the_virtual_process_scope() { let mut service = CoordinatorService::new(7); - service.record_task_completion_event(TaskCompletionEvent { - tenant: TenantId::from("other-tenant"), - project: ProjectId::from("other-project"), - process: ProcessId::from("other-process"), - node: NodeId::from("other-node"), - executor: super::TaskExecutor::Node, - task_definition: TaskDefinitionId::from("other-task"), - task: TaskInstanceId::from("other-task"), - attempt_id: None, - placement: None, - terminal_state: TaskTerminalState::Completed, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - artifact_digest: None, - artifact_size_bytes: None, - result: None, - }); service .handle_request(CoordinatorRequest::AttachNode { tenant: "tenant".to_owned(), @@ -5923,7 +4470,6 @@ fn windows_task_events_share_the_virtual_process_scope() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -5999,28 +4545,6 @@ fn windows_task_events_share_the_virtual_process_scope() { #[test] fn service_schedules_task_across_reported_node_descriptors() { let mut service = CoordinatorService::new(7); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - node: "other-node".to_owned(), - public_key: test_node_public_key("other-node"), - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - node: "other-node".to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: false, - online: true, - }) - .unwrap(); for node in ["cold-node", "warm-node"] { service .handle_request(CoordinatorRequest::AttachNode { @@ -6210,13 +4734,8 @@ fn coordinator_side_task_launch_queues_worker_assignment() { online: true, }) .unwrap(); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - epoch, - .. - } = service + let CoordinatorResponse::ProcessStarted { epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -6232,8 +4751,6 @@ fn coordinator_side_task_launch_queues_worker_assignment() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker-linux".to_owned(), process: "vp-control".to_owned(), epoch, @@ -6542,7 +5059,6 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), @@ -6555,8 +5071,6 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 41, @@ -6739,7 +5253,6 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( replacement_bundle: Some(TaskReplacementBundle { bundle_digest: replacement_bundle_digest.clone(), wasm_module_base64: replacement_module.clone(), - source_snapshot: None, }), }) .unwrap() @@ -6805,7 +5318,6 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order( replacement_bundle: Some(TaskReplacementBundle { bundle_digest: incompatible_bundle_digest, wasm_module_base64: incompatible_module, - source_snapshot: None, }), }) .unwrap() @@ -6871,7 +5383,6 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant-soak".to_owned(), project: "project-soak".to_owned(), actor_user: Some("user-soak".to_owned()), @@ -6884,8 +5395,6 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant-soak".to_owned(), - project: "project-soak".to_owned(), node: "worker-soak".to_owned(), process: "vp-soak".to_owned(), epoch: 73, @@ -6952,29 +5461,14 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { } for index in 0..2_000 { - service.node_replay_nonces.insert( - ( - crate::NodeScopeKey::new( - TenantId::from("tenant-soak"), - ProjectId::from("project-soak"), - NodeId::from("worker-soak"), - ), - format!("expired-{index}"), - ), - 0, - ); + service + .node_replay_nonces + .insert((NodeId::from("worker-soak"), format!("expired-{index}")), 0); } service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant-soak".to_owned(), - project: "project-soak".to_owned(), node: "worker-soak".to_owned(), - node_signature: Some(signed_node_heartbeat_in_scope( - "tenant-soak", - "project-soak", - "worker-soak", - "post-expiry-prune", - )), + node_signature: Some(signed_node_heartbeat("worker-soak", "post-expiry-prune")), }) .unwrap(); @@ -7027,14 +5521,7 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() { let retained_soak_nonces = service .node_replay_nonces .keys() - .filter(|(scope, _)| { - scope - == &crate::NodeScopeKey::new( - TenantId::from("tenant-soak"), - ProjectId::from("project-soak"), - NodeId::from("worker-soak"), - ) - }) + .filter(|(node, _)| node == &NodeId::from("worker-soak")) .count(); assert_eq!(retained_soak_events, super::MAX_TASK_EVENTS_PER_PROCESS); @@ -7092,7 +5579,6 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: Some("user".to_owned()), @@ -7105,8 +5591,6 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() { .unwrap(); service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "worker".to_owned(), process: "vp".to_owned(), epoch: 12, @@ -7205,13 +5689,8 @@ fn coordinator_rejects_named_environment_without_requirements() { online: true, }) .unwrap(); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - epoch, - .. - } = service + let CoordinatorResponse::ProcessStarted { epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -7227,8 +5706,6 @@ fn coordinator_rejects_named_environment_without_requirements() { }; service .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "uncached-worker".to_owned(), process: "vp-environment".to_owned(), epoch, @@ -7268,7 +5745,6 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { let mut service = CoordinatorService::new(10); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -7308,59 +5784,8 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { #[test] fn coordinator_side_task_launch_can_wait_for_capable_worker() { let mut service = CoordinatorService::new(11); - let CoordinatorResponse::ProcessStarted { - epoch: other_epoch, .. - } = service + let CoordinatorResponse::ProcessStarted { epoch, .. } = service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "other-vp-wait".to_owned(), - restart: false, - }) - .unwrap() - else { - panic!("expected unrelated process start"); - }; - let CoordinatorResponse::TaskQueued { - queued_tasks: other_queued_tasks, - .. - } = service - .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask { - task_spec: test_task_spec( - "other-tenant", - "other-project", - "other-vp-wait", - "other-compile", - other_epoch, - [Capability::Command], - ), - tenant: "other-tenant".to_owned(), - project: "other-project".to_owned(), - actor_user: Some("other-user".to_owned()), - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - wait_for_node: true, - artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(), - wasm_module_base64: test_wasm_module_base64(), - }) - .unwrap() - else { - panic!("expected unrelated queued task launch"); - }; - assert_eq!(other_queued_tasks, 1); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - epoch, - .. - } = service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant".to_owned(), project: "project".to_owned(), actor_user: None, @@ -7556,172 +5981,6 @@ fn service_meters_rendezvous_before_direct_transfer_plan() { assert!(quota.to_string().contains("RendezvousAttempt")); } -#[test] -fn enrolled_node_can_request_a_structurally_scoped_rendezvous_with_signed_authority() { - let mut service = CoordinatorService::new(7); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node-a".to_owned(), - public_key: test_node_public_key("node-a"), - }) - .unwrap(); - - let response = service - .handle_signed_node_request_auto(CoordinatorRequest::RequestRendezvous { - scope: data_plane_scope("project"), - source: endpoint("node-a"), - destination: endpoint("node-b"), - direct_connectivity: true, - failure_reason: String::new(), - }) - .unwrap(); - - let CoordinatorResponse::RendezvousPlan { plan, .. } = response else { - panic!("expected signed rendezvous plan"); - }; - assert_eq!(plan.source.node, NodeId::from("node-a")); - assert_eq!(plan.destination.node, NodeId::from("node-b")); -} - -#[test] -fn signed_hostile_artifact_paths_return_errors_and_the_same_service_stays_healthy() { - let mut service = CoordinatorService::new(27); - service - .handle_request(CoordinatorRequest::AttachNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - public_key: test_node_public_key("node"), - }) - .unwrap(); - service - .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, - tenant: "tenant".to_owned(), - project: "project".to_owned(), - actor_user: None, - actor_agent: None, - agent_public_key_fingerprint: None, - agent_signature: None, - process: "hostile-path-process".to_owned(), - restart: false, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: "node".to_owned(), - process: "hostile-path-process".to_owned(), - epoch: 27, - }) - .unwrap(); - - let invalid_metadata = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "metadata-task".to_owned(), - artifact_path: Some("/vfs/artifacts/bad artifact!".to_owned()), - artifact_digest: Some(Digest::sha256("bad")), - artifact_size_bytes: Some(3), - large_bytes_uploaded: false, - }) - .unwrap_err(); - assert!( - invalid_metadata - .to_string() - .contains("invalid VFS artifact path"), - "unexpected error: {invalid_metadata}" - ); - - let valid_metadata = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportVfsMetadata { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "metadata-task".to_owned(), - artifact_path: Some("/vfs/artifacts/valid-artifact".to_owned()), - artifact_digest: Some(Digest::sha256("valid")), - artifact_size_bytes: Some(5), - large_bytes_uploaded: false, - }) - .unwrap(); - assert!(matches!( - valid_metadata, - CoordinatorResponse::VfsMetadataRecorded { .. } - )); - - register_test_task_assignment( - &mut service, - "tenant", - "project", - "hostile-path-process", - "node", - "child", - "child-instance", - 27, - ); - let invalid_completion = service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "child-instance".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/repeated//component".to_owned()), - artifact_digest: Some(Digest::sha256("bad-completion")), - artifact_size_bytes: Some(0), - result: None, - }) - .unwrap_err(); - assert!( - invalid_completion - .to_string() - .contains("invalid VFS artifact path"), - "unexpected error: {invalid_completion}" - ); - - let valid_completion = service - .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - process: "hostile-path-process".to_owned(), - node: "node".to_owned(), - task: "child-instance".to_owned(), - terminal_state: Some(TaskTerminalState::Completed), - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: Some("/vfs/artifacts/valid-completion".to_owned()), - artifact_digest: Some(Digest::sha256("valid-completion")), - artifact_size_bytes: Some(0), - result: None, - }) - .unwrap(); - assert!(matches!( - valid_completion, - CoordinatorResponse::TaskRecorded { .. } - )); -} - #[test] fn service_rejects_task_completion_outside_node_scope() { let mut service = CoordinatorService::new(1); @@ -7735,7 +5994,6 @@ fn service_rejects_task_completion_outside_node_scope() { .unwrap(); service .handle_request(CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "tenant-b".to_owned(), project: "project-b".to_owned(), actor_user: None, @@ -7769,7 +6027,7 @@ fn service_rejects_task_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("outside")); let CoordinatorResponse::TaskEvents { events } = service .handle_request(CoordinatorRequest::ListTaskEvents { tenant: "tenant-b".to_owned(), @@ -7834,7 +6092,7 @@ fn service_rejects_node_capability_report_outside_enrollment_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("tenant/project scope")); assert!(service.node_descriptors.is_empty()); } @@ -7860,7 +6118,7 @@ fn service_rejects_source_preparation_completion_outside_node_scope() { }) .unwrap_err(); - assert!(error.to_string().contains("not enrolled")); + assert!(error.to_string().contains("tenant/project scope")); } #[test] @@ -7869,8 +6127,6 @@ fn service_rejects_unknown_node_heartbeat() { let error = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "missing".to_owned(), node_signature: None, }) @@ -7893,8 +6149,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let unsigned = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: None, }) @@ -7904,8 +6158,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let wrong_private_key = test_node_private_key("other-node"); let wrong_signature = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat_with_private_key( "node", @@ -7919,8 +6171,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let signed = signed_node_heartbeat("node", "fresh-node-heartbeat"); let heartbeat = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed.clone()), }) @@ -7935,8 +6185,6 @@ fn service_requires_signed_node_heartbeat_from_enrolled_key() { let replay = service .handle_request(CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed), }) @@ -8007,8 +6255,6 @@ fn service_stream_accepts_multiple_requests_on_one_connection() { write_coordinator_wire_request( &mut stream, &CoordinatorRequest::NodeHeartbeat { - tenant: "tenant".to_owned(), - project: "project".to_owned(), node: "node".to_owned(), node_signature: Some(signed_node_heartbeat("node", "stream-heartbeat")), }, @@ -8052,7 +6298,6 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { write_coordinator_wire_request( &mut stream, &CoordinatorRequest::StartProcess { - launch_attempt: None, tenant: "victim-tenant".to_owned(), project: "victim-project".to_owned(), actor_user: Some("forged-user".to_owned()), @@ -8067,23 +6312,18 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::Error { error } = + let CoordinatorResponse::Error { message } = serde_json::from_str::(&line).unwrap() else { panic!("expected strict body-authority denial"); }; - assert!(error - .message - .contains("request-body identity fields are not authority")); - assert_eq!(error.request_id, "strict-stream-forged"); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden); + assert!(message.contains("request-body identity fields are not authority")); write_coordinator_wire_request( &mut stream, &CoordinatorRequest::Authenticated { session_secret: "strict-stream-session".to_owned(), request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, process: "vp-authenticated".to_owned(), restart: false, }, @@ -8093,12 +6333,8 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() { line.clear(); reader.read_line(&mut line).unwrap(); - let CoordinatorResponse::ProcessStarted { - launch_attempt: None, - process, - actor, - .. - } = serde_json::from_str::(&line).unwrap() + let CoordinatorResponse::ProcessStarted { process, actor, .. } = + serde_json::from_str::(&line).unwrap() else { panic!("expected authenticated strict process start"); }; @@ -8132,14 +6368,10 @@ fn service_stream_rejects_invalid_versioned_envelope_metadata() { let mut line = String::new(); reader.read_line(&mut line).unwrap(); let response = serde_json::from_str::(&line).unwrap(); - let CoordinatorResponse::Error { error } = response else { + let CoordinatorResponse::Error { message } = response else { panic!("expected invalid wire envelope response"); }; - assert!(error - .message - .contains("operation attach_node does not match payload operation ping")); - assert_eq!(error.request_id, "bad-operation"); - assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError); + assert!(message.contains("operation attach_node does not match payload operation ping")); stream.shutdown(std::net::Shutdown::Both).unwrap(); server.join().unwrap(); @@ -8236,879 +6468,3 @@ fn coordinator_generates_and_bounds_node_enrollment_grants() { assert_ne!(first_grant, second_grant); assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60); } - -#[test] -fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret) in [ - ("tenant-a", "project-a", "user-a", "session-a"), - ("tenant-b", "project-b", "user-b", "session-b"), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - } - service.set_server_time(100); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: "process-one".to_owned(), - restart: false, - }, - }) - .unwrap(); - - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected process summaries"); - }; - assert_eq!(processes.len(), 1); - assert_eq!(processes[0].process, ProcessId::from("process-one")); - assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active); - assert_eq!(processes[0].activity, ProcessActivityState::Running); - assert_eq!(processes[0].started_at_epoch_seconds, 100); - assert!(next_cursor.is_none()); - - let CoordinatorResponse::ProcessSummaries { processes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 10, - }, - }) - .unwrap() - else { - panic!("expected scoped process summaries"); - }; - assert!(processes.is_empty()); - - service.set_server_time(120); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::AbortProcess { - process: "process-one".to_owned(), - launch_attempt: None, - }, - }) - .unwrap(); - let CoordinatorResponse::ProcessSummaries { processes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 10, - }, - }) - .unwrap() - else { - panic!("expected terminal process summary"); - }; - assert_eq!( - processes[0].lifecycle, - ProcessLifecycleState::RecentTerminal - ); - assert_eq!(processes[0].activity, ProcessActivityState::Cancelled); - assert_eq!( - processes[0].final_result, - Some(ProcessFinalResult::Cancelled) - ); - assert_eq!(processes[0].ended_at_epoch_seconds, Some(120)); - - service.set_server_time(130); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: "process-two".to_owned(), - restart: false, - }, - }) - .unwrap(); - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor: Some(cursor), - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected first process summary page"); - }; - assert_eq!(processes[0].process, ProcessId::from("process-two")); - let CoordinatorResponse::ProcessSummaries { - processes, - next_cursor: None, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: Some(cursor), - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected final process summary page"); - }; - assert_eq!(processes[0].process, ProcessId::from("process-one")); - - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListProcessSummaries { - cursor: None, - limit: 101, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("100")); -} - -#[test] -fn process_summary_eviction_releases_live_log_accounting_state() { - let mut service = CoordinatorService::new(7); - let tenant = TenantId::from("tenant-summary-bound"); - let project = ProjectId::from("project-summary-bound"); - let task = TaskInstanceId::from("task"); - - for index in 0..MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT { - let process = ProcessId::new(format!("process-{index:03}")); - service.record_process_started(&tenant, &project, &process, index as u64); - service.record_process_terminal( - &tenant, - &project, - &process, - ProcessFinalResult::Completed, - index as u64 + 1, - ); - let key = ( - tenant.clone(), - project.clone(), - process, - task.clone(), - "stdout".to_owned(), - ); - service.recent_log_accounted_bytes.insert(key.clone(), 10); - service.recent_log_truncated_streams.insert(key); - } - - let evicted = ProcessId::from("process-000"); - service.record_process_started(&tenant, &project, &ProcessId::from("process-next"), 1_000); - - assert!(!service.process_summaries.contains_key(&( - tenant.clone(), - project.clone(), - evicted.clone() - ))); - assert!(!service.recent_log_accounted_bytes.keys().any( - |(entry_tenant, entry_project, process, _, _)| { - entry_tenant == &tenant && entry_project == &project && process == &evicted - } - )); - assert!(!service.recent_log_truncated_streams.iter().any( - |(entry_tenant, entry_project, process, _, _)| { - entry_tenant == &tenant && entry_project == &project && process == &evicted - } - )); -} - -#[test] -fn web_node_summaries_are_scoped_paginated_and_hard_bounded() { - let mut service = CoordinatorService::new(7); - service - .issue_cli_session( - TenantId::from("tenant"), - ProjectId::from("project"), - UserId::from("user"), - "session", - None, - ) - .unwrap(); - for node in ["node-a", "node-b", "node-c"] { - enroll_test_node( - &mut service, - "tenant", - "project", - node, - &test_node_public_key(node), - ); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: "tenant".to_owned(), - project: "project".to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: Vec::new(), - direct_connectivity: true, - online: true, - }) - .unwrap(); - } - - let CoordinatorResponse::NodeSummaries { - nodes, - next_cursor: Some(cursor), - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected first node-summary page"); - }; - assert_eq!( - nodes - .iter() - .map(|node| node.id.as_str()) - .collect::>(), - ["node-a", "node-b"] - ); - - let CoordinatorResponse::NodeSummaries { - nodes, - next_cursor: None, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: Some(cursor), - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected final node-summary page"); - }; - assert_eq!(nodes.len(), 1); - assert_eq!(nodes[0].id, NodeId::from("node-c")); - - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 201, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("200")); -} - -#[test] -fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret, node) in [ - ("tenant-a", "project-a", "user-a", "session-a", "node-a"), - ("tenant-b", "project-b", "user-b", "session-b", "node-b"), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - enroll_test_node( - &mut service, - tenant, - project, - node, - &test_node_public_key(node), - ); - } - service.set_server_time(100); - for (tenant, project, node) in [ - ("tenant-a", "project-a", "node-a"), - ("tenant-b", "project-b", "node-b"), - ] { - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - capabilities: linux_capabilities(), - cached_environment_digests: Vec::new(), - dependency_cache_digests: Vec::new(), - source_snapshots: Vec::new(), - artifact_locations: vec!["shared-artifact".to_owned()], - direct_connectivity: true, - online: false, - }) - .unwrap(); - } - let CoordinatorResponse::NodeSummaries { nodes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected node summaries"); - }; - assert_eq!(nodes.len(), 1); - assert_eq!(nodes[0].id, NodeId::from("node-a")); - assert!(nodes[0].online); - assert!(!nodes[0].stale); - assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); - assert_eq!(nodes[0].capabilities.os, Os::Linux); - for (tenant, project, node, digest) in [ - ("tenant-a", "project-a", "node-a", "tenant-a-bytes"), - ("tenant-b", "project-b", "node-b", "tenant-b-bytes"), - ] { - service.artifact_registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("shared-artifact"), - tenant: TenantId::from(tenant), - project: ProjectId::from(project), - process: ProcessId::from("process-one"), - producer_task: TaskInstanceId::from("task-one"), - retaining_node: NodeId::from(node), - digest: Digest::sha256(digest), - size: digest.len() as u64, - }); - } - service.artifact_registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("second-artifact"), - tenant: TenantId::from("tenant-a"), - project: ProjectId::from("project-a"), - process: ProcessId::from("process-two"), - producer_task: TaskInstanceId::from("task-two"), - retaining_node: NodeId::from("node-a"), - digest: Digest::sha256("second"), - size: 6, - }); - - let CoordinatorResponse::Artifacts { - artifacts, - next_cursor: Some(cursor), - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListArtifacts { - process: None, - cursor: None, - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected first artifact page"); - }; - assert_eq!(artifacts.len(), 1); - assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact")); - assert_eq!(artifacts[0].availability, ArtifactAvailability::Available); - let CoordinatorResponse::Artifacts { - artifacts, - next_cursor: None, - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListArtifacts { - process: None, - cursor: Some(cursor), - limit: 1, - }, - }) - .unwrap() - else { - panic!("expected final artifact page"); - }; - assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact")); - assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes")); - - let cross_tenant = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "tenant-b-only".to_owned(), - }, - }) - .unwrap_err(); - assert!(cross_tenant.to_string().contains("does not exist")); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected tenant-b artifact"); - }; - assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes")); - - service.set_server_time(131); - let CoordinatorResponse::NodeSummaries { nodes, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListNodeSummaries { - cursor: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected stale node summary"); - }; - assert!(!nodes[0].online); - assert!(nodes[0].stale); - assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100)); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected offline artifact metadata"); - }; - assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline); - assert!(!artifact.downloadable_now); - - service - .artifact_registry - .sync_to_explicit_store( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - &ArtifactId::from("shared-artifact"), - "store://tenant-a/shared-artifact", - ) - .unwrap(); - let CoordinatorResponse::Artifact { artifact } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::GetArtifact { - artifact: "shared-artifact".to_owned(), - }, - }) - .unwrap() - else { - panic!("expected explicitly retained artifact metadata"); - }; - assert_eq!(artifact.availability, ArtifactAvailability::Available); - assert_eq!( - artifact.retention_state, - ArtifactRetentionState::ExplicitStorage - ); - assert!(artifact.downloadable_now); -} - -#[test] -fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() { - let mut service = CoordinatorService::new(7); - for (tenant, project, user, secret, node, process) in [ - ( - "tenant-a", - "project-a", - "user-a", - "session-a", - "node-a", - "process-shared", - ), - ( - "tenant-b", - "project-b", - "user-b", - "session-b", - "node-b", - "process-b", - ), - ] { - service - .issue_cli_session( - TenantId::from(tenant), - ProjectId::from(project), - UserId::from(user), - secret, - None, - ) - .unwrap(); - enroll_test_node( - &mut service, - tenant, - project, - node, - &test_node_public_key(node), - ); - service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: secret.to_owned(), - request: AuthenticatedCoordinatorRequest::StartProcess { - launch_attempt: None, - process: process.to_owned(), - restart: false, - }, - }) - .unwrap(); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode { - tenant: tenant.to_owned(), - project: project.to_owned(), - node: node.to_owned(), - process: process.to_owned(), - epoch: 7, - }) - .unwrap(); - } - service.set_server_time(100); - let first = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 0, - source_bytes: 5, - text: "hello".to_owned(), - truncated: false, - }) - .unwrap(); - let CoordinatorResponse::TaskLogChunkRecorded { - sequence: Some(first_sequence), - next_offset: 5, - .. - } = first - else { - panic!("expected first live log sequence"); - }; - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 5, - "live bytes must be charged when accepted" - ); - let retry = service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 0, - source_bytes: 5, - text: "hello".to_owned(), - truncated: false, - }) - .unwrap(); - assert!(matches!( - retry, - CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. } - )); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 5, - "a retried chunk must not be charged twice" - ); - service - .handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk { - tenant: "tenant-a".to_owned(), - project: "project-a".to_owned(), - process: "process-shared".to_owned(), - node: "node-a".to_owned(), - task: "task-one".to_owned(), - stream: TaskLogStream::Stdout, - offset: 8, - source_bytes: 2, - text: "ok".to_owned(), - truncated: false, - }) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 10, - "a gap and the delivered bytes must both count toward source-byte usage" - ); - - let CoordinatorResponse::RecentLogs { - entries, - next_sequence: Some(cursor), - history_truncated: false, - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected first recent-log page"); - }; - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].sequence, first_sequence); - assert_eq!(entries[0].text, "hello"); - assert!(entries[1].text.contains("3 bytes")); - assert!(entries[1].truncated); - let CoordinatorResponse::RecentLogs { entries, .. } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: Some(cursor), - limit: 2, - }, - }) - .unwrap() - else { - panic!("expected second recent-log page"); - }; - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].text, "ok"); - - service - .handle_report_task_log( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - 12, - 0, - "hello???okZZ".to_owned(), - String::new(), - false, - false, - false, - ) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 12, - "the final summary must charge only source bytes not already charged live" - ); - assert_eq!( - service - .recent_logs - .get(&(TenantId::from("tenant-a"), ProjectId::from("project-a"))) - .unwrap() - .back() - .unwrap() - .text, - "ZZ", - "final-tail reconciliation must append only the nonduplicating suffix" - ); - service - .handle_report_task_log( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - 12, - 0, - "hello???okZZ".to_owned(), - String::new(), - false, - false, - false, - ) - .unwrap(); - assert_eq!( - service.quota.used_log_bytes( - &TenantId::from("tenant-a"), - &ProjectId::from("project-a"), - 100, - ), - 12, - "replayed final accounting must be idempotent" - ); - - let marker = service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - 12, - 0, - "[log output truncated at node capture limit]".to_owned(), - true, - ) - .unwrap(); - assert!(matches!( - marker, - CoordinatorResponse::TaskLogChunkRecorded { - sequence: Some(_), - next_offset: 12, - .. - } - )); - let repeated_marker = service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - 12, - 0, - "[log output truncated at node capture limit]".to_owned(), - true, - ) - .unwrap(); - assert!(matches!( - repeated_marker, - CoordinatorResponse::TaskLogChunkRecorded { - sequence: None, - next_offset: 12, - .. - } - )); - - let cross_tenant = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-b".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 10, - }, - }) - .unwrap_err(); - assert!(cross_tenant.to_string().contains("outside")); - let oversized = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 201, - }, - }) - .unwrap_err(); - assert!(oversized.to_string().contains("limit")); - assert!(oversized.to_string().contains("200")); - - service - .handle_report_task_log_chunk( - "tenant-b".to_owned(), - "project-b".to_owned(), - "process-b".to_owned(), - "node-b".to_owned(), - "task-b".to_owned(), - TaskLogStream::Stderr, - 0, - 1, - "b".to_owned(), - false, - ) - .unwrap(); - for offset in 10..310 { - service - .handle_report_task_log_chunk( - "tenant-a".to_owned(), - "project-a".to_owned(), - "process-shared".to_owned(), - "node-a".to_owned(), - "task-one".to_owned(), - TaskLogStream::Stdout, - offset, - 1, - "x".to_owned(), - false, - ) - .unwrap(); - } - let tenant_a_logs = - &service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))]; - assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS); - let tenant_b_logs = - &service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))]; - assert_eq!(tenant_b_logs.len(), 1); - assert_eq!(tenant_b_logs[0].text, "b"); - let CoordinatorResponse::RecentLogs { - entries, - history_truncated, - .. - } = service - .handle_request(CoordinatorRequest::Authenticated { - session_secret: "session-a".to_owned(), - request: AuthenticatedCoordinatorRequest::ListRecentLogs { - process: "process-shared".to_owned(), - task: None, - after_sequence: None, - limit: 200, - }, - }) - .unwrap() - else { - panic!("expected bounded recent-log response"); - }; - assert_eq!(entries.len(), 200); - assert!(history_truncated); -} diff --git a/crates/clusterflux-coordinator/src/service/wire_protocol.rs b/crates/clusterflux-coordinator/src/service/wire_protocol.rs index 07aec2a..4884415 100644 --- a/crates/clusterflux-coordinator/src/service/wire_protocol.rs +++ b/crates/clusterflux-coordinator/src/service/wire_protocol.rs @@ -1,4 +1,4 @@ -use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE}; +use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -12,12 +12,8 @@ pub enum CoordinatorWireRequest { impl CoordinatorWireRequest { pub fn into_request(self) -> Result { - self.into_parts().map(|(_, request)| request) - } - - pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { match self { - Self::Envelope(envelope) => envelope.into_parts(), + Self::Envelope(envelope) => envelope.into_request(), } } } @@ -36,10 +32,6 @@ pub struct CoordinatorRequestEnvelope { impl CoordinatorRequestEnvelope { pub fn into_request(self) -> Result { - self.into_parts().map(|(_, request)| request) - } - - pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> { if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE { return Err(format!( "unsupported coordinator wire request type {}; expected {}", @@ -52,9 +44,9 @@ impl CoordinatorRequestEnvelope { self.protocol_version, COORDINATOR_PROTOCOL_VERSION )); } - RequestId::try_new(self.request_id.clone()) - .map_err(|error| format!("malformed coordinator wire request_id: {error}"))?; - self.payload.validate_external_identifiers()?; + if self.request_id.trim().is_empty() { + return Err("coordinator wire request_id must be non-empty".to_owned()); + } let payload_operation = self.payload.operation()?; if self.operation != payload_operation { return Err(format!( @@ -62,6 +54,6 @@ impl CoordinatorRequestEnvelope { self.operation, payload_operation )); } - Ok((self.request_id, self.payload)) + Ok(self.payload) } } diff --git a/crates/clusterflux-core/src/api_error.rs b/crates/clusterflux-core/src/api_error.rs deleted file mode 100644 index b6625cf..0000000 --- a/crates/clusterflux-core/src/api_error.rs +++ /dev/null @@ -1,296 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::fmt; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ApiErrorCode { - Unauthenticated, - SessionExpired, - AccountSuspended, - Forbidden, - ValidationError, - NotFound, - Conflict, - ActiveProcessExists, - NodeOffline, - NoCapableNode, - TaskNotRestartable, - ArtifactUnavailable, - ArtifactLimitExceeded, - QuotaExceeded, - TemporaryCapacity, - DebugEpochPartial, - InternalError, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ApiErrorCategory { - Authentication, - Authorization, - Validation, - State, - Availability, - Resource, - Internal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ApiError { - pub code: ApiErrorCode, - pub category: ApiErrorCategory, - pub message: String, - pub retryable: bool, - pub request_id: String, -} - -impl fmt::Display for ApiError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "{} (code {:?}, request {})", - self.message, self.code, self.request_id - ) - } -} - -impl std::error::Error for ApiError {} - -impl ApiError { - pub fn new( - code: ApiErrorCode, - category: ApiErrorCategory, - message: impl Into, - retryable: bool, - request_id: impl Into, - ) -> Self { - Self { - code, - category, - message: message.into(), - retryable, - request_id: request_id.into(), - } - } - - pub fn from_message(request_id: impl Into, message: impl Into) -> Self { - let request_id = request_id.into(); - let message = message.into(); - let normalized = message.to_ascii_lowercase(); - let (code, category, retryable) = if normalized.contains("session credential has expired") - || normalized.contains("session expired") - { - ( - ApiErrorCode::SessionExpired, - ApiErrorCategory::Authentication, - false, - ) - } else if normalized.contains("tenant is suspended") - || normalized.contains("account is suspended") - || normalized.contains("suspended by hosted") - { - ( - ApiErrorCode::AccountSuspended, - ApiErrorCategory::Authorization, - false, - ) - } else if normalized.contains("session credential") - || normalized.contains("no authenticated") - || normalized.contains("not authenticated") - || normalized.contains("credential is not") - { - ( - ApiErrorCode::Unauthenticated, - ApiErrorCategory::Authentication, - false, - ) - } else if normalized.contains("project already has active virtual process") { - ( - ApiErrorCode::ActiveProcessExists, - ApiErrorCategory::State, - false, - ) - } else if normalized.contains("no capable node") { - ( - ApiErrorCode::NoCapableNode, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("node offline") - || normalized.contains("node is not live") - || normalized.contains("source node is not connected") - || normalized.contains("direct connectivity unavailable") - { - ( - ApiErrorCode::NodeOffline, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("restart") - && (normalized.contains("not restartable") - || normalized.contains("requires whole") - || normalized.contains("clean boundary")) - { - ( - ApiErrorCode::TaskNotRestartable, - ApiErrorCategory::State, - false, - ) - } else if normalized.contains("artifact") - && (normalized.contains("exceeds download limit") - || normalized.contains("download session limit") - || normalized.contains("artifact limit")) - { - ( - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCategory::Resource, - false, - ) - } else if normalized.contains("artifact") - && (normalized.contains("does not exist") - || normalized.contains("not found") - || normalized.contains("unknown artifact")) - { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } else if normalized.contains("artifact") - && (normalized.contains("unavailable") || normalized.contains("retention")) - { - ( - ApiErrorCode::ArtifactUnavailable, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("resource limit") - || normalized.contains("quota") - || normalized.contains("limit exceeded") - { - ( - ApiErrorCode::QuotaExceeded, - ApiErrorCategory::Resource, - true, - ) - } else if normalized.contains("capacity") - || normalized.contains("temporarily full") - || normalized.contains("replay window is full") - { - ( - ApiErrorCode::TemporaryCapacity, - ApiErrorCategory::Availability, - true, - ) - } else if normalized.contains("partial debug epoch") - || normalized.contains("debug epoch is partially") - { - ( - ApiErrorCode::DebugEpochPartial, - ApiErrorCategory::State, - true, - ) - } else if normalized.contains("malformed") - || normalized.contains("invalid ") - || normalized.contains("protocol") - || normalized.contains("unknown field") - || normalized.contains("missing field") - || normalized.contains("must ") - { - ( - ApiErrorCode::ValidationError, - ApiErrorCategory::Validation, - false, - ) - } else if normalized.contains("outside") - || normalized.contains("unauthorized") - || normalized.contains("denied") - || normalized.contains("requires an authenticated") - || normalized.contains("may only") - { - ( - ApiErrorCode::Forbidden, - ApiErrorCategory::Authorization, - false, - ) - } else if normalized.contains("not found") - || normalized.contains("does not exist") - || normalized.contains("unknown ") - { - (ApiErrorCode::NotFound, ApiErrorCategory::State, false) - } else if normalized.contains("already") - || normalized.contains("conflict") - || normalized.contains("requires an active") - { - (ApiErrorCode::Conflict, ApiErrorCategory::State, false) - } else { - ( - ApiErrorCode::InternalError, - ApiErrorCategory::Internal, - false, - ) - }; - Self::new(code, category, message, retryable, request_id) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn required_machine_codes_are_stably_serialized() { - let required = [ - ApiErrorCode::Unauthenticated, - ApiErrorCode::SessionExpired, - ApiErrorCode::AccountSuspended, - ApiErrorCode::Forbidden, - ApiErrorCode::ValidationError, - ApiErrorCode::NotFound, - ApiErrorCode::Conflict, - ApiErrorCode::ActiveProcessExists, - ApiErrorCode::NodeOffline, - ApiErrorCode::NoCapableNode, - ApiErrorCode::TaskNotRestartable, - ApiErrorCode::ArtifactUnavailable, - ApiErrorCode::ArtifactLimitExceeded, - ApiErrorCode::QuotaExceeded, - ApiErrorCode::TemporaryCapacity, - ApiErrorCode::DebugEpochPartial, - ]; - let serialized = required - .into_iter() - .map(|code| serde_json::to_value(code).unwrap()) - .collect::>(); - assert_eq!( - serialized, - vec![ - "unauthenticated", - "session_expired", - "account_suspended", - "forbidden", - "validation_error", - "not_found", - "conflict", - "active_process_exists", - "node_offline", - "no_capable_node", - "task_not_restartable", - "artifact_unavailable", - "artifact_limit_exceeded", - "quota_exceeded", - "temporary_capacity", - "debug_epoch_partial", - ] - ); - } - - #[test] - fn message_classification_keeps_request_identity() { - let error = ApiError::from_message( - "request-17", - "CLI session credential has expired; run login again", - ); - assert_eq!(error.code, ApiErrorCode::SessionExpired); - assert_eq!(error.category, ApiErrorCategory::Authentication); - assert_eq!(error.request_id, "request-17"); - assert!(!error.retryable); - } -} diff --git a/crates/clusterflux-core/src/artifact.rs b/crates/clusterflux-core/src/artifact.rs index d9c89a3..fffd58a 100644 --- a/crates/clusterflux-core/src/artifact.rs +++ b/crates/clusterflux-core/src/artifact.rs @@ -10,7 +10,7 @@ use crate::{ const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32; const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60; -const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024; +const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -134,27 +134,6 @@ pub struct ArtifactMetadata { pub coordinator_has_large_bytes: bool, } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -pub struct ArtifactScopeKey { - pub tenant: TenantId, - pub project: ProjectId, - pub artifact: ArtifactId, -} - -impl ArtifactScopeKey { - pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self { - Self { - tenant, - project, - artifact, - } - } - - pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self { - Self::new(tenant.clone(), project.clone(), artifact.clone()) - } -} - #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArtifactFlush { pub id: ArtifactId, @@ -169,7 +148,7 @@ pub struct ArtifactFlush { #[derive(Clone, Debug, Default)] pub struct ArtifactRegistry { - artifacts: BTreeMap, + artifacts: BTreeMap, issued_download_links: BTreeMap, next_epoch: u64, } @@ -183,46 +162,42 @@ impl ArtifactRegistry { pub fn flush_metadata_bounded( &mut self, flush: ArtifactFlush, - pinned: &BTreeSet, + pinned: &BTreeSet, ) -> Result { - self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new()) - } - - pub fn flush_metadata_with_protected_processes( - &mut self, - flush: ArtifactFlush, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> Result { - let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id); - let replacing = self.artifacts.contains_key(&key); - let retained_for_project = self - .artifacts - .values() - .filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project) - .count(); - let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT { - None - } else { - let mut protected_processes = protected_processes.clone(); - protected_processes.insert(flush.process.clone()); - Some( - self.project_metadata_eviction_candidate( - &flush.tenant, - &flush.project, - pinned, - &protected_processes, - ) + let replacing_existing = self.artifacts.contains_key(&flush.id); + while !replacing_existing + && self + .artifacts + .values() + .filter(|metadata| { + metadata.tenant == flush.tenant + && metadata.project == flush.project + && metadata.process == flush.process + }) + .count() + >= MAX_ARTIFACT_METADATA_PER_PROCESS + { + let candidate = self + .artifacts + .values() + .filter(|metadata| { + metadata.tenant == flush.tenant + && metadata.project == flush.project + && metadata.process == flush.process + && !pinned.contains(&metadata.id) + && !self.issued_download_links.values().any(|issued| { + issued.link.artifact == metadata.id + }) + }) + .min_by_key(|metadata| metadata.flushed_epoch) + .map(|metadata| metadata.id.clone()) .ok_or_else(|| { - format!( - "artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \ - exhausted by active or retained artifacts" - ) - })?, - ) - }; - - self.next_epoch = self.next_epoch.saturating_add(1); + "artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download" + .to_owned() + })?; + self.artifacts.remove(&candidate); + } + self.next_epoch += 1; let metadata = ArtifactMetadata { id: flush.id.clone(), tenant: flush.tenant, @@ -237,137 +212,43 @@ impl ArtifactRegistry { explicit_locations: Vec::new(), coordinator_has_large_bytes: false, }; - if let Some(eviction) = eviction { - self.artifacts.remove(&eviction); - } - self.artifacts.insert(key, metadata.clone()); + self.artifacts.insert(flush.id, metadata.clone()); Ok(metadata) } - pub fn enforce_project_metadata_limit( - &mut self, - tenant: &TenantId, - project: &ProjectId, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> usize { - let mut evicted = 0; - while self - .artifacts - .values() - .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) - .count() - > MAX_ARTIFACT_METADATA_PER_PROJECT - { - let candidate = self.project_metadata_eviction_candidate( - tenant, - project, - pinned, - protected_processes, - ); - let Some(candidate) = candidate else { - break; - }; - self.artifacts.remove(&candidate); - evicted += 1; - } - evicted - } - - fn project_metadata_eviction_candidate( - &self, - tenant: &TenantId, - project: &ProjectId, - pinned: &BTreeSet, - protected_processes: &BTreeSet, - ) -> Option { - self.artifacts - .values() - .filter(|metadata| { - &metadata.tenant == tenant - && &metadata.project == project - && !pinned.contains(&ArtifactScopeKey::from_refs( - &metadata.tenant, - &metadata.project, - &metadata.id, - )) - && !protected_processes.contains(&metadata.process) - && metadata.explicit_locations.is_empty() - && !self.issued_download_links.values().any(|issued| { - !issued.revoked - && issued.link.tenant == metadata.tenant - && issued.link.project == metadata.project - && issued.link.artifact == metadata.id - }) - }) - .min_by_key(|metadata| metadata.flushed_epoch) - .map(|metadata| { - ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id) - }) - } - pub fn sync_to_explicit_store( &mut self, - tenant: &TenantId, - project: &ProjectId, artifact: &ArtifactId, location: impl Into, ) -> Result<(), ArtifactUnavailable> { let metadata = self .artifacts - .get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact)) + .get_mut(artifact) .ok_or(ArtifactUnavailable)?; metadata.explicit_locations.push(location.into()); Ok(()) } - pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) { - for metadata in self - .artifacts - .values_mut() - .filter(|metadata| &metadata.tenant == tenant && &metadata.project == project) - { + pub fn garbage_collect_node(&mut self, node: &NodeId) { + for metadata in self.artifacts.values_mut() { metadata.retaining_nodes.remove(node); } } pub fn reconcile_node_retention( &mut self, - tenant: &TenantId, - project: &ProjectId, node: &NodeId, retained_artifacts: &BTreeSet, ) { - for (key, metadata) in &mut self.artifacts { - if &key.tenant != tenant || &key.project != project { - continue; - } - if retained_artifacts.contains(&key.artifact) { - metadata.retaining_nodes.insert(node.clone()); - } else { + for (artifact, metadata) in &mut self.artifacts { + if !retained_artifacts.contains(artifact) { metadata.retaining_nodes.remove(node); } } } - pub fn metadata( - &self, - tenant: &TenantId, - project: &ProjectId, - artifact: &ArtifactId, - ) -> Option<&ArtifactMetadata> { - self.artifacts - .get(&ArtifactScopeKey::from_refs(tenant, project, artifact)) - } - - pub fn metadata_for_project<'a>( - &'a self, - tenant: &'a TenantId, - project: &'a ProjectId, - ) -> impl Iterator + 'a { - self.artifacts - .values() - .filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project) + pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> { + self.artifacts.get(artifact) } pub fn download_action( @@ -378,11 +259,7 @@ impl ArtifactRegistry { ) -> Result { let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; let scope = Scope { tenant: metadata.tenant.clone(), @@ -436,11 +313,7 @@ impl ArtifactRegistry { self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; Ok(metadata.size) } @@ -458,11 +331,7 @@ impl ArtifactRegistry { if self .issued_download_links .values() - .filter(|issued| { - issued.link.tenant == context.tenant - && issued.link.project == context.project - && issued.link.artifact == *artifact - }) + .filter(|issued| issued.link.artifact == *artifact) .count() >= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { @@ -474,11 +343,7 @@ impl ArtifactRegistry { let action = self.download_action(context, artifact, policy)?; let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds); let policy_context_digest = @@ -531,11 +396,7 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.tenant != context.tenant - || issued.link.project != context.project - || issued.link.artifact != *artifact - || issued.link.actor != context.actor - { + if issued.link.artifact != *artifact || issued.link.actor != context.actor { return Err(DownloadError::InvalidToken); } self.download_action( @@ -571,9 +432,7 @@ impl ArtifactRegistry { .issued_download_links .get(presented_token_digest) .ok_or(DownloadError::InvalidToken)?; - if issued.link.tenant != context.tenant - || issued.link.project != context.project - || issued.link.artifact != *artifact + if issued.link.artifact != *artifact || issued.link.max_bytes != policy.max_bytes || issued.link.actor != context.actor { @@ -591,11 +450,7 @@ impl ArtifactRegistry { } let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &context.tenant, - &context.project, - artifact, - )) + .get(artifact) .ok_or(DownloadError::NotFound)?; if download_policy_context_digest(metadata, &action.source, policy) != issued.link.policy_context_digest @@ -620,11 +475,7 @@ impl ArtifactRegistry { ) -> Result<(), DownloadError> { let metadata = self .artifacts - .get(&ArtifactScopeKey::from_refs( - &stream.link.tenant, - &stream.link.project, - &stream.link.artifact, - )) + .get(&stream.link.artifact) .ok_or(DownloadError::NotFound)?; if !source_is_available(metadata, &stream.link.source) { return Err(DownloadError::Unavailable); @@ -743,13 +594,7 @@ mod tests { #[test] fn flush_publishes_metadata_without_coordinator_bytes() { let registry = registry_with_artifact(); - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) - .unwrap(); + let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); assert!(!metadata.coordinator_has_large_bytes); assert_eq!(metadata.id, ArtifactId::from("artifact")); @@ -768,11 +613,7 @@ mod tests { #[test] fn unsynced_node_loss_surfaces_as_unavailable() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -793,68 +634,20 @@ mod tests { #[test] fn signed_node_retention_inventory_removes_garbage_collected_location() { let mut registry = registry_with_artifact(); - registry.reconcile_node_retention( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - &BTreeSet::new(), - ); + registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new()); - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) - .unwrap(); + let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap(); assert!(metadata.retaining_nodes.is_empty()); } - #[test] - fn signed_node_retention_inventory_restores_readvertised_location() { - let mut registry = registry_with_artifact(); - let node = NodeId::from("node"); - let artifact = ArtifactId::from("artifact"); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &node, - ); - - registry.reconcile_node_retention( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &node, - &BTreeSet::from([artifact.clone()]), - ); - - let metadata = registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &artifact, - ) - .unwrap(); - assert!(metadata.retaining_nodes.contains(&node)); - } - #[test] fn explicit_user_storage_location_survives_node_retention_loss() { let mut registry = registry_with_artifact(); registry - .sync_to_explicit_store( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - "s3://bucket/app", - ) + .sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app") .unwrap(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -875,11 +668,7 @@ mod tests { ); assert!( !registry - .metadata( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &ArtifactId::from("artifact"), - ) + .metadata(&ArtifactId::from("artifact")) .unwrap() .coordinator_has_large_bytes ); @@ -910,7 +699,7 @@ mod tests { ) .unwrap_err(); - assert_eq!(error, DownloadError::NotFound); + assert!(matches!(error, DownloadError::Unauthorized(_))); } #[test] @@ -930,206 +719,13 @@ mod tests { ) .unwrap_err(); - assert_eq!(error, DownloadError::NotFound); - } - - #[test] - fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() { - let artifact = ArtifactId::from("shared-artifact"); - let tenant_a = TenantId::from("tenant-a"); - let project_a = ProjectId::from("project-a"); - let tenant_b = TenantId::from("tenant-b"); - let project_b = ProjectId::from("project-b"); - let project_c = ProjectId::from("project-c"); - let node_a = NodeId::from("node-a"); - let node_b = NodeId::from("node-b"); - let node_c = NodeId::from("node-c"); - let mut registry = ArtifactRegistry::default(); - for (tenant, project, process, node, content, size) in [ - (&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17), - (&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29), - (&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31), - ] { - registry.flush_metadata(ArtifactFlush { - id: artifact.clone(), - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::from(process), - producer_task: TaskInstanceId::from("producer"), - retaining_node: node.clone(), - digest: Digest::sha256(content), - size, - }); - } - - assert_eq!(registry.artifact_count(), 3); - assert_eq!( - registry - .metadata(&tenant_a, &project_a, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-a") - ); - assert_eq!( - registry - .metadata(&tenant_b, &project_b, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-b") - ); - assert_eq!( - registry - .metadata(&tenant_a, &project_c, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-c") - ); - registry.flush_metadata(ArtifactFlush { - id: artifact.clone(), - tenant: tenant_a.clone(), - project: project_a.clone(), - process: ProcessId::from("process-a"), - producer_task: TaskInstanceId::from("producer-repeat"), - retaining_node: node_a.clone(), - digest: Digest::sha256("bytes-a-repeat"), - size: 18, - }); - assert_eq!(registry.artifact_count(), 3); - assert_eq!( - registry - .metadata(&tenant_a, &project_a, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-a-repeat") - ); - assert_eq!( - registry - .metadata(&tenant_b, &project_b, &artifact) - .unwrap() - .digest, - Digest::sha256("bytes-b") - ); - - registry - .sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a") - .unwrap(); - registry.garbage_collect_node(&tenant_a, &project_a, &node_a); - let context_a = AuthContext { - tenant: tenant_a.clone(), - project: project_a.clone(), - actor: Actor::User(UserId::from("user")), - }; - let context_b = AuthContext { - tenant: tenant_b.clone(), - project: project_b.clone(), - actor: Actor::User(UserId::from("user")), - }; - let policy = DownloadPolicy { max_bytes: 100 }; - assert_eq!( - registry - .download_action(&context_a, &artifact, &policy) - .unwrap() - .source, - StorageLocation::ExplicitStore("store://tenant-a".to_owned()) - ); - assert_eq!( - registry - .download_action(&context_b, &artifact, &policy) - .unwrap() - .source, - StorageLocation::RetainedNode(node_b) - ); - - let first_a = registry - .create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60) - .unwrap(); - let first_b = registry - .create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60) - .unwrap(); - assert_ne!( - first_a.scoped_token_digest, first_b.scoped_token_digest, - "the full artifact scope must contribute to download tokens" - ); - for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT { - registry - .create_download_link( - &context_a, - &artifact, - &policy, - &format!("tenant-a-{index}"), - 10, - 60, - ) - .unwrap(); - } - assert!(matches!( - registry.create_download_link( - &context_a, - &artifact, - &policy, - "tenant-a-over-limit", - 10, - 60, - ), - Err(DownloadError::Usage(_)) - )); - registry - .create_download_link( - &context_b, - &artifact, - &policy, - "tenant-b-still-independent", - 10, - 60, - ) - .unwrap(); - registry - .revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest) - .unwrap(); - - let limits = ResourceLimits { - limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]), - }; - let mut meter = ResourceMeter::default(); - registry - .open_download_stream( - DownloadStreamRequest { - context: &context_b, - artifact: &artifact, - policy: &policy, - presented_token_digest: &first_b.scoped_token_digest, - now_epoch_seconds: 11, - limits: &limits, - }, - &mut meter, - ) - .expect("revoking tenant A's link must not revoke tenant B's link"); - assert_eq!( - registry - .open_download_stream( - DownloadStreamRequest { - context: &context_b, - artifact: &artifact, - policy: &policy, - presented_token_digest: &first_a.scoped_token_digest, - now_epoch_seconds: 11, - limits: &limits, - }, - &mut meter, - ) - .unwrap_err(), - DownloadError::InvalidToken - ); + assert!(matches!(error, DownloadError::Unauthorized(_))); } #[test] fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() { let mut registry = registry_with_artifact(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); let context = AuthContext { tenant: TenantId::from("tenant"), project: ProjectId::from("project"), @@ -1406,200 +1002,51 @@ mod tests { } #[test] - fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() { + fn artifact_metadata_is_bounded_without_evicting_pins() { let mut registry = ArtifactRegistry::default(); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - registry.flush_metadata(ArtifactFlush { - id: ArtifactId::from("artifact-0"), - tenant: TenantId::from("other-tenant"), - project: ProjectId::from("other-project"), - process: ProcessId::from("process"), - producer_task: TaskInstanceId::from("other-task"), - retaining_node: NodeId::from("other-node"), - digest: Digest::sha256("other-content"), - size: 1, - }); - for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { + let mut pinned = BTreeSet::new(); + for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS { let id = ArtifactId::new(format!("artifact-{index}")); - registry.flush_metadata(ArtifactFlush { - id, - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::new(if index == 3 { - "active-process-3".to_owned() - } else { - format!("completed-process-{index}") - }), - producer_task: TaskInstanceId::new(format!("task-{index}")), - retaining_node: NodeId::from("node"), - digest: Digest::sha256(format!("content-{index}")), - size: 1, - }); + pinned.insert(id.clone()); + registry + .flush_metadata_bounded( + ArtifactFlush { + id, + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), + producer_task: TaskInstanceId::new(format!("task-{index}")), + retaining_node: NodeId::from("node"), + digest: Digest::sha256(format!("content-{index}")), + size: 1, + }, + &pinned, + ) + .unwrap(); } - assert_eq!( - registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROJECT + 1 - ); - let pinned = BTreeSet::from([ArtifactScopeKey::new( - tenant.clone(), - project.clone(), - ArtifactId::from("artifact-0"), - )]); - registry - .sync_to_explicit_store( - &tenant, - &project, - &ArtifactId::from("artifact-1"), - "store://retained-export", - ) - .unwrap(); - let context = AuthContext { - tenant: tenant.clone(), - project: project.clone(), - actor: Actor::User(UserId::from("user")), - }; - registry - .create_download_link( - &context, - &ArtifactId::from("artifact-2"), - &DownloadPolicy { max_bytes: 1 }, - "active-download", - 10, - 60, - ) - .unwrap(); - let protected_processes = BTreeSet::from([ - ProcessId::from("active-process-3"), - ProcessId::from("active-process"), - ]); + assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); let next = ArtifactFlush { id: ArtifactId::from("artifact-next"), - tenant: tenant.clone(), - project: project.clone(), - process: ProcessId::from("active-process"), + tenant: TenantId::from("tenant"), + project: ProjectId::from("project"), + process: ProcessId::from("process"), producer_task: TaskInstanceId::from("task-next"), retaining_node: NodeId::from("node"), digest: Digest::sha256("next"), size: 1, }; - registry - .flush_metadata_with_protected_processes(next, &pinned, &protected_processes) - .unwrap(); - assert_eq!( - registry.artifact_count(), - MAX_ARTIFACT_METADATA_PER_PROJECT + 1 - ); - for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] { - assert!( - registry - .metadata(&tenant, &project, &ArtifactId::from(id)) - .is_some(), - "{id} was evicted despite being pinned, exported, downloaded, or live" - ); - } - assert!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-4")) - .is_none(), - "the oldest completed unprotected metadata should be evicted" - ); assert!(registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-next")) - .is_some()); - assert_eq!( - registry - .metadata( - &TenantId::from("other-tenant"), - &ProjectId::from("other-project"), - &ArtifactId::from("artifact-0"), - ) - .unwrap() - .digest, - Digest::sha256("other-content"), - "eviction and pins in one scope must not affect an equal ID in another scope" - ); - } - - #[test] - fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() { - let mut registry = ArtifactRegistry::default(); - let tenant = TenantId::from("tenant"); - let project = ProjectId::from("project"); - let active_process = ProcessId::from("active-process"); - for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT { - registry.flush_metadata(ArtifactFlush { - id: ArtifactId::new(format!("artifact-{index}")), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::new(format!("task-{index}")), - retaining_node: NodeId::from("node"), - digest: Digest::sha256(format!("content-{index}")), - size: 1, - }); - } - - let replacement = ArtifactFlush { - id: ArtifactId::from("artifact-0"), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::from("replacement-task"), - retaining_node: NodeId::from("node"), - digest: Digest::sha256("replacement"), - size: 2, - }; - registry - .flush_metadata_with_protected_processes( - replacement, - &BTreeSet::new(), - &BTreeSet::from([active_process.clone()]), - ) - .expect("replacement must remain possible at the metadata bound"); - assert_eq!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) - .unwrap() - .digest, - Digest::sha256("replacement") - ); - - let epoch_before_rejection = registry.next_epoch; - let result = registry.flush_metadata_with_protected_processes( - ArtifactFlush { - id: ArtifactId::from("artifact-rejected"), - tenant: tenant.clone(), - project: project.clone(), - process: active_process.clone(), - producer_task: TaskInstanceId::from("rejected-task"), - retaining_node: NodeId::from("node"), - digest: Digest::sha256("rejected"), - size: 3, - }, - &BTreeSet::new(), - &BTreeSet::from([active_process]), - ); - - assert!(result + .flush_metadata_bounded(next.clone(), &pinned) .unwrap_err() - .contains("exhausted by active or retained artifacts")); - assert_eq!(registry.next_epoch, epoch_before_rejection); - assert_eq!( - registry.metadata_for_project(&tenant, &project).count(), - MAX_ARTIFACT_METADATA_PER_PROJECT - ); + .contains("pinned")); + + pinned.remove(&ArtifactId::from("artifact-0")); + registry.flush_metadata_bounded(next, &pinned).unwrap(); + assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS); + assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none()); assert!(registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-rejected")) - .is_none()); - assert_eq!( - registry - .metadata(&tenant, &project, &ArtifactId::from("artifact-0")) - .unwrap() - .digest, - Digest::sha256("replacement"), - "rejection must not modify existing metadata" - ); + .metadata(&ArtifactId::from("artifact-next")) + .is_some()); } #[test] @@ -1688,11 +1135,7 @@ mod tests { registry .stream_download_chunk(&mut stream, &limits, &mut meter, 16) .unwrap(); - registry.garbage_collect_node( - &TenantId::from("tenant"), - &ProjectId::from("project"), - &NodeId::from("node"), - ); + registry.garbage_collect_node(&NodeId::from("node")); assert_eq!( registry diff --git a/crates/clusterflux-core/src/auth.rs b/crates/clusterflux-core/src/auth.rs index 51686a2..61e99fb 100644 --- a/crates/clusterflux-core/src/auth.rs +++ b/crates/clusterflux-core/src/auth.rs @@ -364,13 +364,7 @@ pub fn agent_workflow_request_scope_from_payload( .get("task_instance") .and_then(Value::as_str) .ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?; - ( - process, - Some( - TaskInstanceId::try_new(task) - .map_err(|error| format!("malformed launch_task task instance: {error}"))?, - ), - ) + (process, Some(TaskInstanceId::from(task))) } _ => { return Err(format!( @@ -379,13 +373,10 @@ pub fn agent_workflow_request_scope_from_payload( } }; AgentWorkflowRequestScope::new( - TenantId::try_new(tenant) - .map_err(|error| format!("malformed agent workflow tenant: {error}"))?, - ProjectId::try_new(project) - .map_err(|error| format!("malformed agent workflow project: {error}"))?, + TenantId::from(tenant), + ProjectId::from(project), request_kind, - ProcessId::try_new(process) - .map_err(|error| format!("malformed agent workflow process: {error}"))?, + ProcessId::from(process), task, ) } diff --git a/crates/clusterflux-core/src/execution.rs b/crates/clusterflux-core/src/execution.rs index 16348a0..7b9d9ac 100644 --- a/crates/clusterflux-core/src/execution.rs +++ b/crates/clusterflux-core/src/execution.rs @@ -300,27 +300,6 @@ impl TaskBoundaryValue { _ => Vec::new(), } } - - fn rebind_source_snapshot(&mut self, replacement: &Digest) -> usize { - match self { - Self::SourceSnapshot(snapshot) => { - *snapshot = replacement.clone(); - 1 - } - Self::Structured(structured) => structured - .handles - .iter_mut() - .map(|handle| match handle { - TaskBoundaryHandle::SourceSnapshot(snapshot) => { - *snapshot = replacement.clone(); - 1 - } - _ => 0, - }) - .sum(), - _ => 0, - } - } } pub const WASM_TASK_ABI_VERSION: u32 = 1; @@ -867,23 +846,6 @@ impl TaskSpec { } Ok(()) } - - pub fn rebind_source_snapshot(&mut self, replacement: Digest) -> Result<(), String> { - self.validate_boundary_authority()?; - if self.source_snapshot.is_none() { - return Err("task has no SourceSnapshot argument handle to rebind".to_owned()); - } - let rebound = self - .args - .iter_mut() - .map(|argument| argument.rebind_source_snapshot(&replacement)) - .sum::(); - if rebound == 0 { - return Err("task source dependency has no canonical argument handle".to_owned()); - } - self.source_snapshot = Some(replacement); - self.validate_boundary_authority() - } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1082,11 +1044,6 @@ mod tests { bundle_digest: Some(Digest::sha256("bundle")), }; spec.validate_boundary_authority().unwrap(); - let replacement_source = Digest::sha256("replacement-source"); - spec.rebind_source_snapshot(replacement_source.clone()) - .unwrap(); - assert_eq!(spec.source_snapshot, Some(replacement_source.clone())); - assert_eq!(spec.args[0].source_snapshots(), vec![replacement_source]); spec.required_artifacts = vec![ArtifactId::from("forged.bin")]; assert!(spec .validate_boundary_authority() diff --git a/crates/clusterflux-core/src/ids.rs b/crates/clusterflux-core/src/ids.rs index 2e2744d..00b2d51 100644 --- a/crates/clusterflux-core/src/ids.rs +++ b/crates/clusterflux-core/src/ids.rs @@ -1,102 +1,18 @@ -#[cfg(not(target_arch = "wasm32"))] -use serde::{de::Error as _, Deserializer}; use serde::{Deserialize, Serialize}; -pub const MAX_EXTERNAL_ID_BYTES: usize = 255; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct OpaqueTokenError { - reason: String, -} - -impl std::fmt::Display for OpaqueTokenError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.reason) - } -} - -impl std::error::Error for OpaqueTokenError {} - -pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> { - let reason = if value.trim().is_empty() { - Some("value must not be empty or whitespace-only".to_owned()) - } else if value.len() > max_bytes { - Some(format!("value exceeds the {max_bytes}-byte limit")) - } else if value.chars().any(char::is_control) { - Some("control characters are forbidden".to_owned()) - } else { - None - }; - reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason })) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IdParseError { - id_type: &'static str, - reason: &'static str, -} - -impl IdParseError { - fn new(id_type: &'static str, reason: &'static str) -> Self { - Self { id_type, reason } - } -} - -impl std::fmt::Display for IdParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} is invalid: {}", self.id_type, self.reason) - } -} - -impl std::error::Error for IdParseError {} - -fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> { - if value.is_empty() || value.trim().is_empty() { - return Err(IdParseError::new( - id_type, - "value must not be empty or whitespace-only", - )); - } - if value.len() > MAX_EXTERNAL_ID_BYTES { - return Err(IdParseError::new( - id_type, - "value exceeds the 255-byte limit", - )); - } - if value.chars().any(char::is_control) { - return Err(IdParseError::new( - id_type, - "control characters are forbidden", - )); - } - if !value.bytes().all(|byte| { - byte.is_ascii_alphanumeric() - || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+') - }) { - return Err(IdParseError::new( - id_type, - "only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed", - )); - } - Ok(()) -} - macro_rules! id_type { ($name:ident) => { - #[cfg_attr(target_arch = "wasm32", derive(Deserialize))] - #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct $name(String); impl $name { - pub fn try_new(value: impl Into) -> Result { - let value = value.into(); - validate_id(&value, stringify!($name))?; - Ok(Self(value)) - } - - /// Constructs an identifier from a trusted, internally generated value. pub fn new(value: impl Into) -> Self { - Self::try_new(value).unwrap_or_else(|error| panic!("{error}")) + let value = value.into(); + assert!( + !value.trim().is_empty(), + concat!(stringify!($name), " cannot be empty") + ); + Self(value) } pub fn as_str(&self) -> &str { @@ -110,17 +26,6 @@ macro_rules! id_type { } } - #[cfg(not(target_arch = "wasm32"))] - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::try_new(value).map_err(D::Error::custom) - } - } - impl std::fmt::Display for $name { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.0) @@ -133,79 +38,8 @@ id_type!(AgentId); id_type!(ArtifactId); id_type!(NodeId); id_type!(ProcessId); -id_type!(LaunchAttemptId); id_type!(ProjectId); id_type!(TaskDefinitionId); id_type!(TaskInstanceId); id_type!(TenantId); id_type!(UserId); - -pub type DebugSessionId = ProcessId; -pub type RequestId = LaunchAttemptId; - -#[cfg(test)] -mod tests { - use super::*; - - fn assert_hostile_values(parse: impl Fn(String) -> Result) { - for value in [ - String::new(), - " ".to_owned(), - "bad\0id".to_owned(), - "bad id".to_owned(), - "x".repeat(MAX_EXTERNAL_ID_BYTES + 1), - ] { - assert!(parse(value).is_err()); - } - } - - #[test] - fn every_external_identifier_type_rejects_hostile_values() { - assert_hostile_values(AgentId::try_new); - assert_hostile_values(ArtifactId::try_new); - assert_hostile_values(DebugSessionId::try_new); - assert_hostile_values(NodeId::try_new); - assert_hostile_values(ProcessId::try_new); - assert_hostile_values(LaunchAttemptId::try_new); - assert_hostile_values(ProjectId::try_new); - assert_hostile_values(RequestId::try_new); - assert_hostile_values(TaskDefinitionId::try_new); - assert_hostile_values(TaskInstanceId::try_new); - assert_hostile_values(TenantId::try_new); - assert_hostile_values(UserId::try_new); - } - - #[test] - fn every_identifier_type_validates_during_deserialization() { - macro_rules! assert_deserialization { - ($id:ty) => { - assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok()); - let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err(); - assert!( - error.to_string().contains("is invalid"), - "{} produced an unexpected error: {error}", - stringify!($id) - ); - }; - } - - assert_deserialization!(AgentId); - assert_deserialization!(ArtifactId); - assert_deserialization!(NodeId); - assert_deserialization!(ProcessId); - assert_deserialization!(LaunchAttemptId); - assert_deserialization!(ProjectId); - assert_deserialization!(TaskDefinitionId); - assert_deserialization!(TaskInstanceId); - assert_deserialization!(TenantId); - assert_deserialization!(UserId); - } - - #[test] - fn opaque_tokens_are_bounded_without_using_identifier_syntax() { - validate_opaque_token("opaque secret/+==", 64).unwrap(); - assert!(validate_opaque_token("", 64).is_err()); - assert!(validate_opaque_token("bad\0token", 64).is_err()); - assert!(validate_opaque_token(&"x".repeat(65), 64).is_err()); - } -} diff --git a/crates/clusterflux-core/src/lib.rs b/crates/clusterflux-core/src/lib.rs index c9d5b77..38b3e15 100644 --- a/crates/clusterflux-core/src/lib.rs +++ b/crates/clusterflux-core/src/lib.rs @@ -1,4 +1,3 @@ -mod api_error; pub mod artifact; pub mod auth; pub mod bundle; @@ -19,11 +18,10 @@ pub mod transport; pub mod vfs; pub mod wire; -pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode}; pub use artifact::{ ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry, - ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, - DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation, + ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy, + DownloadStreamRequest, RetentionPolicy, StorageLocation, }; pub use auth::{ admin_request_proof, admin_request_proof_from_token_digest, @@ -68,9 +66,8 @@ pub use execution::{ WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION, }; pub use ids::{ - validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId, - NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId, - TenantId, UserId, MAX_EXTERNAL_ID_BYTES, + AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId, + UserId, }; pub use limits::{ LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits, diff --git a/crates/clusterflux-core/src/vfs.rs b/crates/clusterflux-core/src/vfs.rs index 9f1b739..9e86e11 100644 --- a/crates/clusterflux-core/src/vfs.rs +++ b/crates/clusterflux-core/src/vfs.rs @@ -1,48 +1,18 @@ use std::collections::BTreeMap; -use serde::{de::Error as _, Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{Digest, NodeId, TaskInstanceId}; -pub const MAX_VFS_PATH_BYTES: usize = 4096; - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct VfsPath(String); impl VfsPath { pub fn new(path: impl Into) -> Result { let path = path.into(); if !path.starts_with("/vfs/") { - return Err(VfsError::invalid(path, "path must start with /vfs/")); - } - if path.len() > MAX_VFS_PATH_BYTES { - return Err(VfsError::invalid( - path, - "path exceeds the 4096-byte protocol limit", - )); - } - if path.chars().any(char::is_control) { - return Err(VfsError::invalid(path, "control characters are forbidden")); - } - if path.contains('\\') { - return Err(VfsError::invalid(path, "backslashes are forbidden")); - } - let relative = &path["/vfs/".len()..]; - if relative.is_empty() { - return Err(VfsError::invalid( - path, - "path after /vfs/ must not be empty", - )); - } - if relative - .split('/') - .any(|component| component.is_empty() || matches!(component, "." | "..")) - { - return Err(VfsError::invalid( - path, - "empty, '.', and '..' path components are forbidden", - )); + return Err(VfsError::InvalidPath(path)); } Ok(Self(path)) } @@ -52,16 +22,6 @@ impl VfsPath { } } -impl<'de> Deserialize<'de> for VfsPath { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let path = String::deserialize(deserializer)?; - Self::new(path).map_err(D::Error::custom) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VfsObject { pub path: VfsPath, @@ -103,18 +63,12 @@ pub enum ReuseDecision { #[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum VfsError { - #[error("invalid VFS path {path:?}: {reason}")] - InvalidPath { path: String, reason: &'static str }, + #[error("VFS path must start with /vfs/: {0}")] + InvalidPath(String), #[error("path is not visible in the published VFS manifest: {0}")] NotVisible(String), } -impl VfsError { - fn invalid(path: String, reason: &'static str) -> Self { - Self::InvalidPath { path, reason } - } -} - #[derive(Clone, Debug)] pub struct VfsOverlay { task: TaskInstanceId, @@ -284,39 +238,4 @@ mod tests { assert_eq!(overlay.pending_len(), 0); } - - #[test] - fn vfs_path_rejects_the_complete_hostile_protocol_matrix() { - for invalid in [ - "vfs/artifacts/app".to_owned(), - "/vfs/".to_owned(), - "/vfs/artifacts//app".to_owned(), - "/vfs/artifacts/app/".to_owned(), - "/vfs/artifacts/./app".to_owned(), - "/vfs/artifacts/../app".to_owned(), - "/vfs/artifacts\\app".to_owned(), - "/vfs/artifacts/bad\0app".to_owned(), - format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)), - ] { - assert!( - VfsPath::new(&invalid).is_err(), - "hostile VFS path unexpectedly passed: {invalid:?}" - ); - assert!( - serde_json::from_value::(serde_json::json!(invalid)).is_err(), - "hostile VFS path unexpectedly deserialized" - ); - } - } - - #[test] - fn vfs_path_accepts_the_4096_byte_boundary() { - let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len())); - assert_eq!(path.len(), MAX_VFS_PATH_BYTES); - assert_eq!(VfsPath::new(&path).unwrap().as_str(), path); - - let overlong = format!("{path}x"); - assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1); - assert!(VfsPath::new(overlong).is_err()); - } } diff --git a/crates/clusterflux-dap/src/adapter.rs b/crates/clusterflux-dap/src/adapter.rs index 4e144ab..e2857e0 100644 --- a/crates/clusterflux-dap/src/adapter.rs +++ b/crates/clusterflux-dap/src/adapter.rs @@ -1,8 +1,4 @@ -use std::collections::BTreeMap; use std::io::{self, BufReader}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{mpsc, Arc}; -use std::thread; use anyhow::Result; use clusterflux_core::DebugRuntimeState; @@ -19,264 +15,21 @@ use crate::demo_backend::{ is_explicit_demo_backend, start_simulated_backend, LINUX_THREAD, MAIN_THREAD, }; use crate::runtime_client::{ - attach_services_runtime, create_debug_epoch, debug_epoch_runtime_record, - observe_services_runtime, relaunch_services_main_runtime, restart_task, resume_debug_epoch, - run_live_services_runtime, run_local_services_runtime, set_services_debug_breakpoints, - wait_for_debug_epoch_frozen, wait_for_debug_epoch_resumed, LocalRuntimeSession, + attach_services_runtime, create_debug_epoch, relaunch_services_main_runtime, restart_task, + resume_debug_epoch, run_live_services_runtime, run_local_services_runtime, + wait_for_debug_epoch_frozen, wait_for_debug_epoch_resumed, wait_for_services_runtime_outcome, RuntimeContinuationOutcome, }; use crate::variables::variables_response; -use crate::virtual_model::{AdapterState, DapSessionMode, RuntimeBackend, VirtualThread}; - -enum AdapterEvent { - Request(Value), - InputClosed, - InputFailed(String), - LaunchFinished { - generation: u64, - result: std::result::Result, - }, - ObservationUpdate { - generation: u64, - outcome: RuntimeContinuationOutcome, - }, - ObservationFinished { - generation: u64, - }, - PauseFinished { - generation: u64, - result: std::result::Result, - }, - ResumeFinished { - generation: u64, - request: Value, - thread_id: i64, - result: std::result::Result, - }, - BreakpointsUpdated { - generation: u64, - revision: u64, - result: std::result::Result<(), String>, - }, -} - -struct LaunchCompletion { - state: AdapterState, - local_runtime_session: Option, - breakpoint_revision: u64, - observation_diagnostics: Vec, -} +use crate::virtual_model::{AdapterState, DapSessionMode, RuntimeBackend}; pub(crate) fn run_adapter() -> Result<()> { let mut writer = DapWriter::new(); + let mut reader = BufReader::new(io::stdin()); let mut state = AdapterState::default(); let mut _local_runtime_session = None; - let mut runtime_started = false; - let mut runtime_starting = false; - let mut runtime_generation = 0_u64; - let mut observation_cancel = None; - let (event_tx, event_rx) = mpsc::channel(); - spawn_input_reader(event_tx.clone()); - while let Ok(event) = event_rx.recv() { - let request = match event { - AdapterEvent::Request(request) => request, - AdapterEvent::InputClosed => { - cancel_observation(&mut observation_cancel); - break; - } - AdapterEvent::InputFailed(message) => { - cancel_observation(&mut observation_cancel); - return Err(anyhow::anyhow!(message)); - } - AdapterEvent::LaunchFinished { generation, result } => { - if generation != runtime_generation { - continue; - } - runtime_starting = false; - match result { - Ok(mut completion) => { - let requested_breakpoints = state.breakpoints.clone(); - let requested_revision = state.breakpoint_revision; - let installed_revision = completion.breakpoint_revision; - completion.state.breakpoints = requested_breakpoints; - completion.state.breakpoint_revision = requested_revision; - completion.state.breakpoints_installed = - installed_revision == requested_revision; - let previous_threads = state.threads.clone(); - state = completion.state; - emit_thread_lifecycle(&mut writer, &previous_threads, &state.threads)?; - if let Some(session) = completion.local_runtime_session { - _local_runtime_session = Some(session); - } - for diagnostic in completion.observation_diagnostics { - writer.output("stderr", format!("{diagnostic}\n"))?; - } - runtime_started = true; - if state.breakpoints_installed { - emit_verified_breakpoints(&mut writer, &state)?; - } else { - let breakpoint_state = state.clone(); - let breakpoint_tx = event_tx.clone(); - let revision = state.breakpoint_revision; - thread::spawn(move || { - let result = set_services_debug_breakpoints(&breakpoint_state) - .map_err(|error| { - format!("coordinator breakpoint update failed: {error:#}") - }); - let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated { - generation, - revision, - result, - }); - }); - } - writer.output( - "console", - if state.session_mode == DapSessionMode::Attach { - format!( - "Attached to Clusterflux virtual process {} with {:?} runtime\n", - state.process, state.runtime_backend - ) - } else { - format!( - "Clusterflux bundle refreshed for entry `{}`; virtual process {} is running with {:?} runtime\n", - state.entry, state.process, state.runtime_backend - ) - }, - )?; - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); - } - Err(message) => { - runtime_started = false; - writer.output("stderr", format!("{message}\n"))?; - writer.event("terminated", json!({ "restart": false }))?; - } - } - continue; - } - AdapterEvent::ObservationUpdate { - generation, - outcome, - } => { - if generation != runtime_generation { - continue; - } - emit_runtime_outcome(&mut writer, &mut state, outcome)?; - continue; - } - AdapterEvent::ObservationFinished { generation } => { - if generation == runtime_generation { - observation_cancel = None; - } - continue; - } - AdapterEvent::PauseFinished { generation, result } => { - if generation != runtime_generation { - continue; - } - match result { - Ok(paused_state) => { - state = paused_state; - let stopped_thread = default_thread_id(&state); - writer.event( - "stopped", - json!({ - "reason": "pause", - "description": debug_epoch_stop_description(&state, "Clusterflux Debug Epoch pause"), - "threadId": stopped_thread, - "allThreadsStopped": state.debug_all_threads_stopped, - }), - )?; - } - Err(message) => { - writer.output("stderr", format!("{message}\n"))?; - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); - } - } - continue; - } - AdapterEvent::ResumeFinished { - generation, - request, - thread_id, - result, - } => { - if generation != runtime_generation { - continue; - } - match result { - Ok(resumed_state) => { - state = resumed_state; - for thread in state.threads.values_mut() { - if thread.state == DebugRuntimeState::Frozen { - thread.state = DebugRuntimeState::Running; - } - } - writer.response(&request, true, json!({ "allThreadsContinued": true }))?; - writer.event( - "continued", - json!({ - "threadId": thread_id, - "allThreadsContinued": true, - }), - )?; - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); - } - Err(message) => { - writer.output("stderr", format!("{message}\n"))?; - writer.error_response(&request, message)?; - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); - } - } - continue; - } - AdapterEvent::BreakpointsUpdated { - generation, - revision, - result, - } => { - if breakpoint_update_is_current( - generation, - runtime_generation, - revision, - state.breakpoint_revision, - ) { - match result { - Ok(()) => { - state.breakpoints_installed = true; - emit_verified_breakpoints(&mut writer, &state)?; - } - Err(message) => { - state.breakpoints_installed = false; - emit_unverified_breakpoints(&mut writer, &state, &message)?; - writer.output("stderr", format!("{message}\n"))?; - } - } - } - continue; - } - }; + while let Some(request) = read_message(&mut reader)? { let command = request .get("command") .and_then(Value::as_str) @@ -325,26 +78,12 @@ pub(crate) fn run_adapter() -> Result<()> { writer.error_response(&request, err.to_string())?; continue; } - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - runtime_started = false; - runtime_starting = false; - _local_runtime_session = None; state.restart_existing = args .get("restartExisting") .and_then(Value::as_bool) .unwrap_or(false); if let Some(process_id) = args.get("processId").and_then(Value::as_str) { - match clusterflux_core::ProcessId::try_new(process_id.to_owned()) { - Ok(process) => state.process = process, - Err(error) => { - writer.error_response( - &request, - format!("invalid DAP processId: {error}"), - )?; - continue; - } - } + state.process = clusterflux_core::ProcessId::new(process_id); } if command == "attach" { state.session_mode = DapSessionMode::Attach; @@ -371,173 +110,134 @@ pub(crate) fn run_adapter() -> Result<()> { .collect::>() }) .unwrap_or_default(); - let resolved = resolve_breakpoints_for_source( + let response = resolve_breakpoints_for_source( &mut state, requested_source_path, requested_lines, - ); - state.breakpoint_revision = state.breakpoint_revision.saturating_add(1); - let breakpoint_revision = state.breakpoint_revision; - let coordinator_backed = matches!( - state.runtime_backend, - RuntimeBackend::LocalServices | RuntimeBackend::LiveServices - ); - state.breakpoints_installed = !coordinator_backed; - let response = resolved - .iter() - .map(|breakpoint| { - let mut dap = breakpoint.to_dap(); - if coordinator_backed && breakpoint.verified { - dap["verified"] = Value::Bool(false); - dap["message"] = Value::String( - "Pending coordinator breakpoint installation".to_owned(), - ); - } - dap - }) - .collect::>(); + ) + .iter() + .map(|breakpoint| breakpoint.to_dap()) + .collect::>(); writer.response(&request, true, json!({ "breakpoints": response }))?; - if runtime_started - && matches!( - state.runtime_backend, - RuntimeBackend::LocalServices | RuntimeBackend::LiveServices - ) - { - let breakpoint_state = state.clone(); - let breakpoint_tx = event_tx.clone(); - let generation = runtime_generation; - thread::spawn(move || { - let result = - set_services_debug_breakpoints(&breakpoint_state).map_err(|error| { - format!("coordinator breakpoint update failed: {error:#}") - }); - let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated { - generation, - revision: breakpoint_revision, - result, - }); - }); - } } "setExceptionBreakpoints" => { writer.response(&request, true, json!({ "breakpoints": [] }))?; } "configurationDone" => { - if runtime_starting || runtime_started { - writer.error_response( - &request, - "the Clusterflux runtime has already been started for this DAP session", - )?; - continue; + if state.session_mode == DapSessionMode::Attach { + match state.runtime_backend { + RuntimeBackend::Simulated => { + state.command_status = + format!("attached to existing virtual process {}", state.process); + } + RuntimeBackend::LocalServices | RuntimeBackend::LiveServices => { + match attach_services_runtime(&state) { + Ok(record) => state.apply_attach_record(record), + Err(err) => { + writer.error_response( + &request, + format!("services runtime attach failed: {err:#}"), + )?; + continue; + } + } + } + } + } else { + match state.runtime_backend { + RuntimeBackend::LocalServices => match run_local_services_runtime(&state) { + Ok((record, session)) => { + state.apply_runtime_record(record); + _local_runtime_session = Some(session); + } + Err(err) => { + writer.error_response( + &request, + format!("local services runtime launch failed: {err:#}"), + )?; + continue; + } + }, + RuntimeBackend::LiveServices => match run_live_services_runtime(&state) { + Ok(record) => { + state.apply_runtime_record(record); + } + Err(err) => { + writer.error_response( + &request, + format!("live services runtime launch failed: {err:#}"), + )?; + continue; + } + }, + RuntimeBackend::Simulated => { + start_simulated_backend(&mut state); + } + } } writer.response(&request, true, json!({}))?; - if state.runtime_backend == RuntimeBackend::Simulated { - if state.session_mode == DapSessionMode::Attach { - state.command_status = - format!("attached to existing virtual process {}", state.process); - } else { - start_simulated_backend(&mut state); + let session_message = if state.session_mode == DapSessionMode::Attach { + format!( + "Attached to Clusterflux virtual process {} with {:?} runtime\n", + state.process, state.runtime_backend + ) + } else { + format!( + "Clusterflux bundle refreshed for entry `{}`; starting virtual process {} with {:?} runtime\n", + state.entry, state.process, state.runtime_backend + ) + }; + writer.output("console", session_message)?; + if !state.breakpoints.is_empty() { + let stopped_thread = stopped_thread_for_breakpoint(&state); + if matches!( + state.runtime_backend, + RuntimeBackend::LocalServices | RuntimeBackend::LiveServices + ) && state.coordinator_debug_epoch.is_some() + { + position_confirmed_breakpoint_stop(&mut state, stopped_thread); + writer.event( + "stopped", + json!({ + "reason": "breakpoint", + "description": debug_epoch_stop_description(&state, "Clusterflux Debug Epoch all-stop confirmed by every active participant"), + "threadId": stopped_thread, + "allThreadsStopped": state.debug_all_threads_stopped, + }), + )?; + continue; } - writer.output( - "console", - format!( - "Clusterflux explicit demo backend started virtual process {}\n", - state.process - ), - )?; - if !state.breakpoints.is_empty() { - let stopped_thread = stopped_thread_for_breakpoint(&state); - match freeze_all(&mut state, stopped_thread, None) { - Ok(()) => writer.event( + match freeze_all(&mut state, stopped_thread, None) { + Ok(()) => { + if let Err(err) = record_coordinator_debug_epoch( + &mut state, + stopped_thread, + "breakpoint", + ) { + let message = format!("coordinator debug epoch failed: {err:#}"); + writer.output("stderr", format!("{message}\n"))?; + writer.error_response(&request, message)?; + continue; + } + writer.event( "stopped", json!({ "reason": "breakpoint", - "description": "Clusterflux explicit demo all-stop", + "description": debug_epoch_stop_description(&state, "Clusterflux Debug Epoch all-stop"), "threadId": stopped_thread, - "allThreadsStopped": true, + "allThreadsStopped": state.debug_all_threads_stopped, }), - )?, - Err(failure) => { - writer.output("stderr", format!("{}\n", failure.message()))? - } + )?; + } + Err(failure) => { + let message = failure.message(); + writer.output("stderr", format!("{message}\n"))?; + writer.error_response(&request, message)?; } } - runtime_started = true; - continue; + } else { + writer.event("terminated", json!({}))?; } - - runtime_starting = true; - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - let generation = runtime_generation; - let launch_state = state.clone(); - let launch_tx = event_tx.clone(); - thread::spawn(move || { - let mut completed_state = launch_state; - let result = if completed_state.session_mode == DapSessionMode::Attach { - attach_services_runtime(&completed_state) - .and_then(|record| { - completed_state.apply_attach_record(record); - set_services_debug_breakpoints(&completed_state)?; - Ok(()) - }) - .map(|()| { - let breakpoint_revision = completed_state.breakpoint_revision; - LaunchCompletion { - state: completed_state, - local_runtime_session: None, - breakpoint_revision, - observation_diagnostics: Vec::new(), - } - }) - .map_err(|error| format!("services runtime attach failed: {error:#}")) - } else { - match completed_state.runtime_backend.clone() { - RuntimeBackend::LocalServices => { - run_local_services_runtime(&completed_state) - .map(|(record, session)| { - let observation_diagnostics = - runtime_observation_diagnostics(&record); - completed_state.apply_runtime_record(record); - let breakpoint_revision = - completed_state.breakpoint_revision; - LaunchCompletion { - state: completed_state, - local_runtime_session: Some(session), - breakpoint_revision, - observation_diagnostics, - } - }) - .map_err(|error| { - format!("local services runtime launch failed: {error:#}") - }) - } - RuntimeBackend::LiveServices => { - run_live_services_runtime(&completed_state) - .map(|record| { - let observation_diagnostics = - runtime_observation_diagnostics(&record); - completed_state.apply_runtime_record(record); - let breakpoint_revision = - completed_state.breakpoint_revision; - LaunchCompletion { - state: completed_state, - local_runtime_session: None, - breakpoint_revision, - observation_diagnostics, - } - }) - .map_err(|error| { - format!("live services runtime launch failed: {error:#}") - }) - } - RuntimeBackend::Simulated => { - unreachable!("the explicit demo backend is handled synchronously") - } - } - }; - let _ = launch_tx.send(AdapterEvent::LaunchFinished { generation, result }); - }); } "threads" => { let threads = state @@ -553,13 +253,6 @@ pub(crate) fn run_adapter() -> Result<()> { writer.response(&request, true, json!({ "threads": threads }))?; } "stackTrace" => { - if state.threads.is_empty() { - writer.error_response( - &request, - "runtime threads are not available yet; the asynchronous launch is still starting", - )?; - continue; - } let thread = request_thread(&request, &state); let source_path = crate::source::stack_source_path(&state); let stack_frames = if state.runtime_backend != RuntimeBackend::Simulated @@ -599,13 +292,6 @@ pub(crate) fn run_adapter() -> Result<()> { Err(err) => writer.error_response(&request, err.to_string())?, }, "scopes" => { - if state.threads.is_empty() { - writer.error_response( - &request, - "runtime scopes are not available yet; the asynchronous launch is still starting", - )?; - continue; - } let frame_id = request .get("arguments") .and_then(|value| value.get("frameId")) @@ -681,42 +367,6 @@ pub(crate) fn run_adapter() -> Result<()> { )?; } "pause" => { - if matches!( - state.runtime_backend, - RuntimeBackend::LocalServices | RuntimeBackend::LiveServices - ) { - if !runtime_started || state.threads.is_empty() { - writer.error_response( - &request, - "the Clusterflux runtime has not reported an active task to pause yet", - )?; - continue; - } - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - let generation = runtime_generation; - let mut pause_state = state.clone(); - let pause_tx = event_tx.clone(); - let stopped_thread = default_thread_id(&pause_state); - writer.response(&request, true, json!({}))?; - thread::spawn(move || { - let result = freeze_all(&mut pause_state, stopped_thread, None) - .map_err(|failure| failure.message()) - .and_then(|()| { - record_coordinator_debug_epoch( - &mut pause_state, - stopped_thread, - "pause", - ) - .map_err(|error| { - format!("coordinator debug epoch failed: {error:#}") - }) - }) - .map(|()| pause_state); - let _ = pause_tx.send(AdapterEvent::PauseFinished { generation, result }); - }); - continue; - } let stopped_thread = default_thread_id(&state); match freeze_all( &mut state, @@ -753,39 +403,8 @@ pub(crate) fn run_adapter() -> Result<()> { "continue" => { let thread_id = request_thread_id(&request).unwrap_or_else(|| default_thread_id(&state)); - if matches!( - state.runtime_backend, - RuntimeBackend::LocalServices | RuntimeBackend::LiveServices - ) { - if !runtime_started { - writer.error_response( - &request, - "the Clusterflux runtime is still starting", - )?; - continue; - } - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - let generation = runtime_generation; - let mut resume_state = state.clone(); - let resume_tx = event_tx.clone(); - let resume_request = request.clone(); - thread::spawn(move || { - let result = resume_coordinator_epoch(&mut resume_state) - .map(|()| resume_state) - .map_err(|error| { - format!("coordinator debug epoch resume failed: {error:#}") - }); - let _ = resume_tx.send(AdapterEvent::ResumeFinished { - generation, - request: resume_request, - thread_id, - result, - }); - }); - continue; - } let next_breakpoint = next_breakpoint_after(&state, thread_id); + let previous_debug_epoch = state.coordinator_debug_epoch.unwrap_or(state.epoch); if let Err(err) = resume_coordinator_epoch(&mut state) { let message = format!("coordinator debug epoch resume failed: {err:#}"); writer.output("stderr", format!("{message}\n"))?; @@ -805,6 +424,62 @@ pub(crate) fn run_adapter() -> Result<()> { "allThreadsContinued": true, }), )?; + if matches!( + state.runtime_backend, + RuntimeBackend::LocalServices | RuntimeBackend::LiveServices + ) { + match wait_for_services_runtime_outcome(&state, previous_debug_epoch) { + Ok(RuntimeContinuationOutcome::Breakpoint(record)) => { + state.apply_runtime_record(record); + let stopped_thread = stopped_thread_for_breakpoint(&state); + position_confirmed_breakpoint_stop(&mut state, stopped_thread); + writer.event( + "stopped", + json!({ + "reason": "breakpoint", + "description": debug_epoch_stop_description(&state, "Clusterflux Debug Epoch all-stop confirmed by every active participant"), + "threadId": stopped_thread, + "allThreadsStopped": state.debug_all_threads_stopped, + }), + )?; + } + Ok(RuntimeContinuationOutcome::Exception(record)) => { + state.apply_runtime_record(record); + let stopped_thread = stopped_thread_for_breakpoint(&state); + writer.output("stderr", format!("{}\n", state.command_status))?; + writer.event( + "stopped", + json!({ + "reason": "exception", + "description": "Task failed and is awaiting operator action", + "threadId": stopped_thread, + "allThreadsStopped": false, + }), + )?; + } + Ok(RuntimeContinuationOutcome::Terminal(record)) => { + state.apply_runtime_record(record); + if state.last_task_failed { + writer.output("stderr", format!("{}\n", state.command_status))?; + } + writer.event("terminated", json!({}))?; + } + Err(err) => { + let message = format!("continued runtime observation failed: {err:#}"); + writer.output("stderr", format!("{message}\n"))?; + writer.event( + "stopped", + json!({ + "reason": "exception", + "description": message, + "threadId": thread_id, + "allThreadsStopped": false, + }), + )?; + } + } + continue; + } if let Some((stopped_thread, line)) = next_breakpoint { match freeze_all_at_line(&mut state, stopped_thread, line, None) { Ok(()) => { @@ -899,23 +574,22 @@ pub(crate) fn run_adapter() -> Result<()> { match relaunch_services_main_runtime(&state) { Ok(record) => { state.apply_runtime_record(record); - state.epoch = 0; - state.coordinator_debug_epoch = None; - state.stopped_task = None; - state.stopped_probe_symbol = None; + let stopped_thread = stopped_thread_for_breakpoint(&state); + position_confirmed_breakpoint_stop(&mut state, stopped_thread); writer.response(&request, true, json!({}))?; writer.output( "console", "Rebuilt the current bundle and restarted the failed coordinator main from its entry boundary\n", )?; - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); + writer.event( + "stopped", + json!({ + "reason": "breakpoint", + "description": debug_epoch_stop_description(&state, "Restarted main from the rebuilt bundle; every active participant confirmed all-stop"), + "threadId": stopped_thread, + "allThreadsStopped": state.debug_all_threads_stopped, + }), + )?; } Err(err) => { let message = format!("coordinator main restart failed: {err:#}"); @@ -927,6 +601,8 @@ pub(crate) fn run_adapter() -> Result<()> { } match restart_task_through_coordinator(&mut state, thread_id) { Ok(message) => { + let previous_debug_epoch = + state.coordinator_debug_epoch.unwrap_or(state.epoch); if let Some(thread) = state.threads.get_mut(&thread_id) { thread.state = DebugRuntimeState::Running; thread.recent_output.push(message.clone()); @@ -935,14 +611,61 @@ pub(crate) fn run_adapter() -> Result<()> { state.command_status = message.clone(); writer.response(&request, true, json!({}))?; writer.output("console", format!("{message}\n"))?; - cancel_observation(&mut observation_cancel); - runtime_generation = runtime_generation.saturating_add(1); - spawn_runtime_observer( - &event_tx, - &state, - runtime_generation, - &mut observation_cancel, - ); + if matches!( + state.runtime_backend, + RuntimeBackend::LocalServices | RuntimeBackend::LiveServices + ) { + match wait_for_services_runtime_outcome( + &state, + previous_debug_epoch, + ) { + Ok(RuntimeContinuationOutcome::Breakpoint(record)) => { + state.apply_runtime_record(record); + let stopped_thread = stopped_thread_for_breakpoint(&state); + position_confirmed_breakpoint_stop( + &mut state, + stopped_thread, + ); + writer.event( + "stopped", + json!({ + "reason": "breakpoint", + "description": debug_epoch_stop_description(&state, "Restarted task reached a Wasm probe and every active participant confirmed all-stop"), + "threadId": stopped_thread, + "allThreadsStopped": state.debug_all_threads_stopped, + }), + )?; + } + Ok(RuntimeContinuationOutcome::Exception(record)) => { + state.apply_runtime_record(record); + let stopped_thread = stopped_thread_for_breakpoint(&state); + writer.event( + "stopped", + json!({ + "reason": "exception", + "description": "Replacement attempt failed and is awaiting operator action", + "threadId": stopped_thread, + "allThreadsStopped": false, + }), + )?; + } + Ok(RuntimeContinuationOutcome::Terminal(record)) => { + state.apply_runtime_record(record); + writer.event("terminated", json!({}))?; + } + Err(err) => { + writer.event( + "stopped", + json!({ + "reason": "exception", + "description": format!("restarted runtime observation failed: {err:#}"), + "threadId": thread_id, + "allThreadsStopped": false, + }), + )?; + } + } + } } Err(err) => { let message = format!("coordinator task restart failed: {err:#}"); @@ -983,7 +706,6 @@ pub(crate) fn run_adapter() -> Result<()> { )?; } "disconnect" | "terminate" => { - cancel_observation(&mut observation_cancel); writer.response(&request, true, json!({}))?; writer.event("terminated", json!({}))?; break; @@ -995,235 +717,11 @@ pub(crate) fn run_adapter() -> Result<()> { Ok(()) } -fn spawn_input_reader(event_tx: mpsc::Sender) { - thread::spawn(move || { - let mut reader = BufReader::new(io::stdin()); - loop { - match read_message(&mut reader) { - Ok(Some(request)) => { - if event_tx.send(AdapterEvent::Request(request)).is_err() { - return; - } - } - Ok(None) => { - let _ = event_tx.send(AdapterEvent::InputClosed); - return; - } - Err(error) => { - let _ = event_tx.send(AdapterEvent::InputFailed(format!( - "DAP input failed: {error:#}" - ))); - return; - } - } - } - }); -} - -fn cancel_observation(observation_cancel: &mut Option>) { - if let Some(cancelled) = observation_cancel.take() { - cancelled.store(true, Ordering::Release); - } -} - -fn spawn_runtime_observer( - event_tx: &mpsc::Sender, - state: &AdapterState, - generation: u64, - observation_cancel: &mut Option>, -) { - cancel_observation(observation_cancel); - if state.runtime_backend == RuntimeBackend::Simulated { - return; - } - let cancelled = Arc::new(AtomicBool::new(false)); - *observation_cancel = Some(cancelled.clone()); - let observer_state = state.clone(); - let observer_tx = event_tx.clone(); - let previous_debug_epoch = state.coordinator_debug_epoch.unwrap_or(state.epoch); - thread::spawn(move || { - let result = observe_services_runtime( - &observer_state, - previous_debug_epoch, - cancelled.as_ref(), - |outcome| { - observer_tx - .send(AdapterEvent::ObservationUpdate { - generation, - outcome, - }) - .is_ok() - }, - ); - if let Err(error) = result { - let _ = observer_tx.send(AdapterEvent::ObservationUpdate { - generation, - outcome: RuntimeContinuationOutcome::Diagnostic(format!( - "runtime observer stopped unexpectedly: {error:#}" - )), - }); - } - let _ = observer_tx.send(AdapterEvent::ObservationFinished { generation }); - }); -} - -pub(crate) fn breakpoint_update_is_current( - update_generation: u64, - runtime_generation: u64, - update_revision: u64, - current_revision: u64, -) -> bool { - update_generation == runtime_generation && update_revision == current_revision -} - -fn runtime_observation_diagnostics( - record: &crate::virtual_model::RuntimeLaunchRecord, -) -> Vec { - record - .node_report - .get("observation_diagnostics") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() -} - -fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> { - if !state.breakpoints_installed { - return Ok(()); - } - for (index, line) in state.breakpoints.iter().enumerate() { - writer.event( - "breakpoint", - json!({ - "reason": "changed", - "breakpoint": { - "id": index + 1, - "verified": true, - "line": line, - "message": "Installed by the Clusterflux coordinator", - } - }), - )?; - } - Ok(()) -} - -fn emit_unverified_breakpoints( - writer: &mut DapWriter, - state: &AdapterState, - coordinator_error: &str, -) -> Result<()> { - for (index, line) in state.breakpoints.iter().enumerate() { - writer.event( - "breakpoint", - json!({ - "reason": "changed", - "breakpoint": { - "id": index + 1, - "verified": false, - "line": line, - "message": format!("Coordinator breakpoint installation failed: {coordinator_error}"), - } - }), - )?; - } - Ok(()) -} - -fn emit_thread_lifecycle( - writer: &mut DapWriter, - previous: &BTreeMap, - current: &BTreeMap, -) -> Result<()> { - for (id, thread) in previous { - if current - .get(id) - .is_none_or(|current| current.task != thread.task) - { - writer.event("thread", json!({ "reason": "exited", "threadId": id }))?; - } - } - for (id, thread) in current { - if previous - .get(id) - .is_none_or(|previous| previous.task != thread.task) - { - writer.event("thread", json!({ "reason": "started", "threadId": id }))?; - } - } - Ok(()) -} - -fn apply_runtime_record_with_thread_events( - writer: &mut DapWriter, - state: &mut AdapterState, - record: crate::virtual_model::RuntimeLaunchRecord, -) -> Result<()> { - let previous = state.threads.clone(); - state.apply_runtime_record(record); - emit_thread_lifecycle(writer, &previous, &state.threads) -} - -fn emit_runtime_outcome( - writer: &mut DapWriter, - state: &mut AdapterState, - outcome: RuntimeContinuationOutcome, -) -> Result<()> { - match outcome { - RuntimeContinuationOutcome::Snapshot(record) => { - apply_runtime_record_with_thread_events(writer, state, record)?; - } - RuntimeContinuationOutcome::Diagnostic(message) => { - writer.output("stderr", format!("{message}\n"))?; - } - RuntimeContinuationOutcome::Breakpoint(record) => { - apply_runtime_record_with_thread_events(writer, state, record)?; - let stopped_thread = stopped_thread_for_breakpoint(state); - position_confirmed_breakpoint_stop(state, stopped_thread); - writer.event( - "stopped", - json!({ - "reason": "breakpoint", - "description": debug_epoch_stop_description(state, "Clusterflux Debug Epoch all-stop confirmed by every active participant"), - "threadId": stopped_thread, - "allThreadsStopped": state.debug_all_threads_stopped, - }), - )?; - } - RuntimeContinuationOutcome::Exception(record) => { - apply_runtime_record_with_thread_events(writer, state, record)?; - let stopped_thread = stopped_thread_for_breakpoint(state); - writer.output("stderr", format!("{}\n", state.command_status))?; - writer.event( - "stopped", - json!({ - "reason": "exception", - "description": "Task failed and is awaiting operator action", - "threadId": stopped_thread, - "allThreadsStopped": false, - }), - )?; - } - RuntimeContinuationOutcome::Terminal(record) => { - let exit_code = record.status_code.unwrap_or(1); - apply_runtime_record_with_thread_events(writer, state, record)?; - if state.last_task_failed { - writer.output("stderr", format!("{}\n", state.command_status))?; - } - writer.event("exited", json!({ "exitCode": exit_code }))?; - writer.event("terminated", json!({}))?; - } - } - Ok(()) -} - fn restart_task_through_coordinator(state: &mut AdapterState, thread_id: i64) -> Result { let task = state .threads .get(&thread_id) + .or_else(|| state.threads.get(&default_thread_id(state))) .map(|thread| thread.task.clone()) .ok_or_else(|| anyhow::anyhow!("selected debug thread does not map to a virtual task"))?; let record = restart_task(state, &task)?; @@ -1280,8 +778,6 @@ fn record_coordinator_debug_epoch( .unwrap_or_else(|| state.threads[&default_thread_id(state)].task.clone()); let record = create_debug_epoch(state, &stopped_task, reason)?; let status = wait_for_debug_epoch_frozen(state, record.epoch)?; - let runtime_record = debug_epoch_runtime_record(state, &status, &stopped_task)?; - state.apply_runtime_record(runtime_record); state.debug_all_threads_stopped = status.fully_frozen; state.epoch = record.epoch; state.coordinator_debug_epoch = Some(record.epoch); diff --git a/crates/clusterflux-dap/src/breakpoints.rs b/crates/clusterflux-dap/src/breakpoints.rs index c305794..caebbb3 100644 --- a/crates/clusterflux-dap/src/breakpoints.rs +++ b/crates/clusterflux-dap/src/breakpoints.rs @@ -231,27 +231,10 @@ pub(crate) fn stopped_thread_for_breakpoint(state: &AdapterState) -> i64 { pub(crate) fn position_confirmed_breakpoint_stop(state: &mut AdapterState, stopped_thread: i64) { let line = state - .stopped_probe_symbol - .as_deref() - .and_then(|symbol| symbol.strip_prefix("clusterflux.probe.")) - .and_then(|function| { - state - .debug_probes - .iter() - .find(|probe| probe.function == function) - .and_then(|probe| { - state.breakpoints.iter().copied().find(|line| { - *line >= i64::from(probe.line_start) && *line <= i64::from(probe.line_end) - }) - }) - }) - .or_else(|| { - state - .breakpoints - .iter() - .copied() - .find(|line| thread_for_source_line(state, *line) == stopped_thread) - }) + .breakpoints + .iter() + .copied() + .find(|line| thread_for_source_line(state, *line) == stopped_thread) .or_else(|| state.breakpoints.first().copied()); if let (Some(line), Some(thread)) = (line, state.threads.get_mut(&stopped_thread)) { thread.line = line; @@ -275,29 +258,11 @@ pub(crate) fn next_breakpoint_after(state: &AdapterState, thread_id: i64) -> Opt fn thread_for_source_line(state: &AdapterState, line: i64) -> i64 { if let Some(probe) = debug_probe_for_line(state, line) { - if state.runtime_backend == crate::virtual_model::RuntimeBackend::Simulated { - let function = probe.function.to_ascii_lowercase(); - if function.contains("linux") { - return LINUX_THREAD; - } - if function.contains("windows") { - return WINDOWS_THREAD; - } - if function.contains("package") { - return PACKAGE_THREAD; - } - return MAIN_THREAD; - } return state - .stopped_task - .as_ref() - .and_then(|task| { - state - .threads - .values() - .find(|thread| &thread.task == task) - .map(|thread| thread.id) - }) + .threads + .values() + .find(|thread| thread.task_definition == probe.task) + .map(|thread| thread.id) .unwrap_or(MAIN_THREAD); } diff --git a/crates/clusterflux-dap/src/main.rs b/crates/clusterflux-dap/src/main.rs index b0a9c14..32336b2 100644 --- a/crates/clusterflux-dap/src/main.rs +++ b/crates/clusterflux-dap/src/main.rs @@ -22,7 +22,7 @@ use breakpoints::{ #[cfg(test)] use dap_protocol::{initialize_capabilities, read_message}; #[cfg(test)] -use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code}; +use runtime_client::{client_user_request, parse_task_restart_response}; #[cfg(test)] use variables::variables_response; #[cfg(test)] @@ -36,26 +36,6 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD}; use virtual_model::{process_id, RuntimeBackend}; fn main() -> Result<()> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux Debug Adapter Protocol server.\n\n\ - Usage: clusterflux-debug-dap\n\n\ - The adapter communicates over standard input and output.\n\n\ - Options:\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - [] => {} - [argument, ..] => anyhow::bail!("unknown argument: {argument}"), - } adapter::run_adapter() } diff --git a/crates/clusterflux-dap/src/runtime_client.rs b/crates/clusterflux-dap/src/runtime_client.rs index 9b483ac..00d097e 100644 --- a/crates/clusterflux-dap/src/runtime_client.rs +++ b/crates/clusterflux-dap/src/runtime_client.rs @@ -1,12 +1,10 @@ use std::io::{BufRead, BufReader}; use std::path::Path; use std::process::{Child, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use anyhow::{anyhow, Result}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; -use clusterflux_control::MAX_CONTROL_FRAME_BYTES; use clusterflux_core::TaskInstanceId; use serde_json::{json, Value}; @@ -21,7 +19,7 @@ use debug_protocol::{ }; use local_tools::{child_stderr_suffix, local_tool_command}; pub(crate) use transport::client_user_request; -use transport::{coordinator_request, coordinator_request_allow_error, CoordinatorSession}; +use transport::coordinator_request; pub(crate) struct LocalRuntimeSession { coordinator: Option, @@ -41,16 +39,11 @@ impl Drop for LocalRuntimeSession { struct DebugBundle { module_base64: String, - module_size_bytes: usize, digest: String, entry_export: String, entry_definition: String, } -const INLINE_BUNDLE_REQUEST_OVERHEAD_BYTES: usize = 96 * 1024; -const MAX_INLINE_WASM_MODULE_BYTES: usize = - ((MAX_CONTROL_FRAME_BYTES - INLINE_BUNDLE_REQUEST_OVERHEAD_BYTES) / 4) * 3; - struct ChildGuard(Option); impl ChildGuard { @@ -109,8 +102,6 @@ pub(crate) struct TaskRestartRecord { } pub(crate) enum RuntimeContinuationOutcome { - Snapshot(RuntimeLaunchRecord), - Diagnostic(String), Breakpoint(RuntimeLaunchRecord), Exception(RuntimeLaunchRecord), Terminal(RuntimeLaunchRecord), @@ -285,7 +276,6 @@ fn build_debug_bundle(state: &AdapterState, repo: &Path) -> Result .ok_or_else(|| anyhow!("bundle build omitted bundle digest"))? .to_owned(); Ok(DebugBundle { - module_size_bytes: module.len(), module_base64: BASE64_STANDARD.encode(module), digest, entry_export, @@ -299,9 +289,7 @@ fn launch_services_debug_entrypoint( repo: &Path, ) -> Result { let bundle = build_debug_bundle(state, repo)?; - validate_inline_bundle_size(bundle.module_size_bytes)?; - let launch_attempt = new_launch_attempt_id(); - let started = match coordinator_request_allow_error( + let started = coordinator_request( coordinator, client_user_request( state, @@ -311,58 +299,35 @@ fn launch_services_debug_entrypoint( "project": state.project_id, "actor_user": state.actor_user, "process": state.process.as_str(), - "launch_attempt": launch_attempt, "restart": state.restart_existing, }), ), - ) { - Ok(started) => started, - Err(error) => { - return Err(debug_launch_error_with_rollback( - coordinator, - state, - &launch_attempt, - error, - )); - } - }; - if started.get("type").and_then(Value::as_str) == Some("error") { + )?; + let epoch = started + .get("epoch") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow!("coordinator did not report a process epoch"))?; + let probe_symbols = state.requested_probe_symbols(); + if probe_symbols.is_empty() { return Err(anyhow!( - "{}", - started - .get("message") - .and_then(Value::as_str) - .unwrap_or("coordinator rejected the debug launch") + "no executable Clusterflux probe corresponds to the configured source breakpoints" )); } - if started.get("launch_attempt").and_then(Value::as_str) != Some(launch_attempt.as_str()) { - return Err(debug_launch_error_with_rollback( - coordinator, + coordinator_request( + coordinator, + client_user_request( state, - &launch_attempt, - anyhow!("coordinator returned a debug launch owned by a different attempt"), - )); - } - let epoch = match started.get("epoch").and_then(Value::as_u64) { - Some(epoch) => epoch, - None => { - return Err(debug_launch_error_with_rollback( - coordinator, - state, - &launch_attempt, - anyhow!("coordinator did not report a process epoch"), - )); - } - }; - if let Err(error) = set_services_debug_breakpoints_at(coordinator, state) { - return Err(debug_launch_error_with_rollback( - coordinator, - state, - &launch_attempt, - error, - )); - } - let launch = match coordinator_request( + json!({ + "type": "set_debug_breakpoints", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + "process": state.process.as_str(), + "probe_symbols": probe_symbols, + }), + ), + )?; + let launch = coordinator_request( coordinator, client_user_request( state, @@ -398,60 +363,69 @@ fn launch_services_debug_entrypoint( "wasm_module_base64": bundle.module_base64, }), ), - ) { - Ok(launch) => launch, - Err(error) => { - return Err(debug_launch_error_with_rollback( - coordinator, - state, - &launch_attempt, - error, - )); - } - }; + )?; if launch.get("type").and_then(Value::as_str) != Some("main_launched") { - return Err(debug_launch_error_with_rollback( - coordinator, - state, - &launch_attempt, - anyhow!( - "coordinator did not start the capless main runtime: {}", - serde_json::to_string(&launch)? - ), + return Err(anyhow!( + "coordinator did not start the capless main runtime: {}", + serde_json::to_string(&launch)? )); } - // The launch acknowledgement is the commit point. Failures while fetching - // observation state after this line must not abort a process that is running. - let mut observation_diagnostics = Vec::new(); - let inject_post_commit_observation_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_POST_COMMIT_OBSERVATION_FAILURE").is_some(); - let task_snapshots = (if inject_post_commit_observation_failure { - Err(anyhow!("injected post-commit task observation failure")) - } else { - fetch_task_snapshots(coordinator, state) - }) - .unwrap_or_else(|error| { - observation_diagnostics.push(format!( - "initial task observation failed after main_launched: {error:#}" - )); - json!({ "snapshots": [] }) - }); - let (process_statuses, process_status) = (if inject_post_commit_observation_failure { - Err(anyhow!("injected post-commit process observation failure")) - } else { - fetch_current_process_status(coordinator, state) - }) - .unwrap_or_else(|error| { - observation_diagnostics.push(format!( - "initial process observation failed after main_launched: {error:#}" - )); - (json!({ "processes": [] }), None) - }); - let node = process_status - .as_ref() - .and_then(|status| status.get("connected_nodes")) + let breakpoint = wait_for_breakpoint_hit(coordinator, state)?; + let debug_epoch = breakpoint + .get("hit_epoch") + .and_then(Value::as_u64) + .ok_or_else(|| anyhow!("breakpoint status omitted the hit debug epoch"))?; + let frozen = wait_for_debug_epoch_state_at(coordinator, state, debug_epoch, true)?; + let fully_frozen = frozen + .get("fully_frozen") + .and_then(Value::as_bool) + .unwrap_or(false); + let partially_frozen = frozen + .get("partially_frozen") + .and_then(Value::as_bool) + .unwrap_or(false); + let expected = frozen + .get("expected_tasks") .and_then(Value::as_array) - .and_then(|nodes| nodes.first()) + .map(Vec::len) + .unwrap_or(0); + let acknowledged = frozen + .get("acknowledgements") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0); + if expected == 0 || (!fully_frozen && !partially_frozen) { + return Err(anyhow!( + "debug epoch {debug_epoch} settled without a frozen participant" + )); + } + let task_snapshots = fetch_task_snapshots(coordinator, state)?; + let process_statuses = coordinator_request( + coordinator, + client_user_request( + state, + json!({ + "type": "list_processes", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + }), + ), + )?; + let process_status = process_statuses + .get("processes") + .and_then(Value::as_array) + .and_then(|processes| { + processes.iter().find(|process| { + process.get("process").and_then(Value::as_str) == Some(state.process.as_str()) + }) + }) + .cloned(); + let node = frozen + .get("acknowledgements") + .and_then(Value::as_array) + .and_then(|items| items.first()) + .and_then(|item| item.get("node")) .and_then(Value::as_str) .unwrap_or("coordinator-main") .to_owned(); @@ -461,10 +435,11 @@ fn launch_services_debug_entrypoint( node_report: json!({ "process": started, "task_launch": launch, + "breakpoint": breakpoint, + "debug_epoch": frozen, "process_status": process_status, "process_statuses": process_statuses, "task_snapshots": task_snapshots, - "observation_diagnostics": observation_diagnostics, }), task_events: json!({ "events": [] }), placed_task_launched: true, @@ -477,111 +452,19 @@ fn launch_services_debug_entrypoint( stderr_truncated: false, artifact_path: None, event_count: 0, - debug_epoch: None, - stopped_task: None, - stopped_probe_symbol: None, - all_participants_frozen: false, + debug_epoch: Some(debug_epoch), + stopped_task: breakpoint + .get("hit_task") + .and_then(Value::as_str) + .map(str::to_owned), + stopped_probe_symbol: breakpoint + .get("hit_probe_symbol") + .and_then(Value::as_str) + .map(str::to_owned), + all_participants_frozen: fully_frozen && acknowledged == expected, }) } -fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> { - if module_size_bytes <= MAX_INLINE_WASM_MODULE_BYTES { - return Ok(()); - } - Err(anyhow!( - "built Wasm module is {module_size_bytes} bytes, but the current {MAX_CONTROL_FRAME_BYTES}-byte inline control frame supports at most {MAX_INLINE_WASM_MODULE_BYTES} raw bytes; no virtual process was created" - )) -} - -fn debug_launch_error_with_rollback( - coordinator: &str, - state: &AdapterState, - launch_attempt: &str, - launch_error: anyhow::Error, -) -> anyhow::Error { - let rollback = coordinator_request( - coordinator, - client_user_request( - state, - json!({ - "type": "abort_process", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - "launch_attempt": launch_attempt, - }), - ), - ); - match rollback { - Ok(response) if response.get("type").and_then(Value::as_str) == Some("process_aborted") => { - launch_error - } - Ok(response) => { - anyhow!("{launch_error}; debug launch rollback was not acknowledged: {response}") - } - Err(rollback_error) => { - anyhow!("{launch_error}; debug launch rollback also failed: {rollback_error}") - } - } -} - -static NEXT_LAUNCH_ATTEMPT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); - -fn new_launch_attempt_id() -> String { - let sequence = NEXT_LAUNCH_ATTEMPT.fetch_add(1, Ordering::Relaxed); - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - format!("dap-launch-{}-{nanos}-{sequence}", std::process::id()) -} - -fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> Result<()> { - static INJECTED_INSTALL_FAILURE: AtomicBool = AtomicBool::new(false); - if std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION") - .ok() - .and_then(|value| value.parse::().ok()) - == Some(state.breakpoint_revision) - { - let delay_ms = std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(750); - std::thread::sleep(Duration::from_millis(delay_ms)); - } - let revision_failure = - std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION") - .ok() - .and_then(|value| value.parse::().ok()) - == Some(state.breakpoint_revision); - if (revision_failure - || std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some()) - && !state.breakpoints.is_empty() - && !INJECTED_INSTALL_FAILURE.swap(true, Ordering::SeqCst) - { - return Err(anyhow!( - "coordinator breakpoint installation failed: injected live-install failure" - )); - } - coordinator_request( - coordinator, - client_user_request( - state, - json!({ - "type": "set_debug_breakpoints", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - "revision": state.breakpoint_revision, - "probe_symbols": state.requested_probe_symbols(), - }), - ), - )?; - Ok(()) -} - fn wait_for_local_node( coordinator: &str, state: &AdapterState, @@ -628,6 +511,34 @@ fn wait_for_local_node( } } +fn wait_for_breakpoint_hit(coordinator: &str, state: &AdapterState) -> Result { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let response = coordinator_request( + coordinator, + client_user_request( + state, + json!({ + "type": "inspect_debug_breakpoints", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + "process": state.process.as_str(), + }), + ), + )?; + if response.get("hit_epoch").and_then(Value::as_u64).is_some() { + return Ok(response); + } + if Instant::now() >= deadline { + return Err(anyhow!( + "executing Wasm did not reach any configured source breakpoint within 30 seconds" + )); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + fn wait_for_debug_epoch_state_at( coordinator: &str, state: &AdapterState, @@ -695,12 +606,6 @@ pub(crate) fn run_live_services_runtime(state: &AdapterState) -> Result Result<()> { - let coordinator = - crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); - set_services_debug_breakpoints_at(&coordinator, state) -} - fn fetch_task_snapshots(coordinator: &str, state: &AdapterState) -> Result { coordinator_request( coordinator, @@ -717,22 +622,6 @@ fn fetch_task_snapshots(coordinator: &str, state: &AdapterState) -> Result Result { - session.request(client_user_request( - state, - json!({ - "type": "list_task_snapshots", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - }), - )) -} - fn fetch_current_process_status( coordinator: &str, state: &AdapterState, @@ -742,16 +631,14 @@ fn fetch_current_process_status( client_user_request( state, json!({ - "type": "list_process_summaries", + "type": "list_processes", "tenant": state.tenant, "project": state.project_id, "actor_user": state.actor_user, - "cursor": null, - "limit": 100, }), ), )?; - let summary = statuses + let current = statuses .get("processes") .and_then(Value::as_array) .and_then(|processes| { @@ -760,81 +647,9 @@ fn fetch_current_process_status( }) }) .cloned(); - let current = merge_active_process_status(state, summary, |request| { - coordinator_request(coordinator, request) - })?; Ok((statuses, current)) } -fn fetch_current_process_status_in( - session: &mut CoordinatorSession, - state: &AdapterState, -) -> Result<(Value, Option)> { - let statuses = session.request(client_user_request( - state, - json!({ - "type": "list_process_summaries", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "cursor": null, - "limit": 100, - }), - ))?; - let summary = statuses - .get("processes") - .and_then(Value::as_array) - .and_then(|processes| { - processes.iter().find(|process| { - process.get("process").and_then(Value::as_str) == Some(state.process.as_str()) - }) - }) - .cloned(); - let current = merge_active_process_status(state, summary, |request| session.request(request))?; - Ok((statuses, current)) -} - -fn merge_active_process_status( - state: &AdapterState, - summary: Option, - mut request: F, -) -> Result> -where - F: FnMut(Value) -> Result, -{ - let Some(mut summary) = summary else { - return Ok(None); - }; - if summary.get("lifecycle").and_then(Value::as_str) != Some("active") { - return Ok(Some(summary)); - } - let active_statuses = request(client_user_request( - state, - json!({ - "type": "list_processes", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - }), - ))?; - let active = active_statuses - .get("processes") - .and_then(Value::as_array) - .and_then(|processes| { - processes.iter().find(|process| { - process.get("process").and_then(Value::as_str) == Some(state.process.as_str()) - }) - }); - if let (Some(summary), Some(active)) = - (summary.as_object_mut(), active.and_then(Value::as_object)) - { - for (key, value) in active { - summary.entry(key.clone()).or_insert_with(|| value.clone()); - } - } - Ok(Some(summary)) -} - pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result { let coordinator = crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); @@ -908,7 +723,27 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result Result { - let coordinator = - crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); - let task_snapshots = fetch_task_snapshots(&coordinator, state)?; - let (process_statuses, process_status) = fetch_current_process_status(&coordinator, state)?; - let node = status - .acknowledgements - .first() - .and_then(|acknowledgement| acknowledgement.get("node")) - .and_then(Value::as_str) - .unwrap_or("coordinator-main") - .to_owned(); - Ok(RuntimeLaunchRecord { - coordinator, - node, - node_report: json!({ - "debug_epoch": { - "epoch": status.epoch, - "command": status.command, - "expected_tasks": status.expected_tasks, - "acknowledgements": status.acknowledgements, - "fully_frozen": status.fully_frozen, - "partially_frozen": status.partially_frozen, - "fully_resumed": status.fully_resumed, - "failed": status.failed, - "failure_messages": status.failure_messages, - }, - "task_snapshots": task_snapshots, - "process_status": process_status, - "process_statuses": process_statuses, - }), - task_events: json!({ "events": [] }), - placed_task_launched: true, - status_code: None, - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - event_count: state.runtime_event_count, - debug_epoch: Some(status.epoch), - stopped_task: Some(stopped_task.to_string()), - stopped_probe_symbol: None, - all_participants_frozen: status.fully_frozen, - }) -} - -pub(crate) fn observe_services_runtime( +pub(crate) fn wait_for_services_runtime_outcome( state: &AdapterState, previous_debug_epoch: u64, - cancelled: &AtomicBool, - mut emit: impl FnMut(RuntimeContinuationOutcome) -> bool, -) -> Result<()> { +) -> Result { let coordinator = crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); - let mut session = None; - let mut reconnect_delay = Duration::from_millis(100); - let mut poll_delay = Duration::from_millis(100); - let mut last_snapshot_fingerprint = state.runtime_snapshot_fingerprint.clone(); - let inject_connection_loss = - std::env::var("CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS").as_deref() == Ok("1"); - let mut connection_loss_injected = false; - let inject_snapshot_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE").is_some(); - let mut snapshot_failure_injected = false; - let inject_process_status_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE").is_some(); - let mut process_status_failure_injected = false; - let inject_fallback_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE").is_some(); - let mut fallback_failure_stage = 0_u8; - let inject_debug_epoch_wait_failure = - std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_DEBUG_EPOCH_WAIT_FAILURE").is_some(); - let mut debug_epoch_wait_failure_injected = false; + let join_timeout = clusterflux_core::limits::task_join_timeout(); + let deadline = Instant::now() + join_timeout; loop { - if cancelled.load(Ordering::Acquire) { - return Ok(()); - } - if session.is_none() { - match CoordinatorSession::connect(&coordinator) { - Ok(connected) => { - session = Some(connected); - reconnect_delay = Duration::from_millis(100); - } - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime observer reconnect failed: {error:#}; retrying in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - } - } - // Events remain useful for output and attempt details, but terminal - // authority comes from the durable process summary read below. - let current_session = session.as_mut().expect("observer session connected"); - let events = match current_session.request(client_user_request( - state, - json!({ - "type": "list_task_events", - "tenant": state.tenant, - "project": state.project_id, - "actor_user": state.actor_user, - "process": state.process.as_str(), - }), - )) { - Ok(events) => events, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime observer connection lost: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if inject_connection_loss && !connection_loss_injected { - connection_loss_injected = true; - session = None; - if !emit(RuntimeContinuationOutcome::Diagnostic( - "runtime observer injected one transient connection loss; reconnecting".to_owned(), - )) { - return Ok(()); - } - std::thread::sleep(reconnect_delay); - continue; - } - let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected { - snapshot_failure_injected = true; - Err(anyhow!("injected task snapshot transport failure")) - } else { - fetch_task_snapshots_in(current_session, state) - }; - let task_snapshots = match snapshot_request { - Ok(snapshots) => snapshots, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime snapshot observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - let process_status_request = - if inject_process_status_failure && !process_status_failure_injected { - process_status_failure_injected = true; - Err(anyhow!("injected process-status transport failure")) - } else { - fetch_current_process_status_in(current_session, state) - }; - let (process_statuses, process_status) = match process_status_request { - Ok(status) => status, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime process observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - reconnect_delay = Duration::from_millis(100); - if !has_current_runtime_task(&task_snapshots) { - if let Some(mut outcome) = - terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) - { - if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { - record.node_report = json!({ - "terminal_event": record.node_report.get("terminal_event"), - "task_snapshots": task_snapshots, - "process_status": process_status, - "process_statuses": process_statuses, - }); - } - emit(outcome); - return Ok(()); - } + // Terminal task events remain inspectable after the active process slot is + // released. Read them before active-process debug state so a fast main exit + // cannot turn a successful continuation into an authorization error. + let events = coordinator_request( + &coordinator, + client_user_request( + state, + json!({ + "type": "list_task_events", + "tenant": state.tenant, + "project": state.project_id, + "actor_user": state.actor_user, + "process": state.process.as_str(), + }), + ), + )?; + if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) { + return Ok(outcome); } + let task_snapshots = fetch_task_snapshots(&coordinator, state)?; if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) { let failed_task = failed_task.to_owned(); let failed_event = events @@ -1225,7 +888,9 @@ pub(crate) fn observe_services_runtime( .and_then(Value::as_array) .map(Vec::len) .unwrap_or(state.runtime_event_count); - emit(RuntimeContinuationOutcome::Exception(RuntimeLaunchRecord { + let (process_statuses, process_status) = + fetch_current_process_status(&coordinator, state)?; + return Ok(RuntimeContinuationOutcome::Exception(RuntimeLaunchRecord { coordinator, node: failed_event .get("node") @@ -1281,14 +946,11 @@ pub(crate) fn observe_services_runtime( stopped_probe_symbol: None, all_participants_frozen: false, })); - return Ok(()); } - let breakpoint_request = if inject_fallback_failure && fallback_failure_stage == 0 { - fallback_failure_stage = 1; - Err(anyhow!("injected breakpoint inspection transport failure")) - } else { - current_session.request(client_user_request( + let breakpoint = match coordinator_request( + &coordinator, + client_user_request( state, json!({ "type": "inspect_debug_breakpoints", @@ -1297,9 +959,8 @@ pub(crate) fn observe_services_runtime( "actor_user": state.actor_user, "process": state.process.as_str(), }), - )) - }; - let breakpoint = match breakpoint_request { + ), + ) { Ok(breakpoint) => breakpoint, Err(inspect_error) => { // Main completion records its terminal event and clears ephemeral @@ -1307,11 +968,9 @@ pub(crate) fn observe_services_runtime( // the event read above and this inspection request. Re-read the // durable event stream before treating the missing debug state as // an adapter failure. - let fallback_request = if inject_fallback_failure && fallback_failure_stage == 1 { - fallback_failure_stage = 2; - Err(anyhow!("injected fallback event transport failure")) - } else { - current_session.request(client_user_request( + let events = coordinator_request( + &coordinator, + client_user_request( state, json!({ "type": "list_task_events", @@ -1320,88 +979,17 @@ pub(crate) fn observe_services_runtime( "actor_user": state.actor_user, "process": state.process.as_str(), }), - )) - }; - let events = match fallback_request { - Ok(events) => events, - Err(fallback_error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime breakpoint and fallback event observation failed: {inspect_error:#}; {fallback_error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if let Some(mut outcome) = - terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref()) - { - let task_snapshots = match fetch_task_snapshots_in(current_session, state) { - Ok(snapshots) => snapshots, - Err(snapshot_error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime fallback terminal snapshot observation failed: {snapshot_error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - if !has_current_runtime_task(&task_snapshots) { - if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome { - record.node_report = json!({ - "terminal_event": record.node_report.get("terminal_event"), - "task_snapshots": task_snapshots, - }); - } - emit(outcome); - return Ok(()); - } + ), + )?; + if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) { + return Ok(outcome); } - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime breakpoint observation failed: {inspect_error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; + return Err(inspect_error); } }; if let Some(epoch) = breakpoint.get("hit_epoch").and_then(Value::as_u64) { if epoch > previous_debug_epoch { - let frozen_request = - if inject_debug_epoch_wait_failure && !debug_epoch_wait_failure_injected { - debug_epoch_wait_failure_injected = true; - Err(anyhow!("injected Debug Epoch wait transport failure")) - } else { - wait_for_debug_epoch_state_at(&coordinator, state, epoch, true) - }; - let frozen = match frozen_request { - Ok(frozen) => frozen, - Err(error) => { - if !emit(RuntimeContinuationOutcome::Diagnostic(format!( - "runtime Debug Epoch observation failed: {error:#}; reconnecting in {} ms", - reconnect_delay.as_millis() - ))) { - return Ok(()); - } - session = None; - std::thread::sleep(reconnect_delay); - reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5)); - continue; - } - }; + let frozen = wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)?; let node = frozen .get("acknowledgements") .and_then(Value::as_array) @@ -1410,11 +998,14 @@ pub(crate) fn observe_services_runtime( .and_then(Value::as_str) .unwrap_or("unknown") .to_owned(); + let task_snapshots = fetch_task_snapshots(&coordinator, state)?; + let (process_statuses, process_status) = + fetch_current_process_status(&coordinator, state)?; let fully_frozen = frozen .get("fully_frozen") .and_then(Value::as_bool) .unwrap_or(false); - emit(RuntimeContinuationOutcome::Breakpoint( + return Ok(RuntimeContinuationOutcome::Breakpoint( RuntimeLaunchRecord { coordinator, node, @@ -1448,55 +1039,20 @@ pub(crate) fn observe_services_runtime( all_participants_frozen: fully_frozen, }, )); - return Ok(()); } } - let snapshot_fingerprint = serde_json::to_string(&json!({ - "task_snapshots": task_snapshots, - "process_status": process_status, - }))?; - if snapshot_fingerprint != last_snapshot_fingerprint { - last_snapshot_fingerprint = snapshot_fingerprint.clone(); - poll_delay = Duration::from_millis(100); - if !emit(RuntimeContinuationOutcome::Snapshot(RuntimeLaunchRecord { - coordinator: coordinator.clone(), - node: process_status - .as_ref() - .and_then(|status| status.get("connected_nodes")) - .and_then(Value::as_array) - .and_then(|nodes| nodes.first()) - .and_then(Value::as_str) - .unwrap_or("coordinator-main") - .to_owned(), - node_report: json!({ - "task_snapshots": task_snapshots, - "process_status": process_status, - "process_statuses": process_statuses, - "snapshot_fingerprint": snapshot_fingerprint, - }), - task_events: events, - placed_task_launched: true, - status_code: None, - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - event_count: state.runtime_event_count, - debug_epoch: None, - stopped_task: None, - stopped_probe_symbol: None, - all_participants_frozen: false, - })) { - return Ok(()); - } - } else { - poll_delay = (poll_delay * 2).min(Duration::from_secs(5)); + if Instant::now() >= deadline { + return Err(anyhow::Error::new(clusterflux_core::limits::TaskJoinError::timeout( + clusterflux_core::TaskInstanceId::from("coordinator-main"), + join_timeout, + )) + .context(format!( + "continued virtual process {} neither reached another executing Wasm breakpoint nor recorded terminal entrypoint state", + state.process + ))); } - std::thread::sleep(poll_delay); + std::thread::sleep(Duration::from_millis(100)); } } @@ -1517,126 +1073,68 @@ pub(crate) fn failed_awaiting_action_snapshot(task_snapshots: &Value) -> Option< }) } -fn has_current_runtime_task(task_snapshots: &Value) -> bool { - task_snapshots - .get("snapshots") - .and_then(Value::as_array) - .is_some_and(|snapshots| { - snapshots.iter().any(|snapshot| { - snapshot.get("current").and_then(Value::as_bool) == Some(true) - && matches!( - snapshot.get("state").and_then(Value::as_str), - Some("queued" | "running" | "failed_awaiting_action") - ) - }) - }) -} - -#[cfg(test)] -pub(crate) fn whole_process_status_code( - main_status_code: Option, - task_snapshots: &Value, -) -> Option { - let main_status_code = main_status_code?; - if main_status_code != 0 { - return Some(main_status_code); - } - - let snapshots = task_snapshots.get("snapshots").and_then(Value::as_array)?; - for snapshot in snapshots - .iter() - .filter(|snapshot| snapshot.get("current").and_then(Value::as_bool) == Some(true)) - { - match snapshot.get("state").and_then(Value::as_str) { - Some("completed") => {} - Some("failed" | "cancelled" | "failed_awaiting_action") => { - return Some( - snapshot - .get("status_code") - .and_then(Value::as_i64) - .and_then(|status| i32::try_from(status).ok()) - .filter(|status| *status != 0) - .unwrap_or(1), - ); - } - Some("queued" | "running") => return None, - _ => return Some(1), - } - } - Some(0) -} - fn terminal_runtime_outcome( coordinator: &str, - _state: &AdapterState, + state: &AdapterState, events: &Value, - process_status: Option<&Value>, ) -> Option { - let final_result = process_status? - .get("final_result") - .and_then(Value::as_str)?; - let status_code = match final_result { - "completed" => 0, - "failed" | "cancelled" => 1, - _ => return None, - }; let event = events .get("events") - .and_then(Value::as_array) - .and_then(|events| { - events.iter().rev().find(|event| { - event.get("executor").and_then(Value::as_str) == Some("coordinator_main") - }) - }); + .and_then(Value::as_array)? + .get(state.runtime_event_count..)? + .iter() + .rev() + .find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?; + let status_code = event + .get("status_code") + .and_then(Value::as_i64) + .map(|status| status as i32) + .or_else( + || match event.get("terminal_state").and_then(Value::as_str) { + Some("completed") => Some(0), + Some("failed" | "cancelled") => Some(1), + _ => None, + }, + ); Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord { coordinator: coordinator.to_owned(), node: event - .and_then(|event| event.get("node")) + .get("node") .and_then(Value::as_str) - .or_else(|| { - process_status - .and_then(|status| status.get("connected_nodes")) - .and_then(Value::as_array) - .and_then(|nodes| nodes.first()) - .and_then(Value::as_str) - }) .unwrap_or("coordinator-main") .to_owned(), - node_report: json!({ - "terminal_event": event, - "process_status": process_status, - }), + node_report: json!({ "terminal_event": event }), task_events: events.clone(), placed_task_launched: true, - status_code: Some(status_code), + status_code, stdout_bytes: event - .and_then(|event| event.get("stdout_bytes")) + .get("stdout_bytes") .and_then(Value::as_u64) .unwrap_or(0), stderr_bytes: event - .and_then(|event| event.get("stderr_bytes")) + .get("stderr_bytes") .and_then(Value::as_u64) .unwrap_or(0), stdout_tail: event - .and_then(|event| event.get("stdout_tail")) + .get("stdout_tail") .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stderr_tail: event - .and_then(|event| event.get("stderr_tail")) + .get("stderr_tail") .and_then(Value::as_str) .unwrap_or_default() .to_owned(), stdout_truncated: event - .and_then(|event| event.get("stdout_truncated")) + .get("stdout_truncated") .and_then(Value::as_bool) .unwrap_or(false), stderr_truncated: event - .and_then(|event| event.get("stderr_truncated")) + .get("stderr_truncated") .and_then(Value::as_bool) .unwrap_or(false), artifact_path: event - .and_then(|event| event.get("artifact_path")) + .get("artifact_path") .and_then(Value::as_str) .map(str::to_owned), event_count: events @@ -1657,14 +1155,6 @@ pub(crate) fn restart_task( ) -> Result { let repo = std::env::current_dir()?; let replacement = build_debug_bundle(state, &repo)?; - let project_root = Path::new(&state.project); - let project_root = if project_root.is_absolute() { - project_root.to_path_buf() - } else { - repo.join(project_root) - }; - let source_snapshot = clusterflux_node::snapshot_project_digest(&project_root) - .map_err(|error| anyhow!("snapshot replacement source checkout: {error}"))?; let response = coordinator_debug_epoch_request( state, client_user_request( @@ -1679,132 +1169,9 @@ pub(crate) fn restart_task( "replacement_bundle": { "bundle_digest": replacement.digest, "wasm_module_base64": replacement.module_base64, - "source_snapshot": source_snapshot, }, }), ), )?; parse_task_restart_response(response) } - -#[cfg(test)] -mod transactional_launch_tests { - use std::io::{BufRead as _, Write as _}; - use std::net::TcpListener; - - use super::*; - - #[test] - fn current_child_task_keeps_runtime_observation_alive_after_main_completion() { - assert!(has_current_runtime_task(&json!({ - "snapshots": [{ - "current": true, - "state": "running", - "task": "child" - }] - }))); - assert!(!has_current_runtime_task(&json!({ - "snapshots": [{ - "current": false, - "state": "completed", - "task": "main" - }] - }))); - } - - #[test] - fn inline_bundle_limit_is_checked_before_process_creation() { - assert_eq!(MAX_CONTROL_FRAME_BYTES, 1024 * 1024); - validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES).unwrap(); - let error = validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES + 1) - .unwrap_err() - .to_string(); - assert!(error.contains("no virtual process was created")); - } - - #[test] - fn durable_process_summary_is_terminal_authority_after_event_rotation() { - let state = AdapterState { - runtime_event_count: 10_000, - ..AdapterState::default() - }; - let completed = terminal_runtime_outcome( - "127.0.0.1:1", - &state, - &json!({ "events": [] }), - Some(&json!({ - "process": state.process.as_str(), - "lifecycle": "recent_terminal", - "final_result": "completed", - "connected_nodes": [] - })), - ) - .expect("the durable summary should terminate observation"); - let RuntimeContinuationOutcome::Terminal(completed) = completed else { - panic!("expected terminal outcome"); - }; - assert_eq!(completed.status_code, Some(0)); - - let failed = terminal_runtime_outcome( - "127.0.0.1:1", - &state, - &json!({ - "events": [{ - "executor": "coordinator_main", - "terminal_state": "completed", - "status_code": 0 - }] - }), - Some(&json!({ - "process": state.process.as_str(), - "lifecycle": "recent_terminal", - "final_result": "failed", - "connected_nodes": [] - })), - ) - .expect("the aggregate summary should override a successful main event"); - let RuntimeContinuationOutcome::Terminal(failed) = failed else { - panic!("expected terminal outcome"); - }; - assert_eq!(failed.status_code, Some(1)); - } - - #[test] - fn failed_debug_launch_reconnects_and_aborts_the_process() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap().to_string(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut line = String::new(); - std::io::BufReader::new(stream.try_clone().unwrap()) - .read_line(&mut line) - .unwrap(); - assert!(line.contains("\"type\":\"abort_process\"")); - writeln!( - stream, - "{}", - json!({ - "type": "process_aborted", - "process": "vp-test", - "aborted_tasks": [], - "affected_nodes": [] - }) - ) - .unwrap(); - }); - let state = AdapterState { - process: clusterflux_core::ProcessId::from("vp-test"), - ..AdapterState::default() - }; - let error = debug_launch_error_with_rollback( - &address, - &state, - "launch-test", - anyhow!("launch acknowledgement was lost"), - ); - assert!(error - .to_string() - .contains("launch acknowledgement was lost")); - server.join().unwrap(); - } -} diff --git a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs index add7be2..38dbc2f 100644 --- a/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs +++ b/crates/clusterflux-dap/src/runtime_client/debug_protocol.rs @@ -173,9 +173,7 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result V }) } -pub(super) struct CoordinatorSession { - session: ControlSession, -} - -impl CoordinatorSession { - pub(super) fn connect(addr: &str) -> Result { - Ok(Self { - session: ControlSession::connect(addr)?, - }) - } - - pub(super) fn request(&mut self, request: Value) -> Result { - let response = self.request_allow_error(request)?; - if response.get("type").and_then(Value::as_str) == Some("error") { - return Err(anyhow!( - "{}", - response - .get("message") - .and_then(Value::as_str) - .unwrap_or("coordinator returned an error") - )); - } - Ok(response) - } - - pub(super) fn request_allow_error(&mut self, request: Value) -> Result { - let wire_request = coordinator_wire_request("dap-1", request); - Ok(self.session.request(&wire_request)?) - } -} - pub(super) fn coordinator_request(addr: &str, request: Value) -> Result { - CoordinatorSession::connect(addr)?.request(request) -} - -pub(super) fn coordinator_request_allow_error(addr: &str, request: Value) -> Result { - CoordinatorSession::connect(addr)?.request_allow_error(request) + let mut session = ControlSession::connect(addr)?; + let wire_request = coordinator_wire_request("dap-1", request); + let response = session.request(&wire_request)?; + if response.get("type").and_then(Value::as_str) == Some("error") { + return Err(anyhow!( + "{}", + response + .get("message") + .and_then(Value::as_str) + .unwrap_or("coordinator returned an error") + )); + } + Ok(response) } diff --git a/crates/clusterflux-dap/src/source.rs b/crates/clusterflux-dap/src/source.rs index 6517b9e..1a3e019 100644 --- a/crates/clusterflux-dap/src/source.rs +++ b/crates/clusterflux-dap/src/source.rs @@ -7,12 +7,12 @@ use serde_json::{json, Value}; use crate::virtual_model::AdapterState; pub(crate) fn infer_source_path(project: &str) -> String { - if Path::new(project).join("src/lib.rs").is_file() { - "src/lib.rs".to_owned() + if Path::new(project).join("src/build.rs").is_file() { + "src/build.rs".to_owned() } else if Path::new(project).join("src/main.rs").is_file() { "src/main.rs".to_owned() - } else if Path::new(project).join("src/build.rs").is_file() { - "src/build.rs".to_owned() + } else if Path::new(project).join("src/lib.rs").is_file() { + "src/lib.rs".to_owned() } else { "src/main.rs".to_owned() } diff --git a/crates/clusterflux-dap/src/tests.rs b/crates/clusterflux-dap/src/tests.rs index ed1c82f..b8fa0ea 100644 --- a/crates/clusterflux-dap/src/tests.rs +++ b/crates/clusterflux-dap/src/tests.rs @@ -45,13 +45,6 @@ fn initialize_capabilities_use_standard_dap_flags() { .all(|key| key.starts_with("supports") && !key.contains("clusterflux"))); } -#[test] -fn stale_breakpoint_success_and_failure_completions_are_not_current() { - assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9)); - assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9)); - assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9)); -} - #[test] fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() { assert_eq!( @@ -196,19 +189,10 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() { state.apply_runtime_record(RuntimeLaunchRecord { coordinator: "127.0.0.1:7999".to_owned(), node: "node".to_owned(), - node_report: json!({ - "task_snapshots": { "snapshots": [{ - "task": "compile-linux-1", - "task_definition": "compile-linux", - "attempt_id": "attempt-1", - "current": true, - "state": "failed_awaiting_action" - }] }, - "terminal_event": { - "task": "compile-linux-1", - "task_definition": "compile-linux" - } - }), + node_report: json!({ "terminal_event": { + "task": "compile-linux-1", + "task_definition": "compile-linux" + } }), task_events: json!({ "events": [] }), placed_task_launched: true, status_code: Some(1), @@ -225,25 +209,20 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() { stopped_probe_symbol: None, all_participants_frozen: false, }); - let exact_runtime_thread = state + let linux = state .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("compile-linux-1")) - .expect("the exact terminal task instance should have its own thread"); + .get(&LINUX_THREAD) + .expect("linux task should exist"); assert!(state.last_task_failed); assert!(state .command_status .contains("failed through local services")); - assert!(matches!( - exact_runtime_thread.state, - DebugRuntimeState::Failed(_) - )); - assert!(exact_runtime_thread + assert!(matches!(linux.state, DebugRuntimeState::Failed(_))); + assert!(linux .recent_output .iter() .any(|line| line.contains("task failed"))); - assert_eq!(state.threads.len(), 1); } #[test] @@ -296,7 +275,7 @@ fn breakpoint_resolution_uses_bundle_debug_probe_metadata() { let mut state = AdapterState::default(); state.debug_probes = vec![BundleDebugProbe { id: "probe-linux".to_owned(), - source_path: "src/lib.rs".to_owned(), + source_path: "src/build.rs".to_owned(), line_start: 20, line_end: 30, function: "compile_linux".to_owned(), @@ -324,7 +303,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints( let mut state = AdapterState::default(); state.project = project.path().to_string_lossy().into_owned(); - state.source_path = "src/lib.rs".to_owned(); + state.source_path = "src/build.rs".to_owned(); state.breakpoints = vec![1]; let requested_source = source_dir.join("main.rs"); @@ -332,7 +311,9 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints( assert_eq!(resolved.len(), 1); assert!(!resolved[0].verified); - assert!(resolved[0].message.contains("configured for `src/lib.rs`")); + assert!(resolved[0] + .message + .contains("configured for `src/build.rs`")); assert_eq!(state.breakpoints, vec![1]); } @@ -377,11 +358,7 @@ fn windows_thread_runtime_variables_share_virtual_process() { variable["name"] == "virtual_process_id" && variable["value"] == state.process.to_string() })); assert!(variables.iter().any(|variable| { - variable["name"] == "virtual_thread" && variable["value"] == "compile-windows" - })); - assert!(variables.iter().any(|variable| { - variable["name"] == "task_attempt_id" - && variable["value"] == format!("demo-attempt-{WINDOWS_THREAD}") + variable["name"] == "virtual_thread" && variable["value"] == "compile windows" })); } @@ -392,19 +369,10 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() { state.apply_runtime_record(RuntimeLaunchRecord { coordinator: "127.0.0.1:7999".to_owned(), node: "node".to_owned(), - node_report: json!({ - "task_snapshots": { "snapshots": [{ - "task": "compile-linux-1", - "task_definition": "compile-linux", - "attempt_id": "attempt-1", - "current": true, - "state": "running" - }] }, - "terminal_event": { - "task": "compile-linux-1", - "task_definition": "compile-linux" - } - }), + node_report: json!({ "terminal_event": { + "task": "compile-linux-1", + "task_definition": "compile-linux" + } }), task_events: json!({ "events": [] }), placed_task_launched: true, status_code: Some(0), @@ -421,28 +389,21 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() { stopped_probe_symbol: None, all_participants_frozen: false, }); - let exact_thread_id = state - .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("compile-linux-1")) - .map(|thread| thread.id) - .expect("the exact terminal task instance should have its own thread"); { - let runtime_thread = state.threads.get_mut(&exact_thread_id).unwrap(); - runtime_thread.runtime_task_args = - vec![("source".to_owned(), "source://checkout".to_owned())]; - runtime_thread.runtime_handles = vec![( + let linux = state.threads.get_mut(&LINUX_THREAD).unwrap(); + linux.runtime_task_args = vec![("source".to_owned(), "source://checkout".to_owned())]; + linux.runtime_handles = vec![( "artifact".to_owned(), "artifact://runtime/output".to_owned(), )]; - runtime_thread.runtime_command_status = Some("frozen native command pid 42".to_owned()); + linux.runtime_command_status = Some("frozen native command pid 42".to_owned()); } - let runtime_thread = state + let linux = state .threads - .get(&exact_thread_id) - .expect("exact runtime thread should exist"); + .get(&LINUX_THREAD) + .expect("linux task should exist"); - let args = variables_response(&state, runtime_thread.args_ref); + let args = variables_response(&state, linux.args_ref); let args = args["variables"] .as_array() .expect("variables response should be an array"); @@ -457,7 +418,7 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() { && variable["type"] == "runtime-handle" })); - let runtime = variables_response(&state, runtime_thread.runtime_ref); + let runtime = variables_response(&state, linux.runtime_ref); let runtime = runtime["variables"].as_array().unwrap(); let command_ref = runtime .iter() @@ -672,74 +633,15 @@ fn terminal_events_do_not_resurrect_threads_absent_from_current_snapshots() { assert!(state.threads.is_empty()); } -#[test] -fn terminal_record_without_snapshots_clears_active_threads() { - let mut state = AdapterState::default(); - state.runtime_backend = RuntimeBackend::LiveServices; - assert!(!state.threads.is_empty()); - - state.apply_runtime_record(RuntimeLaunchRecord { - coordinator: "127.0.0.1:9443".to_owned(), - node: "coordinator-main".to_owned(), - node_report: json!({ "terminal_event": { "terminal_state": "completed" } }), - task_events: json!({ "events": [] }), - placed_task_launched: true, - status_code: Some(0), - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - event_count: 0, - debug_epoch: None, - stopped_task: None, - stopped_probe_symbol: None, - all_participants_frozen: false, - }); - - assert!(state.threads.is_empty()); -} - #[test] fn source_locals_infer_clusterflux_api_values_from_runtime_state() { - let project = std::env::temp_dir().join(format!( - "clusterflux-dap-source-locals-{}", - std::process::id() - )); - let src = project.join("src"); - fs::create_dir_all(&src).unwrap(); - fs::write( - src.join("lib.rs"), - r#"async fn run_build_workflow() { - let source = prepare_source_snapshot(); - let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) - .name("compile linux") - .env(linux_env()) - .start() - .await - .unwrap(); - let linux_thread = linux.virtual_thread_id(); - let linux_artifact = linux.join().await.unwrap(); - let package = clusterflux::spawn::async_task_with_arg( - vec![linux_artifact.clone()], - package_release, - ) - .name("package artifacts") - .env(linux_env()) - .start() - .await - .unwrap(); - let package_artifact = package.join().await.unwrap(); -} -"#, - ) - .unwrap(); - let mut state = AdapterState::default(); + let project = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../examples/launch-build-demo") + .canonicalize() + .unwrap(); state.project = project.to_string_lossy().into_owned(); - state.source_path = "src/lib.rs".to_owned(); + state.source_path = "src/build.rs".to_owned(); let source = fs::read_to_string(project.join(&state.source_path)).unwrap(); let inspection_line = source .lines() @@ -773,8 +675,6 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() { variable["name"] == "unavailable-local-diagnostic" && variable["type"] == "unavailable-local" })); - - let _ = fs::remove_dir_all(project); } #[test] @@ -894,21 +794,12 @@ fn package_release(inputs: Vec) -> Artifact { #[test] fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { - let project = std::env::temp_dir().join(format!( - "clusterflux-dap-wasm-locals-{}", - std::process::id() - )); - let src = project.join("src"); - fs::create_dir_all(&src).unwrap(); - fs::write( - src.join("lib.rs"), - "pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n input + 1\n}\n", - ) - .unwrap(); - let mut state = AdapterState::default(); - state.project = project.to_string_lossy().into_owned(); - state.source_path = "src/lib.rs".to_owned(); + state.project = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../examples/launch-build-demo") + .to_string_lossy() + .into_owned(); + state.source_path = "src/build.rs".to_owned(); let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap(); state.threads.get_mut(&MAIN_THREAD).unwrap().line = source .lines() @@ -935,8 +826,6 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() { assert!(!locals .iter() .any(|variable| variable["name"] == "wasm-local-diagnostic")); - - let _ = fs::remove_dir_all(project); } #[test] @@ -963,188 +852,3 @@ fn detects_current_failed_attempt_awaiting_operator_action() { Some(("task-current", "attempt-current")) ); } - -#[test] -fn whole_process_terminal_status_covers_every_current_logical_task_attempt() { - let snapshots = |items: serde_json::Value| json!({ "snapshots": items }); - - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(0), - "a successful main and final child must succeed" - ); - for (state, status_code) in [("failed", 23), ("cancelled", 1)] { - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": state, - "status_code": status_code - } - ])) - ), - Some(status_code), - "a terminal child must determine the whole-process result" - ); - } - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "accepted-failure", - "current": true, - "state": "failed", - "command_state": "failure_accepted", - "status_code": 17 - } - ])) - ), - Some(17), - "accepting an AwaitOperator failure releases the process but does not turn failure into success" - ); - assert_eq!( - whole_process_status_code( - Some(0), - &snapshots(json!([ - { - "task": "restarted", - "attempt_id": "attempt-1", - "current": false, - "state": "failed", - "status_code": 7 - }, - { - "task": "restarted", - "attempt_id": "attempt-2", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(0), - "a successful replacement attempt is authoritative over stale failed attempts" - ); - assert_eq!( - whole_process_status_code( - Some(9), - &snapshots(json!([ - { - "task": "child", - "current": true, - "state": "completed", - "status_code": 0 - } - ])) - ), - Some(9), - "a failed coordinator main cannot be masked by successful children" - ); -} - -#[test] -fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() { - let mut state = AdapterState::default(); - state.runtime_backend = RuntimeBackend::LiveServices; - let record = |snapshots: serde_json::Value| RuntimeLaunchRecord { - coordinator: "127.0.0.1:9443".to_owned(), - node: "node-a".to_owned(), - node_report: json!({ - "task_snapshots": { "snapshots": snapshots }, - "process_status": null - }), - task_events: json!({ "events": [] }), - placed_task_launched: true, - status_code: None, - stdout_bytes: 0, - stderr_bytes: 0, - stdout_tail: String::new(), - stderr_tail: String::new(), - stdout_truncated: false, - stderr_truncated: false, - artifact_path: None, - event_count: 0, - debug_epoch: None, - stopped_task: None, - stopped_probe_symbol: None, - all_participants_frozen: false, - }; - state.apply_attach_record(record(json!([ - { - "task": "lane-stable", - "task_definition": "build_lane", - "attempt_id": "stable-1", - "current": true, - "state": "running" - }, - { - "task": "lane-recovering", - "task_definition": "build_lane", - "attempt_id": "recovering-1", - "current": true, - "state": "failed_awaiting_action" - } - ]))); - let stable_id = state - .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("lane-stable")) - .unwrap() - .id; - let recovering_id = state - .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("lane-recovering")) - .unwrap() - .id; - assert_ne!(stable_id, recovering_id); - - state.apply_runtime_record(record(json!([ - { - "task": "lane-new", - "task_definition": "build_lane", - "attempt_id": "new-1", - "current": true, - "state": "running" - }, - { - "task": "lane-stable", - "task_definition": "build_lane", - "attempt_id": "stable-2", - "current": true, - "state": "running" - } - ]))); - - let stable = state - .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("lane-stable")) - .unwrap(); - let new_lane = state - .threads - .values() - .find(|thread| thread.task == TaskInstanceId::from("lane-new")) - .unwrap(); - assert_eq!(stable.id, stable_id); - assert_eq!(stable.attempt_id, "stable-2"); - assert!(state - .threads - .values() - .all(|thread| thread.task != TaskInstanceId::from("lane-recovering"))); - assert!(new_lane.id > recovering_id); -} diff --git a/crates/clusterflux-dap/src/variables.rs b/crates/clusterflux-dap/src/variables.rs index 7d6614a..97483d8 100644 --- a/crates/clusterflux-dap/src/variables.rs +++ b/crates/clusterflux-dap/src/variables.rs @@ -149,7 +149,7 @@ pub(crate) fn variables_response(state: &AdapterState, reference: i64) -> Value }, { "name": "virtual_thread", - "value": thread.task.to_string(), + "value": thread.name, "variablesReference": 0 }, { diff --git a/crates/clusterflux-dap/src/virtual_model.rs b/crates/clusterflux-dap/src/virtual_model.rs index 9ef76eb..a1b03f3 100644 --- a/crates/clusterflux-dap/src/virtual_model.rs +++ b/crates/clusterflux-dap/src/virtual_model.rs @@ -66,16 +66,11 @@ pub(crate) struct AdapterState { pub(crate) last_task_failed: bool, pub(crate) runtime_artifact_path: Option, pub(crate) runtime_event_count: usize, - pub(crate) runtime_snapshot_fingerprint: String, - pub(crate) next_thread_id: i64, pub(crate) coordinator_debug_epoch: Option, pub(crate) debug_all_threads_stopped: bool, pub(crate) stopped_task: Option, - pub(crate) stopped_probe_symbol: Option, pub(crate) debug_probes: Vec, pub(crate) breakpoints: Vec, - pub(crate) breakpoints_installed: bool, - pub(crate) breakpoint_revision: u64, pub(crate) threads: BTreeMap, } @@ -112,9 +107,9 @@ impl Default for AdapterState { threads: launch_threads(&entry), entry, project, - source_path: "src/lib.rs".to_owned(), + source_path: "src/build.rs".to_owned(), runtime_backend: RuntimeBackend::Simulated, - coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(), + coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(), tenant: TenantId::from("tenant"), project_id: ProjectId::from("project"), actor_user: UserId::from("dap"), @@ -125,16 +120,11 @@ impl Default for AdapterState { last_task_failed: false, runtime_artifact_path: None, runtime_event_count: 0, - runtime_snapshot_fingerprint: String::new(), - next_thread_id: LINUX_THREAD, coordinator_debug_epoch: None, debug_all_threads_stopped: true, stopped_task: None, - stopped_probe_symbol: None, debug_probes: Vec::new(), breakpoints: Vec::new(), - breakpoints_installed: false, - breakpoint_revision: 0, } } } @@ -200,24 +190,18 @@ impl AdapterState { } self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(&session.coordinator); - self.tenant = TenantId::try_new(session.tenant) - .map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?; - self.project_id = ProjectId::try_new(session.project) - .map_err(|error| anyhow!("invalid project in CLI session: {error}"))?; - self.actor_user = UserId::try_new(session.user) - .map_err(|error| anyhow!("invalid user in CLI session: {error}"))?; + self.tenant = TenantId::new(session.tenant); + self.project_id = ProjectId::new(session.project); + self.actor_user = UserId::new(session.user); self.client_session_secret = Some(session.session_secret); } else { self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint( coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"), ); if let Some(scope) = project_scope { - self.tenant = TenantId::try_new(scope.tenant) - .map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?; - self.project_id = ProjectId::try_new(scope.project) - .map_err(|error| anyhow!("invalid project in project scope: {error}"))?; - self.actor_user = UserId::try_new(scope.user) - .map_err(|error| anyhow!("invalid user in project scope: {error}"))?; + self.tenant = TenantId::new(scope.tenant); + self.project_id = ProjectId::new(scope.project); + self.actor_user = UserId::new(scope.user); } self.client_session_secret = None; } @@ -227,18 +211,13 @@ impl AdapterState { self.last_task_failed = false; self.runtime_artifact_path = None; self.runtime_event_count = 0; - self.runtime_snapshot_fingerprint.clear(); - self.next_thread_id = LINUX_THREAD; self.coordinator_debug_epoch = None; self.debug_all_threads_stopped = true; self.stopped_task = None; - self.stopped_probe_symbol = None; self.debug_probes = crate::breakpoints::load_bundle_debug_probes(&self.project, &self.source_path); self.epoch = 0; self.breakpoints.clear(); - self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated; - self.breakpoint_revision = 0; self.threads = if self.runtime_backend == RuntimeBackend::Simulated { launch_threads(&self.entry) } else { @@ -251,18 +230,13 @@ impl AdapterState { self.coordinator_endpoint = record.coordinator.clone(); if self.runtime_backend != RuntimeBackend::Simulated { if let Some(snapshots) = record.node_report.get("task_snapshots") { - self.reconcile_runtime_threads(snapshots, record.node_report.get("process_status")); - } else if record.node_report.get("terminal_event").is_some() { - self.threads.clear(); + self.threads = coordinator_threads_from_snapshots( + &self.entry, + snapshots, + record.node_report.get("process_status"), + ); } } - if let Some(fingerprint) = record - .node_report - .get("snapshot_fingerprint") - .and_then(Value::as_str) - { - self.runtime_snapshot_fingerprint = fingerprint.to_owned(); - } if !record.placed_task_launched { self.command_status = format!( "virtual process started through {}; no runtime task event observed yet", @@ -313,11 +287,7 @@ impl AdapterState { self.debug_all_threads_stopped = record.all_participants_frozen; self.epoch = record.debug_epoch.unwrap_or(self.epoch); self.coordinator_debug_epoch = record.debug_epoch; - self.stopped_task = record - .stopped_task - .as_deref() - .and_then(|task| TaskInstanceId::try_new(task).ok()); - self.stopped_probe_symbol = record.stopped_probe_symbol.clone(); + self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new); if let Some(acknowledgements) = record .node_report .pointer("/debug_epoch/acknowledgements") @@ -333,12 +303,6 @@ impl AdapterState { else { continue; }; - let Ok(task_id) = TaskInstanceId::try_new(task) else { - continue; - }; - let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else { - continue; - }; let coordinator_main = acknowledgement.get("node").and_then(Value::as_str) == Some("coordinator-main"); let thread_id = if coordinator_main { @@ -348,19 +312,14 @@ impl AdapterState { .values() .find(|thread| thread.task.as_str() == task) .map(|thread| thread.id) - .unwrap_or_else(|| { - self.insert_runtime_thread( - task_id.clone(), - task_definition_id.clone(), - ) - }) + .unwrap_or_else(|| self.insert_runtime_thread(task, task_definition)) }; let thread = self .threads .get_mut(&thread_id) .expect("runtime thread was found or inserted"); - thread.task = task_id; - thread.task_definition = task_definition_id; + thread.task = TaskInstanceId::new(task); + thread.task_definition = TaskDefinitionId::new(task_definition); thread.runtime_stack_frames = acknowledgement .get("stack_frames") .and_then(Value::as_array) @@ -403,12 +362,23 @@ impl AdapterState { .and_then(Value::as_str) .unwrap_or(self.entry.as_str()) .to_owned(); + let terminal_task_definition = terminal_event + .get("task_definition") + .and_then(Value::as_str) + .unwrap_or(self.entry.as_str()) + .to_owned(); let terminal_thread_id = self .threads .values() - .find(|thread| thread.task.as_str() == terminal_task.as_str()) - .map(|thread| thread.id); - if let Some(thread) = terminal_thread_id.and_then(|id| self.threads.get_mut(&id)) { + .find(|thread| { + thread.task.as_str() == terminal_task.as_str() + || thread.task_definition.as_str() == terminal_task_definition.as_str() + }) + .map(|thread| thread.id) + .unwrap_or_else(|| { + self.insert_runtime_thread(&terminal_task, &terminal_task_definition) + }); + if let Some(thread) = self.threads.get_mut(&terminal_thread_id) { thread.stdout_bytes = record.stdout_bytes; thread.stderr_bytes = record.stderr_bytes; thread.stdout_tail = record.stdout_tail.clone(); @@ -442,22 +412,24 @@ impl AdapterState { let _ = crate::view_state::write_view_state(self, &record); } - fn insert_runtime_thread( - &mut self, - task: TaskInstanceId, - task_definition: TaskDefinitionId, - ) -> i64 { - let id = self.allocate_runtime_thread_id(); + fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 { + let id = self + .threads + .keys() + .next_back() + .copied() + .unwrap_or(MAIN_THREAD) + + 1; let line = self .debug_probes .iter() .find(|probe| { - probe.task == task_definition || probe.function == task_definition.as_str() + probe.task.as_str() == task_definition || probe.function == task_definition }) .map(|probe| i64::from(probe.line_start)) .unwrap_or(1); - let definition_name = task_definition.as_str().replace(['_', '-'], " "); - let name = if task.as_str() == task_definition.as_str() { + let definition_name = task_definition.replace(['_', '-'], " "); + let name = if task == task_definition { definition_name } else { format!("{definition_name} ({task})") @@ -475,9 +447,9 @@ impl AdapterState { target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task, + task: TaskInstanceId::new(task), attempt_id: "unknown-attempt".to_owned(), - task_definition, + task_definition: TaskDefinitionId::new(task_definition), name, line, state: DebugRuntimeState::Running, @@ -503,51 +475,6 @@ impl AdapterState { id } - fn allocate_runtime_thread_id(&mut self) -> i64 { - while self.threads.contains_key(&self.next_thread_id) || self.next_thread_id == MAIN_THREAD - { - self.next_thread_id += 1; - } - let id = self.next_thread_id; - self.next_thread_id += 1; - id - } - - fn reconcile_runtime_threads( - &mut self, - task_snapshots: &Value, - process_status: Option<&Value>, - ) { - let desired = - coordinator_threads_from_snapshots(&self.entry, task_snapshots, process_status); - let previous = std::mem::take(&mut self.threads); - let mut reconciled = BTreeMap::new(); - for (_, mut thread) in desired { - let coordinator_main = process_status - .and_then(|status| status.get("main_task_instance")) - .and_then(Value::as_str) - .is_some_and(|task| task == thread.task.as_str()); - let existing = if coordinator_main { - previous.get(&MAIN_THREAD) - } else { - previous - .values() - .find(|current| current.task == thread.task) - }; - if let Some(existing) = existing { - preserve_thread_identity(&mut thread, existing); - } else if !coordinator_main { - thread.id = self.allocate_runtime_thread_id(); - assign_thread_references(&mut thread); - } else { - thread.id = MAIN_THREAD; - assign_thread_references(&mut thread); - } - reconciled.insert(thread.id, thread); - } - self.threads = reconciled; - } - pub(crate) fn apply_attach_record(&mut self, record: RuntimeLaunchRecord) { self.coordinator_endpoint = record.coordinator.clone(); self.command_status = format!( @@ -558,7 +485,8 @@ impl AdapterState { self.last_task_failed = false; self.runtime_artifact_path = None; self.runtime_event_count = record.event_count; - self.reconcile_runtime_threads( + self.threads = coordinator_threads_from_snapshots( + &self.entry, record .node_report .get("task_snapshots") @@ -579,35 +507,6 @@ impl AdapterState { } } -fn preserve_thread_identity(thread: &mut VirtualThread, existing: &VirtualThread) { - thread.id = existing.id; - thread.frame_id = existing.frame_id; - thread.locals_ref = existing.locals_ref; - thread.wasm_locals_ref = existing.wasm_locals_ref; - thread.args_ref = existing.args_ref; - thread.runtime_ref = existing.runtime_ref; - thread.output_ref = existing.output_ref; - thread.target_ref = existing.target_ref; - thread.vfs_ref = existing.vfs_ref; - thread.command_ref = existing.command_ref; - if thread.recent_output.is_empty() { - thread.recent_output = existing.recent_output.clone(); - } -} - -fn assign_thread_references(thread: &mut VirtualThread) { - let id = thread.id; - thread.frame_id = 1000 + id; - thread.locals_ref = 5000 + id; - thread.wasm_locals_ref = 9000 + id; - thread.args_ref = 2000 + id; - thread.runtime_ref = 3000 + id; - thread.output_ref = 4000 + id; - thread.target_ref = 6000 + id; - thread.vfs_ref = 7000 + id; - thread.command_ref = 8000 + id; -} - fn value_string_pairs(value: Option<&Value>) -> Vec<(String, String)> { value .and_then(Value::as_array) @@ -743,12 +642,7 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId { ProcessId::new(format!("vp-{suffix}")) } -fn coordinator_main_thread( - entry: &str, - task: TaskInstanceId, - task_definition: TaskDefinitionId, -) -> VirtualThread { - let name = format!("{entry} coordinator main ({task})"); +fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread { VirtualThread { id: MAIN_THREAD, frame_id: 1_001, @@ -760,10 +654,10 @@ fn coordinator_main_thread( target_ref: 4_501, vfs_ref: 5_001, command_ref: 5_501, - task, + task: TaskInstanceId::from(task), attempt_id: "main:unknown".to_owned(), - task_definition, - name, + task_definition: TaskDefinitionId::from(task_definition), + name: format!("{entry} coordinator main ({task})"), line: 0, state: DebugRuntimeState::Running, freeze_supported: true, @@ -799,14 +693,10 @@ fn coordinator_threads_from_events( .expect("launch thread model should include main"); if let Some(status) = process_status { if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) { - if let Ok(task) = TaskInstanceId::try_new(task) { - main.task = task; - } + main.task = TaskInstanceId::from(task); } if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) { - if let Ok(definition) = TaskDefinitionId::try_new(definition) { - main.task_definition = definition; - } + main.task_definition = TaskDefinitionId::from(definition); } main.name = format!("{entry} coordinator main ({})", main.task); main.state = if status @@ -868,12 +758,6 @@ fn coordinator_threads_from_snapshots( .get("main_task_definition") .and_then(Value::as_str) .unwrap_or(entry); - let (Ok(task), Ok(task_definition)) = ( - TaskInstanceId::try_new(task), - TaskDefinitionId::try_new(task_definition), - ) else { - return threads; - }; let mut main = coordinator_main_thread(entry, task, task_definition); main.attempt_id = format!( "main:{}", @@ -928,12 +812,6 @@ fn coordinator_threads_from_snapshots( let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else { continue; }; - let Ok(task_id) = TaskInstanceId::try_new(task) else { - continue; - }; - let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else { - continue; - }; let attempt_id = snapshot .get("attempt_id") .and_then(Value::as_str) @@ -984,9 +862,9 @@ fn coordinator_threads_from_snapshots( target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: task_id, + task: TaskInstanceId::from(task), attempt_id, - task_definition: task_definition_id, + task_definition: TaskDefinitionId::from(task_definition), name: format!("{display_name} ({task}, attempt {short_attempt})"), line: snapshot .get("source_line") @@ -1033,8 +911,6 @@ fn coordinator_threads_from_snapshots( fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option { let task = event.get("task").and_then(Value::as_str)?; let task_definition = event.get("task_definition").and_then(Value::as_str)?; - let task_id = TaskInstanceId::try_new(task).ok()?; - let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?; let definition_name = task_definition.replace(['_', '-'], " "); let name = if task == task_definition { definition_name @@ -1080,13 +956,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti target_ref: 6000 + id, vfs_ref: 7000 + id, command_ref: 8000 + id, - task: task_id, + task: TaskInstanceId::from(task), attempt_id: event .get("attempt_id") .and_then(Value::as_str) .unwrap_or("unknown-attempt") .to_owned(), - task_definition: task_definition_id, + task_definition: TaskDefinitionId::from(task_definition), name, line: 0, state, diff --git a/crates/clusterflux-node/src/assignment_runner.rs b/crates/clusterflux-node/src/assignment_runner.rs index 54f5a00..0e6ce6d 100644 --- a/crates/clusterflux-node/src/assignment_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Read; use std::path::PathBuf; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -46,40 +46,6 @@ use validation::{ resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot, }; -#[derive(Debug)] -struct AssignmentExecutionError { - message: String, - stdout_source_bytes: u64, - stderr_source_bytes: u64, -} - -impl std::fmt::Display for AssignmentExecutionError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(&self.message) - } -} - -impl std::error::Error for AssignmentExecutionError {} - -fn execution_error_with_log_bytes( - message: impl Into, - stdout_source_bytes: &AtomicU64, - stderr_source_bytes: &AtomicU64, -) -> Box { - Box::new(AssignmentExecutionError { - message: message.into(), - stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed), - stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed), - }) -} - -pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) { - error - .downcast_ref::() - .map(|error| (error.stdout_source_bytes, error.stderr_source_bytes)) - .unwrap_or((0, 0)) -} - pub(crate) fn run_verified_wasmtime_assignment( args: &Args, task: &RuntimeTask, @@ -129,8 +95,6 @@ pub(crate) fn run_verified_wasmtime_assignment( return Err("Wasm entrypoint assignment omitted its descriptor export".into()); } }; - let command_stdout_source_bytes = Arc::new(AtomicU64::new(0)); - let command_stderr_source_bytes = Arc::new(AtomicU64::new(0)); let (stdout, boundary_result) = match abi { WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => { let invocation = WasmTaskInvocation::new( @@ -138,28 +102,18 @@ pub(crate) fn run_verified_wasmtime_assignment( task_spec.task_instance.clone(), task_spec.args.clone(), ); - let result = WasmtimeTaskRuntime::new()? - .run_task_export_verified_with_task_host( + let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host( + &module, + expected_bundle_digest, + export, + &invocation, + Box::new(CoordinatorWasmTaskHost::new( + args, + task, + node_private_key, &module, - expected_bundle_digest, - export, - &invocation, - Box::new(CoordinatorWasmTaskHost::new( - args, - task, - node_private_key, - &module, - Arc::clone(&command_stdout_source_bytes), - Arc::clone(&command_stderr_source_bytes), - )?), - ) - .map_err(|error| { - execution_error_with_log_bytes( - error.to_string(), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })?; + )?), + )?; if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() { eprintln!( "clusterflux debug control: Wasm assignment returned for task {}", @@ -168,35 +122,17 @@ pub(crate) fn run_verified_wasmtime_assignment( } match result.outcome { WasmTaskOutcome::Completed => { - let boundary = result.result.ok_or_else(|| { - execution_error_with_log_bytes( - "completed Wasm task omitted result", - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })?; + let boundary = result.result.ok_or("completed Wasm task omitted result")?; ( - format!( - "{}\n", - serde_json::to_string(&boundary).map_err(|error| { - execution_error_with_log_bytes( - error.to_string(), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - ) - })? - ), + format!("{}\n", serde_json::to_string(&boundary)?), Some(boundary), ) } WasmTaskOutcome::Failed => { - return Err(execution_error_with_log_bytes( - result - .error - .unwrap_or_else(|| "Wasm task failed without an error".to_owned()), - &command_stdout_source_bytes, - &command_stderr_source_bytes, - )) + return Err(result + .error + .unwrap_or_else(|| "Wasm task failed without an error".to_owned()) + .into()) } } } @@ -204,18 +140,12 @@ pub(crate) fn run_verified_wasmtime_assignment( let task_id = TaskInstanceId::new(task.task.clone()); let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone())); let manifest = artifacts.flush(); - let stdout_source_bytes = command_stdout_source_bytes - .load(Ordering::Relaxed) - .saturating_add(stdout.len() as u64); - let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed); Ok(( CommandOutput { virtual_thread: task_id, status_code: Some(0), stdout, stderr: String::new(), - stdout_source_bytes, - stderr_source_bytes, stdout_truncated: false, stderr_truncated: false, log_backpressured: false, @@ -246,8 +176,6 @@ struct CoordinatorWasmTaskHost { next_handle_id: u64, handles: Arc>>, command_status: Arc>>, - command_stdout_source_bytes: Arc, - command_stderr_source_bytes: Arc, cancellation_requested: Arc, abort_requested: Arc, debug_control: Arc, @@ -260,8 +188,6 @@ impl CoordinatorWasmTaskHost { parent: &RuntimeTask, node_private_key: &str, module: &[u8], - command_stdout_source_bytes: Arc, - command_stderr_source_bytes: Arc, ) -> Result> { let task_spec = parent .task_spec @@ -342,8 +268,6 @@ impl CoordinatorWasmTaskHost { next_handle_id: 1, handles, command_status, - command_stdout_source_bytes, - command_stderr_source_bytes, cancellation_requested, abort_requested, debug_control, @@ -369,16 +293,7 @@ impl CoordinatorWasmTaskHost { runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?; let execution = run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key); - let ( - terminal_state, - status_code, - stdout, - stderr, - stdout_source_bytes, - stderr_source_bytes, - result, - retained, - ) = match execution { + let (terminal_state, status_code, stdout, stderr, result, retained) = match execution { Ok((output, _manifest, result)) => { let retained = retained_result_artifact( self.args.project_root.as_deref(), @@ -391,40 +306,20 @@ impl CoordinatorWasmTaskHost { output.status_code, output.stdout, output.stderr, - output.stdout_source_bytes, - output.stderr_source_bytes, result, retained, ), - Err(error) => ( - "failed", - Some(1), - String::new(), - error.clone(), - output.stdout_source_bytes, - output - .stderr_source_bytes - .saturating_add(error.len() as u64), - None, - None, - ), + Err(error) => ("failed", Some(1), String::new(), error, None, None), } } - Err(error) => { - let (stdout_source_bytes, stderr_source_bytes) = - assignment_error_log_bytes(error.as_ref()); - let error = error.to_string(); - ( - "failed", - Some(1), - String::new(), - error.clone(), - stdout_source_bytes, - stderr_source_bytes.saturating_add(error.len() as u64), - None, - None, - ) - } + Err(error) => ( + "failed", + Some(1), + String::new(), + error.to_string(), + None, + None, + ), }; let artifact_path = retained .as_ref() @@ -444,8 +339,8 @@ impl CoordinatorWasmTaskHost { "task": runtime_task.task, "terminal_state": terminal_state, "status_code": status_code, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": stdout.len(), + "stderr_bytes": stderr.len(), "stdout_tail": stdout, "stderr_tail": stderr, "stdout_truncated": false, @@ -781,7 +676,6 @@ impl WasmTaskHost for CoordinatorWasmTaskHost { let mut runner = CoordinatorControlledProcessRunner::new( self, Duration::from_millis(request.timeout_ms), - configured_secrets.clone(), ); let output = LinuxRootlessPodmanBackend .execute_local_checkout_task( diff --git a/crates/clusterflux-node/src/assignment_runner/process_runner.rs b/crates/clusterflux-node/src/assignment_runner/process_runner.rs index 0e2da69..2b6dbd3 100644 --- a/crates/clusterflux-node/src/assignment_runner/process_runner.rs +++ b/crates/clusterflux-node/src/assignment_runner/process_runner.rs @@ -1,83 +1,4 @@ use super::*; -use std::sync::mpsc::{self, Receiver, SyncSender}; - -struct LiveLogChunk { - stream: &'static str, - offset: u64, - source_bytes: u64, - bytes: Vec, - truncated: bool, -} - -fn append_bounded_tail(tail: &mut Vec, bytes: &[u8], maximum: usize) { - if maximum == 0 { - tail.clear(); - return; - } - if bytes.len() >= maximum { - tail.clear(); - tail.extend_from_slice(&bytes[bytes.len() - maximum..]); - return; - } - let overflow = tail - .len() - .saturating_add(bytes.len()) - .saturating_sub(maximum); - if overflow > 0 { - tail.drain(..overflow); - } - tail.extend_from_slice(bytes); -} - -fn redact_safe_live_log_prefix( - pending: &[u8], - configured_secrets: &[String], - final_chunk: bool, -) -> Option<(usize, String)> { - if pending.is_empty() { - return None; - } - let secret_bytes = configured_secrets - .iter() - .filter(|secret| secret.len() >= 4) - .map(String::as_bytes) - .collect::>(); - let maximum_secret_bytes = secret_bytes - .iter() - .map(|secret| secret.len()) - .max() - .unwrap_or(0); - if !final_chunk && pending.len() <= maximum_secret_bytes { - return None; - } - let mut consumed = if final_chunk { - pending.len() - } else { - pending.len() - maximum_secret_bytes - }; - loop { - let previous = consumed; - for secret in &secret_bytes { - for start in 0..=pending.len().saturating_sub(secret.len()) { - let end = start + secret.len(); - if start < consumed && end > consumed && &pending[start..end] == *secret { - consumed = end; - } - } - } - if consumed == previous { - break; - } - } - if consumed == 0 { - return None; - } - let text = redact_configured_values( - String::from_utf8_lossy(&pending[..consumed]).into_owned(), - configured_secrets, - ); - Some((consumed, text)) -} pub(super) struct CoordinatorControlledProcessRunner { pub(super) args: Args, @@ -86,20 +7,13 @@ pub(super) struct CoordinatorControlledProcessRunner { pub(super) node_private_key: String, pub(super) debug_control: Arc, pub(super) command_status: Arc>>, - pub(super) stdout_source_bytes: Arc, - pub(super) stderr_source_bytes: Arc, pub(super) timeout: Duration, - pub(super) configured_secrets: Vec, } impl CoordinatorControlledProcessRunner { const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1; - pub(super) fn new( - host: &CoordinatorWasmTaskHost, - timeout: Duration, - configured_secrets: Vec, - ) -> Self { + pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self { Self { args: host.args.clone(), process: host.process.clone(), @@ -107,10 +21,7 @@ impl CoordinatorControlledProcessRunner { node_private_key: host.node_private_key.clone(), debug_control: Arc::clone(&host.debug_control), command_status: Arc::clone(&host.command_status), - stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes), - stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes), timeout, - configured_secrets, } } @@ -290,18 +201,10 @@ impl CoordinatorControlledProcessRunner { fn drain_bounded( mut reader: impl Read + Send + 'static, maximum: usize, - stream: &'static str, - sender: SyncSender, - configured_secrets: Vec, - source_bytes_total: Arc, ) -> thread::JoinHandle, String>> { thread::spawn(move || { let mut captured = Vec::new(); let mut buffer = [0_u8; 16 * 1024]; - let stream_base = source_bytes_total.load(Ordering::Relaxed); - let mut source_bytes_read = 0_u64; - let mut pending_offset = stream_base; - let mut pending = Vec::new(); loop { let count = reader .read(&mut buffer) @@ -309,128 +212,12 @@ impl CoordinatorControlledProcessRunner { if count == 0 { break; } - let _ = source_bytes_total.fetch_update( - Ordering::Relaxed, - Ordering::Relaxed, - |current| Some(current.saturating_add(count as u64)), - ); - source_bytes_read = source_bytes_read.saturating_add(count as u64); - append_bounded_tail(&mut captured, &buffer[..count], maximum); - pending.extend_from_slice(&buffer[..count]); - if let Some((consumed, text)) = - redact_safe_live_log_prefix(&pending, &configured_secrets, false) - { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: pending_offset, - source_bytes: consumed as u64, - bytes: text.into_bytes(), - truncated: false, - }); - pending.drain(..consumed); - pending_offset = pending_offset.saturating_add(consumed as u64); - } - } - if let Some((consumed, text)) = - redact_safe_live_log_prefix(&pending, &configured_secrets, true) - { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: pending_offset, - source_bytes: consumed as u64, - bytes: text.into_bytes(), - truncated: false, - }); - } - if source_bytes_read > maximum as u64 { - let _ = sender.try_send(LiveLogChunk { - stream, - offset: stream_base.saturating_add(source_bytes_read), - source_bytes: 0, - bytes: b"[log output truncated at node capture limit]".to_vec(), - truncated: true, - }); + let remaining = maximum.saturating_sub(captured.len()); + captured.extend_from_slice(&buffer[..count.min(remaining)]); } Ok(captured) }) } - - fn spawn_live_log_reporter(&self, receiver: Receiver) -> thread::JoinHandle<()> { - let args = self.args.clone(); - let process = self.process.clone(); - let task = self.task.clone(); - let node_private_key = self.node_private_key.clone(); - let configured_secrets = self.configured_secrets.clone(); - let command_status = Arc::clone(&self.command_status); - thread::spawn(move || { - let mut log_session = None; - let mut delivery_available = true; - while let Ok(chunk) = receiver.recv() { - if !delivery_available { - continue; - } - let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned(); - text = redact_configured_values(text, &configured_secrets); - let mut delivered = false; - for _ in 0..2 { - if log_session.is_none() { - log_session = CoordinatorSession::connect_with_timeouts( - &args.coordinator, - Duration::from_millis(500), - Duration::from_millis(500), - ) - .ok(); - } - let Some(session) = log_session.as_mut() else { - continue; - }; - let request = signed_node_request_json( - &args, - &node_private_key, - "report_task_log_chunk", - serde_json::json!({ - "type": "report_task_log_chunk", - "tenant": &args.tenant, - "project": &args.project, - "process": &process, - "node": &args.node, - "task": &task, - "stream": chunk.stream, - "offset": chunk.offset, - "source_bytes": chunk.source_bytes, - "text": &text, - "truncated": chunk.truncated, - }), - ); - let result = match request { - Ok(request) => session - .request(request) - .map(|_| ()) - .map_err(|error| error.to_string()), - Err(error) => Err(error.to_string()), - }; - match result { - Ok(()) => { - delivered = true; - break; - } - Err(_) => { - log_session = None; - } - } - } - if !delivered { - delivery_available = false; - if let Ok(mut current) = command_status.lock() { - *current = Some( - "live log delivery was interrupted; final bounded output remains available" - .to_owned(), - ); - } - } - } - }) - } } impl ProcessRunner for CoordinatorControlledProcessRunner { @@ -458,17 +245,12 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { command.program, command.args.join(" ") )); - let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64); let stdout = Self::drain_bounded( child .stdout .take() .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, - "stdout", - live_log_sender.clone(), - self.configured_secrets.clone(), - Arc::clone(&self.stdout_source_bytes), ); let stderr = Self::drain_bounded( child @@ -476,19 +258,11 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .take() .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?, Self::MAX_CAPTURE_BYTES, - "stderr", - live_log_sender, - self.configured_secrets.clone(), - Arc::clone(&self.stderr_source_bytes), ); - let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver); let mut session = match CoordinatorSession::connect(&self.args.coordinator) { Ok(session) => session, Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); - let _ = stdout.join(); - let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "establish execution control channel: {error}" ))); @@ -503,9 +277,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Ok(None) => {} Err(error) => { Self::terminate_execution(&mut child, podman_container.as_deref()); - let _ = stdout.join(); - let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(error.to_string())); } } @@ -518,7 +289,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Command(format!( "native command exceeded wall-clock timeout of {} ms", self.timeout.as_millis() @@ -533,7 +303,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(BackendError::Cancelled( "coordinator requested cancellation or abort".to_owned(), )); @@ -543,7 +312,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { Self::terminate_execution(&mut child, podman_container.as_deref()); let _ = stdout.join(); let _ = stderr.join(); - let _ = live_log_reporter.join(); return Err(error); } } @@ -592,9 +360,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { .join() .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))? .map_err(BackendError::Command)?; - live_log_reporter - .join() - .map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?; self.set_command_status(format!( "native command exited with status {:?}", status.code() @@ -606,52 +371,3 @@ impl ProcessRunner for CoordinatorControlledProcessRunner { }) } } - -#[cfg(test)] -mod tests { - use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner}; - use std::io::Cursor; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{mpsc, Arc}; - - #[test] - fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() { - let secrets = vec!["correct-horse".to_owned()]; - let mut pending = b"prefix correct-".to_vec(); - let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap(); - pending.drain(..consumed); - pending.extend_from_slice(b"horse suffix"); - let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap(); - - let combined = format!("{first}{second}"); - assert_eq!(combined, "prefix [REDACTED] suffix"); - assert!(!combined.contains("correct-")); - assert!(!combined.contains("horse")); - } - - #[test] - fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() { - let source_bytes = Arc::new(AtomicU64::new(5)); - let (sender, receiver) = mpsc::sync_channel(8); - let reader = CoordinatorControlledProcessRunner::drain_bounded( - Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()), - 8, - "stdout", - sender, - Vec::new(), - Arc::clone(&source_bytes), - ); - let captured = reader.join().unwrap().unwrap(); - let chunks = receiver.into_iter().collect::>(); - - assert_eq!(captured, b"opqrstuv"); - assert_eq!(source_bytes.load(Ordering::Relaxed), 37); - assert_eq!( - chunks.iter().map(|chunk| chunk.source_bytes).sum::(), - 32 - ); - assert_eq!(chunks[0].offset, 5); - assert_eq!(chunks.last().unwrap().offset, 37); - assert!(chunks.iter().any(|chunk| chunk.truncated)); - } -} diff --git a/crates/clusterflux-node/src/assignment_runner/tests.rs b/crates/clusterflux-node/src/assignment_runner/tests.rs index d654d1f..4d2d864 100644 --- a/crates/clusterflux-node/src/assignment_runner/tests.rs +++ b/crates/clusterflux-node/src/assignment_runner/tests.rs @@ -48,10 +48,7 @@ fn test_controlled_runner( ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), - stdout_source_bytes: Arc::new(AtomicU64::new(0)), - stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout, - configured_secrets: Vec::new(), } } @@ -92,10 +89,7 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() { ), debug_control: Arc::new(WasmDebugControl::default()), command_status: Arc::new(Mutex::new(None)), - stdout_source_bytes: Arc::new(AtomicU64::new(0)), - stderr_source_bytes: Arc::new(AtomicU64::new(0)), timeout: Duration::from_secs(30), - configured_secrets: Vec::new(), }; let started = Instant::now(); let error = runner diff --git a/crates/clusterflux-node/src/command_runner.rs b/crates/clusterflux-node/src/command_runner.rs index 790b838..a1ebd33 100644 --- a/crates/clusterflux-node/src/command_runner.rs +++ b/crates/clusterflux-node/src/command_runner.rs @@ -1,6 +1,6 @@ use clusterflux_core::{ - CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay, - VfsPath, + CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, + VfsOverlay, VfsPath, }; use serde::{Deserialize, Serialize}; @@ -42,8 +42,6 @@ pub struct CommandOutput { pub status_code: Option, pub stdout: String, pub stderr: String, - pub stdout_source_bytes: u64, - pub stderr_source_bytes: u64, pub stdout_truncated: bool, pub stderr_truncated: bool, pub log_backpressured: bool, @@ -85,8 +83,6 @@ impl LocalCommandExecutor { &output.stderr, max_log_bytes, ); - let stdout_source_bytes = output.stdout.len() as u64; - let stderr_source_bytes = output.stderr.len() as u64; let staged_artifact = if let Some(path) = command.stage_stdout_as { Some(overlay.write( path, @@ -102,8 +98,6 @@ impl LocalCommandExecutor { status_code: output.status.code(), stdout: logs.stdout, stderr: logs.stderr, - stdout_source_bytes, - stderr_source_bytes, stdout_truncated: logs.stdout_truncated, stderr_truncated: logs.stderr_truncated, log_backpressured: logs.backpressured, @@ -113,20 +107,24 @@ impl LocalCommandExecutor { } pub(super) fn capture_command_logs( - _task: &TaskInstanceId, + task: &TaskInstanceId, stdout: &[u8], stderr: &[u8], max_log_bytes: usize, ) -> CapturedCommandLogs { - let stdout_truncated = stdout.len() > max_log_bytes; - let stderr_truncated = stderr.len() > max_log_bytes; - let stdout_start = stdout.len().saturating_sub(max_log_bytes); - let stderr_start = stderr.len().saturating_sub(max_log_bytes); + let mut logs = LogBuffer::new(max_log_bytes); + logs.push(task.clone(), stdout); + logs.push(task.clone(), stderr); + let records = logs.records(); + let stdout_record = &records[0]; + let stderr_record = &records[1]; + debug_assert_eq!(&stdout_record.task, task); + debug_assert_eq!(&stderr_record.task, task); CapturedCommandLogs { - stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(), - stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(), - stdout_truncated, - stderr_truncated, - backpressured: stdout_truncated || stderr_truncated, + stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(), + stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(), + stdout_truncated: stdout_record.truncated, + stderr_truncated: stderr_record.truncated, + backpressured: logs.backpressured(), } } diff --git a/crates/clusterflux-node/src/coordinator_session.rs b/crates/clusterflux-node/src/coordinator_session.rs index 13e64ec..997e0f0 100644 --- a/crates/clusterflux-node/src/coordinator_session.rs +++ b/crates/clusterflux-node/src/coordinator_session.rs @@ -3,7 +3,6 @@ use clusterflux_control::endpoint_identity; use clusterflux_control::ControlSession; use clusterflux_core::coordinator_wire_request; use serde_json::Value; -use std::time::Duration; pub(crate) struct CoordinatorSession { inner: ControlSession, @@ -16,16 +15,6 @@ impl CoordinatorSession { }) } - pub(crate) fn connect_with_timeouts( - addr: &str, - connect_timeout: Duration, - io_timeout: Duration, - ) -> Result> { - Ok(Self { - inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?, - }) - } - pub(crate) fn request(&mut self, value: Value) -> Result> { let request_id = format!("node-{}", self.inner.requests() + 1); let wire_request = coordinator_wire_request(request_id, value); diff --git a/crates/clusterflux-node/src/daemon.rs b/crates/clusterflux-node/src/daemon.rs index b223ebc..5a6ee0a 100644 --- a/crates/clusterflux-node/src/daemon.rs +++ b/crates/clusterflux-node/src/daemon.rs @@ -7,11 +7,11 @@ use std::time::{Duration, Instant}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_core::{ sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId, - ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS, + TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS, }; use serde_json::{json, Value}; -use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment}; +use crate::assignment_runner::run_verified_wasmtime_assignment; #[cfg(test)] use crate::coordinator_session::control_endpoint_identity; use crate::coordinator_session::CoordinatorSession; @@ -84,8 +84,6 @@ pub(crate) fn run() -> Result<(), Box> { let registration = establish_node_identity(&mut session, &args, &node_private_key)?; let heartbeat_request = json!({ "type": "node_heartbeat", - "tenant": &args.tenant, - "project": &args.project, "node": &args.node, }); let heartbeat_signature = sign_node_request( @@ -224,7 +222,7 @@ fn worker_loop( let expiry = Instant::now() .checked_add(Duration::from_secs(retention_limits.restart_pin_seconds)) .unwrap_or_else(Instant::now); - restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry); + restart_pins.insert(ArtifactId::new(artifact), expiry); } println!("{}", serde_json::to_string(&report)?); std::io::stdout().flush()?; @@ -291,7 +289,7 @@ fn service_pending_artifact_transfer( return Ok(false); }; let transfer_id = required_string(transfer, "transfer_id")?; - let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?; + let artifact = ArtifactId::new(required_string(transfer, "artifact")?); let expected_digest: Digest = serde_json::from_value( transfer .get("expected_digest") @@ -371,11 +369,9 @@ pub(crate) fn runtime_task_from_assignment( .cloned() .ok_or("task assignment missing task_spec")?, )?; - let process = ProcessId::try_new(required_string(value, "process")?)?; - let task = TaskInstanceId::try_new(required_string(value, "task")?)?; Ok(RuntimeTask { - process: process.to_string(), - task: task.to_string(), + process: required_string(value, "process")?, + task: required_string(value, "task")?, epoch: value.get("epoch").and_then(Value::as_u64), bundle_digest: task_spec.bundle_digest.clone(), task_spec: Some(task_spec), @@ -422,8 +418,6 @@ fn run_runtime_task( "reconnect_node", json!({ "type": "reconnect_node", - "tenant": &args.tenant, - "project": &args.project, "node": &args.node, "process": &task.process, "epoch": epoch, @@ -464,8 +458,6 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, - 0, - 0, ); } @@ -500,13 +492,9 @@ fn run_runtime_task( debug_command, node_private_key, &error, - output.stdout_source_bytes, - output.stderr_source_bytes, ), }, Err(error) => { - let (stdout_source_bytes, stderr_source_bytes) = - assignment_error_log_bytes(error.as_ref()); let error = error.to_string(); if error.contains("task execution cancelled:") { record_cancelled_task( @@ -518,8 +506,6 @@ fn run_runtime_task( capability_report, debug_command, node_private_key, - stdout_source_bytes, - stderr_source_bytes, ) } else { record_failed_task( @@ -532,8 +518,6 @@ fn run_runtime_task( debug_command, node_private_key, &error, - stdout_source_bytes, - stderr_source_bytes, ) } } @@ -586,12 +570,6 @@ fn parse_args() -> Result> { } } - TenantId::try_new(tenant.clone())?; - ProjectId::try_new(project.clone())?; - NodeId::try_new(node.clone())?; - if let Some(grant) = enrollment_grant.as_ref() { - clusterflux_core::validate_opaque_token(grant, 512)?; - } Ok(Args { coordinator: coordinator.ok_or("--coordinator is required")?, tenant, @@ -644,12 +622,13 @@ mod tests { #[test] fn hosted_url_remains_an_https_control_endpoint() { assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( - control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(), - "https://clusterflux.lesstuff.com/api/v1/control" + control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control") + .unwrap(), + "https://clusterflux.michelpaulissen.com/api/v1/control" ); assert_eq!( control_endpoint_identity("127.0.0.1:7999").unwrap(), diff --git a/crates/clusterflux-node/src/lib.rs b/crates/clusterflux-node/src/lib.rs index d9db699..468b353 100644 --- a/crates/clusterflux-node/src/lib.rs +++ b/crates/clusterflux-node/src/lib.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use clusterflux_core::{ Capability, CommandBackendKind, CommandInvocation, CommandPlan, Digest, EnvironmentKind, @@ -9,7 +9,6 @@ use sha2::{Digest as ShaDigest, Sha256}; use thiserror::Error; mod command_runner; -mod source_snapshot; mod windows_dev; pub use clusterflux_wasm_runtime::{ WasmDebugControl, WasmTaskError, WasmTaskHost, WasmtimeDebugProbe, WasmtimeTaskRuntime, @@ -21,10 +20,6 @@ pub use command_runner::{ }; pub use windows_dev::{WindowsCommandDevBackend, WindowsSandboxStubBackend}; -pub fn snapshot_project_digest(project_root: &Path) -> Result { - source_snapshot::snapshot_project(project_root).map(|snapshot| snapshot.digest) -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct MaterializedEnvironment { pub name: String, @@ -741,7 +736,7 @@ mod tests { } #[test] - fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() { + fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() { let invocation = CommandInvocation { program: "cargo".to_owned(), args: vec!["build".to_owned()], @@ -774,7 +769,7 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "cdef"); + assert_eq!(output.stdout, "abcd"); assert!(output.stdout_truncated); assert!(output.log_backpressured); assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6); @@ -992,7 +987,7 @@ mod tests { #[cfg(unix)] #[test] - fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() { + fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() { let executor = LocalCommandExecutor { node: clusterflux_core::NodeId::from("node"), hosted_control_plane: false, @@ -1021,15 +1016,15 @@ mod tests { .unwrap(); assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux")); - assert_eq!(output.stdout, "cdef"); - assert_eq!(output.stderr, "err"); + assert_eq!(output.stdout, "abcd"); + assert_eq!(output.stderr, ""); assert!(output.stdout_truncated); - assert!(!output.stderr_truncated); + assert!(output.stderr_truncated); assert!(output.log_backpressured); } #[test] - fn public_node_crate_does_not_require_hosted_service_types() { + fn public_node_crate_does_not_require_hosted_private_types() { let _tenant = TenantId::from("tenant"); let _project = ProjectId::from("project"); let _backend = LinuxRootlessPodmanBackend; diff --git a/crates/clusterflux-node/src/main.rs b/crates/clusterflux-node/src/main.rs index 2ada582..8f2960b 100644 --- a/crates/clusterflux-node/src/main.rs +++ b/crates/clusterflux-node/src/main.rs @@ -8,34 +8,5 @@ mod task_artifacts; mod task_reports; fn main() -> Result<(), Box> { - let raw_args = std::env::args().skip(1).collect::>(); - match raw_args.as_slice() { - [flag] if matches!(flag.as_str(), "--version" | "-V") => { - println!("clusterflux-node {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - [flag] if matches!(flag.as_str(), "--help" | "-h") => { - println!( - "Clusterflux node worker.\n\n\ - Usage: clusterflux-node --coordinator [OPTIONS]\n\n\ - Options:\n \ - --coordinator \n \ - --tenant [default: tenant]\n \ - --project-id [default: project]\n \ - --node [default: node]\n \ - --project-root \n \ - --enrollment-grant \n \ - --public-key \n \ - --worker\n \ - --emit-ready\n \ - --control-poll-ms \n \ - --assignment-poll-ms [default: 500]\n \ - -h, --help\n \ - -V, --version" - ); - return Ok(()); - } - _ => {} - } daemon::run() } diff --git a/crates/clusterflux-node/src/task_reports.rs b/crates/clusterflux-node/src/task_reports.rs index afaf0c8..a700e76 100644 --- a/crates/clusterflux-node/src/task_reports.rs +++ b/crates/clusterflux-node/src/task_reports.rs @@ -47,8 +47,8 @@ pub(crate) fn record_completed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -85,8 +85,8 @@ pub(crate) fn record_completed_task( "node": &args.node, "task": &task.task, "status_code": output.status_code, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -123,11 +123,8 @@ pub(crate) fn record_failed_task( debug_command: Value, node_private_key: &str, error: &str, - stdout_source_bytes: u64, - command_stderr_source_bytes: u64, ) -> Result> { let error = bounded_runtime_error(error); - let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64); let log_event = session.request(signed_node_request_json( args, node_private_key, @@ -139,8 +136,8 @@ pub(crate) fn record_failed_task( "process": &task.process, "node": &args.node, "task": &task.task, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -178,8 +175,8 @@ pub(crate) fn record_failed_task( "task": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, @@ -192,8 +189,6 @@ pub(crate) fn record_failed_task( Ok(failed_node_report( task, &error, - stdout_source_bytes, - stderr_source_bytes, registration, heartbeat, capability_report, @@ -215,8 +210,6 @@ pub(crate) fn record_cancelled_task( capability_report: Value, debug_command: Value, node_private_key: &str, - stdout_source_bytes: u64, - stderr_source_bytes: u64, ) -> Result> { let recorded = session.request(signed_node_request_json( args, @@ -231,12 +224,12 @@ pub(crate) fn record_cancelled_task( "task": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": stdout_source_bytes > 0, - "stderr_truncated": stderr_source_bytes > 0, + "stdout_truncated": false, + "stderr_truncated": false, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, @@ -245,8 +238,6 @@ pub(crate) fn record_cancelled_task( )?)?; Ok(cancelled_node_report( task, - stdout_source_bytes, - stderr_source_bytes, registration, heartbeat, capability_report, @@ -290,8 +281,8 @@ pub(crate) fn completed_node_report( "virtual_thread": output.virtual_thread, "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" }, "status_code": output.status_code, - "stdout_bytes": output.stdout_source_bytes, - "stderr_bytes": output.stderr_source_bytes, + "stdout_bytes": output.stdout.len(), + "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, @@ -314,8 +305,6 @@ pub(crate) fn completed_node_report( #[allow(clippy::too_many_arguments)] pub(crate) fn cancelled_node_report( task: &RuntimeTask, - stdout_source_bytes: u64, - stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -329,12 +318,12 @@ pub(crate) fn cancelled_node_report( "virtual_thread": &task.task, "terminal_state": "cancelled", "status_code": null, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", - "stdout_truncated": stdout_source_bytes > 0, - "stderr_truncated": stderr_source_bytes > 0, + "stdout_truncated": false, + "stderr_truncated": false, "log_backpressured": false, "staged_artifact": null, "large_bytes_uploaded": false, @@ -354,8 +343,6 @@ pub(crate) fn cancelled_node_report( pub(crate) fn failed_node_report( task: &RuntimeTask, error: &str, - stdout_source_bytes: u64, - stderr_source_bytes: u64, registration_response: Value, heartbeat_response: Value, capability_response: Value, @@ -370,8 +357,8 @@ pub(crate) fn failed_node_report( "virtual_thread": &task.task, "terminal_state": "failed", "status_code": -1, - "stdout_bytes": stdout_source_bytes, - "stderr_bytes": stderr_source_bytes, + "stdout_bytes": 0, + "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": error, "stdout_truncated": false, @@ -390,40 +377,3 @@ pub(crate) fn failed_node_report( "coordinator_response": coordinator_response, }) } - -#[cfg(test)] -mod tests { - use super::*; - use clusterflux_core::TaskInstanceId; - - #[test] - fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() { - let report = completed_node_report( - CommandOutput { - virtual_thread: TaskInstanceId::from("task"), - status_code: Some(0), - stdout: "bounded tail".to_owned(), - stderr: String::new(), - stdout_source_bytes: 519, - stderr_source_bytes: 0, - stdout_truncated: true, - stderr_truncated: false, - log_backpressured: false, - staged_artifact: None, - }, - false, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - Value::Null, - 1, - ); - - assert_eq!(report["stdout_bytes"], 519); - assert_eq!(report["stdout_tail"], "bounded tail"); - } -} diff --git a/crates/clusterflux-sdk/src/lib.rs b/crates/clusterflux-sdk/src/lib.rs index 9b887d6..7aabad9 100644 --- a/crates/clusterflux-sdk/src/lib.rs +++ b/crates/clusterflux-sdk/src/lib.rs @@ -1,5 +1,3 @@ -extern crate self as clusterflux; - use serde::Serialize; pub use clusterflux_core as core; @@ -12,26 +10,6 @@ pub use task_args::{ TaskArgBudget, TaskArgError, TaskArgKind, TaskArgValidation, Vfs, }; -pub use clusterflux_core::TaskFailurePolicy; -pub use sdk_runtime::TaskRuntimeError as Error; -pub type Result = std::result::Result; - -pub mod prelude { - pub use crate::{ - command::Command, fs, source, Artifact, Result, SourceSnapshot, TaskFailurePolicy, - }; -} - -#[macro_export] -macro_rules! spawn { - ($task:ident()) => { - $crate::spawn::__product_async_task($task, stringify!($task)) - }; - ($task:ident($argument:expr)) => { - $crate::spawn::__product_async_task_with_arg($argument, $task, stringify!($task)) - }; -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] pub struct EntrypointDescriptor { pub name: &'static str, @@ -85,8 +63,6 @@ impl RegisteredProgram { pub mod __private; pub mod spawn { - use std::future::{Future, IntoFuture}; - use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, OnceLock}; @@ -139,273 +115,6 @@ pub mod spawn { runtime_threads().lock().unwrap().clone() } - #[doc(hidden)] - pub fn __product_async_task( - entry: F, - task_id: &'static str, - ) -> ProductAsyncTaskBuilder - where - F: FnOnce() -> Fut, - Fut: Future>, - R: TaskArg, - { - ProductAsyncTaskBuilder { - entry: Some(entry), - env: None, - name: task_id, - task_id, - failure_policy: TaskFailurePolicy::FailFast, - marker: std::marker::PhantomData, - } - } - - #[doc(hidden)] - pub fn __product_async_task_with_arg( - arg: A, - entry: F, - task_id: &'static str, - ) -> ProductAsyncTaskWithArgBuilder - where - A: TaskArg, - F: FnOnce(A) -> Fut, - Fut: Future>, - R: TaskArg, - { - ProductAsyncTaskWithArgBuilder { - arg: Some(arg), - entry: Some(entry), - env: None, - name: task_id, - task_id, - arg_budget: TaskArgBudget::default(), - failure_policy: TaskFailurePolicy::FailFast, - marker: std::marker::PhantomData, - } - } - - pub struct ProductAsyncTaskBuilder - where - F: FnOnce() -> Fut, - Fut: Future>, - R: TaskArg, - { - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - entry: Option, - env: Option, - name: &'static str, - task_id: &'static str, - failure_policy: TaskFailurePolicy, - marker: std::marker::PhantomData (Fut, R)>, - } - - impl ProductAsyncTaskBuilder - where - F: FnOnce() -> Fut, - Fut: Future>, - R: TaskArg + DeserializeOwned, - { - pub fn on(mut self, env: EnvRef) -> Self { - self.env = Some(env); - self - } - - pub fn name(mut self, name: &'static str) -> Self { - self.name = name; - self - } - - pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self { - self.failure_policy = failure_policy; - self - } - - pub async fn start(self) -> crate::Result> { - let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst); - let runtime_event = register_runtime_thread(id, self.name, self.env); - #[cfg(target_arch = "wasm32")] - { - let remote = start_guest_host_task( - clusterflux_core::TaskDefinitionId::from(self.task_id), - self.env, - Vec::new(), - self.failure_policy, - )?; - return Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: None, - remote: Some(remote), - }); - } - #[cfg(not(target_arch = "wasm32"))] - { - if let Some(config) = ProductRuntimeConfig::from_env()? { - let remote = start_remote_task( - config, - self.task_id.to_owned(), - self.env, - Vec::new(), - self.failure_policy, - )?; - return Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: None, - remote: Some(remote), - }); - } - let result = self.entry.expect("task entry used once")().await?; - Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: Some(result), - remote: None, - }) - } - } - } - - impl IntoFuture for ProductAsyncTaskBuilder - where - F: FnOnce() -> Fut + 'static, - Fut: Future> + 'static, - R: TaskArg + DeserializeOwned + 'static, - { - type Output = crate::Result>; - type IntoFuture = Pin>>; - - fn into_future(self) -> Self::IntoFuture { - Box::pin(self.start()) - } - } - - pub struct ProductAsyncTaskWithArgBuilder - where - A: TaskArg, - F: FnOnce(A) -> Fut, - Fut: Future>, - R: TaskArg, - { - arg: Option, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] - entry: Option, - env: Option, - name: &'static str, - task_id: &'static str, - arg_budget: TaskArgBudget, - failure_policy: TaskFailurePolicy, - marker: std::marker::PhantomData (Fut, R)>, - } - - impl ProductAsyncTaskWithArgBuilder - where - A: TaskArg, - F: FnOnce(A) -> Fut, - Fut: Future>, - R: TaskArg + DeserializeOwned, - { - pub fn on(mut self, env: EnvRef) -> Self { - self.env = Some(env); - self - } - - pub fn name(mut self, name: &'static str) -> Self { - self.name = name; - self - } - - pub fn arg_budget(mut self, arg_budget: TaskArgBudget) -> Self { - self.arg_budget = arg_budget; - self - } - - pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self { - self.failure_policy = failure_policy; - self - } - - pub async fn start(self) -> crate::Result> { - let arg = self.arg.expect("task argument used once"); - validate_task_arg(&arg, self.arg_budget) - .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?; - let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst); - let runtime_event = register_runtime_thread(id, self.name, self.env); - #[cfg(target_arch = "wasm32")] - { - let boundary = arg - .task_boundary_value() - .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?; - let remote = start_guest_host_task( - clusterflux_core::TaskDefinitionId::from(self.task_id), - self.env, - vec![boundary], - self.failure_policy, - )?; - return Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: None, - remote: Some(remote), - }); - } - #[cfg(not(target_arch = "wasm32"))] - { - if let Some(config) = ProductRuntimeConfig::from_env()? { - let boundary = arg - .task_boundary_value() - .map_err(|error| TaskRuntimeError::Argument(error.to_string()))?; - let remote = start_remote_task( - config, - self.task_id.to_owned(), - self.env, - vec![boundary], - self.failure_policy, - )?; - return Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: None, - remote: Some(remote), - }); - } - let result = self.entry.expect("task entry used once")(arg).await?; - Ok(TaskHandle { - virtual_thread_id: id, - name: self.name, - env: self.env, - debugger_visible: runtime_event.debugger_visible, - result: Some(result), - remote: None, - }) - } - } - } - - impl IntoFuture for ProductAsyncTaskWithArgBuilder - where - A: TaskArg + 'static, - F: FnOnce(A) -> Fut + 'static, - Fut: Future> + 'static, - R: TaskArg + DeserializeOwned + 'static, - { - type Output = crate::Result>; - type IntoFuture = Pin>>; - - fn into_future(self) -> Self::IntoFuture { - Box::pin(self.start()) - } - } - pub fn task(entry: F) -> TaskBuilder where F: FnOnce() -> R, @@ -1002,10 +711,6 @@ pub mod command { self } - pub fn cwd(self, directory: impl Into) -> Self { - self.current_dir(directory) - } - pub fn env(mut self, name: impl Into, value: impl Into) -> Self { self.environment_variables.insert(name.into(), value.into()); self @@ -1056,34 +761,6 @@ pub mod command { .to_owned(), )) } - - pub async fn run(self) -> Result { - let program = self.program.clone(); - let output = self.output().await?; - if output.status_code == Some(0) { - return Ok(output); - } - Err(TaskRuntimeError::CommandFailed { - program, - status_code: output.status_code, - stdout: bounded_tail(output.stdout), - stderr: bounded_tail(output.stderr), - stdout_truncated: output.stdout_truncated, - stderr_truncated: output.stderr_truncated, - }) - } - } - - fn bounded_tail(value: String) -> String { - const LIMIT: usize = 4 * 1024; - if value.len() <= LIMIT { - return value; - } - let mut start = value.len() - LIMIT; - while !value.is_char_boundary(start) { - start += 1; - } - value[start..].to_owned() } } @@ -1105,34 +782,6 @@ pub mod source { "source snapshots are produced only by a source-capable Clusterflux node".to_owned(), )) } - - pub struct CurrentProject; - - pub fn current_project() -> CurrentProject { - CurrentProject - } - - impl CurrentProject { - pub async fn snapshot(self) -> Result { - crate::spawn::__product_async_task(snapshot_current_project, "snapshot_current_project") - .await? - .join() - .await - } - } - - #[clusterflux::task(capabilities = "source_filesystem")] - async fn snapshot_current_project() -> crate::Result { - snapshot().await - } -} - -impl SourceSnapshot { - pub fn mount(&self) -> Result<&'static str> { - crate::task_args::parse_handle_digest(&self.digest) - .map_err(|error| Error::Argument(error.to_string()))?; - Ok("/workspace") - } } pub mod fs { @@ -1165,10 +814,6 @@ pub mod fs { }) } - pub fn output(relative: impl Into) -> Result { - output_path(relative) - } - pub async fn flush(path: &OutputPath) -> Result { #[cfg(target_arch = "wasm32")] { @@ -1193,10 +838,6 @@ pub mod fs { } } - pub async fn publish(path: &OutputPath) -> Result { - flush(path).await - } - pub async fn materialize( artifact: &Artifact, relative: impl Into, diff --git a/crates/clusterflux-sdk/src/sdk_runtime.rs b/crates/clusterflux-sdk/src/sdk_runtime.rs index 7724988..fddf18c 100644 --- a/crates/clusterflux-sdk/src/sdk_runtime.rs +++ b/crates/clusterflux-sdk/src/sdk_runtime.rs @@ -106,14 +106,6 @@ pub enum TaskRuntimeError { waited: Duration, }, ResultDecode(String), - CommandFailed { - program: String, - status_code: Option, - stdout: String, - stderr: String, - stdout_truncated: bool, - stderr_truncated: bool, - }, } impl std::fmt::Display for TaskRuntimeError { @@ -132,21 +124,6 @@ impl std::fmt::Display for TaskRuntimeError { ); } Self::ResultDecode(message) => ("remote result", message), - Self::CommandFailed { - program, - status_code, - stdout, - stderr, - stdout_truncated, - stderr_truncated, - } => { - return write!( - formatter, - "command `{program}` failed with status {status_code:?}; stdout{}: {stdout:?}; stderr{}: {stderr:?}", - if *stdout_truncated { " (truncated)" } else { "" }, - if *stderr_truncated { " (truncated)" } else { "" }, - ); - } }; write!(formatter, "{kind} error: {message}") } diff --git a/crates/clusterflux-sdk/tests/product_api.rs b/crates/clusterflux-sdk/tests/product_api.rs deleted file mode 100644 index 72f5a5b..0000000 --- a/crates/clusterflux-sdk/tests/product_api.rs +++ /dev/null @@ -1,37 +0,0 @@ -#[clusterflux::task] -async fn plus_one(value: i32) -> clusterflux::Result { - Ok(value + 1) -} - -#[test] -fn unified_spawn_defaults_to_function_identity_and_unwraps_task_result() { - futures_executor::block_on(async { - let handle = clusterflux::spawn!(plus_one(41)) - .on(clusterflux::env!("linux")) - .await - .unwrap(); - assert_eq!(handle.name(), "plus_one"); - assert_eq!(handle.env().unwrap().name, "linux"); - assert_eq!(handle.join().await.unwrap(), 42); - }); -} - -#[test] -fn source_mount_and_command_failure_are_public_typed_contracts() { - let source = clusterflux::SourceSnapshot { - digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - .to_owned(), - }; - assert_eq!(source.mount().unwrap(), "/workspace"); - - let error = clusterflux::Error::CommandFailed { - program: "cc".to_owned(), - status_code: Some(2), - stdout: "out".to_owned(), - stderr: "bad input".to_owned(), - stdout_truncated: false, - stderr_truncated: false, - }; - assert!(error.to_string().contains("command")); - assert!(error.to_string().contains("cc")); -} diff --git a/docs/artifacts.md b/docs/artifacts.md index a1521a7..9320564 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context. The reverse stream counts framing, base64 expansion, failed bytes, and abandoned transfers. The CLI verifies the final digest. Hosted policy may impose per-project, per-tenant, and per-account size, concurrency, and period limits. -There is no hosted global circuit breaker, and one tenant's period usage does -not consume a shared customer byte quota. +Operators may disable relay traffic as an emergency safety control; one tenant's +period usage does not consume a shared customer byte quota. diff --git a/docs/debugging.md b/docs/debugging.md index 3c0c99f..4558251 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -1,22 +1,14 @@ # Debugging -The DAP runtime observer keeps one coordinator session open and reconnects it -with a 100 ms to 5 second exponential backoff. Polling is responsive around -debug actions (100 ms) and backs off to one observation cycle every 5 seconds -while idle. A cycle makes at most four coordinator requests, so steady-state -idle traffic is bounded at 0.8 requests per second per debug session. - The VS Code extension uses the Debug Adapter Protocol and the coordinator's authoritative process, task, attempt, and Debug Epoch APIs. ## Launch Open your project and start "Clusterflux: Launch Virtual Process". The adapter -acknowledges launch configuration immediately, then builds and starts the -selected entrypoint in the background. You can keep using the debug UI while a -long-running process is attached. The adapter replaces its thread view with -coordinator task snapshots as they arrive; product mode does not infer threads -from completion events or demo fixtures. +builds the bundle, launches the selected entrypoint, and replaces its thread +view with coordinator task snapshots. Product mode does not infer threads from +completion events or demo fixtures. Each task view includes its logical task, attempt ID, definition, state, node, environment, arguments, typed handles, command status, VFS checkpoint, probe @@ -27,12 +19,6 @@ source when known, and restart compatibility. Unknown source remains unknown. When a breakpoint is hit, the coordinator asks every active participant to freeze. A fully frozen epoch is a consistent all-participant view. -You do not need to configure a breakpoint before launch. Breakpoint changes are -sent to the running coordinator, and a stopped event is emitted only after an -executing Wasm probe reports a real hit. Pause, continue, inspection, and -disconnect requests remain responsive while the adapter observes the runtime in -the background. Attach does not fabricate an initial breakpoint. - If one or more participants cannot freeze within five seconds, the epoch becomes partially frozen when at least one participant did freeze. The adapter warns that running and frozen tasks do not form a consistent global snapshot. You may @@ -45,8 +31,7 @@ runtime did not report it. ## Continue and restart -Continue responds immediately, then resumes only participants that acknowledged -the freeze while runtime observation continues in the background. +Continue resumes only participants that acknowledged the freeze. Restarting a terminal task rebuilds the bundle and checks its clean entry checkpoint. A compatible restart creates a new attempt under the same logical diff --git a/docs/environments.md b/docs/environments.md index 1153bc2..7ceb3bc 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -1,9 +1,5 @@ # Environments -Clusterflux disables network access while a task executes. Materializing an -environment for the first time can still fetch declared inputs; subsequent task -execution uses the materialized environment with networking disabled. - Declare environments in the bundle under: ~~~text diff --git a/docs/getting-started.md b/docs/getting-started.md index be09923..6bcc84f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,9 +11,6 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap ~~~ -On NixOS or another system with Nix, the equivalent package is available with -`nix profile install .#clusterflux-tools`. - Install rootless Podman on each Linux node that will execute container-backed environments. @@ -55,7 +52,7 @@ same stored identity: ~~~bash clusterflux-node \ - --coordinator https://clusterflux.lesstuff.com \ + --coordinator https://clusterflux.michelpaulissen.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ @@ -84,21 +81,6 @@ clusterflux bundle inspect --project . clusterflux run --project . build ~~~ -From this checkout, the complete source-to-task-to-artifact path is: - -~~~bash -clusterflux bundle inspect --project examples/hello-build -clusterflux run --project examples/hello-build build -clusterflux artifact list --process -clusterflux artifact download --to ./hello-clusterflux -chmod +x ./hello-clusterflux -./hello-clusterflux -~~~ - -The workflow snapshots `examples/hello-build`, starts its `compile` task in the -network-disabled Linux execution environment, and retains the real static executable returned by -that task. - Choose another entrypoint by replacing `build`. Clusterflux rejects an oversized or invalid bundle before it creates the virtual process. @@ -119,23 +101,12 @@ Restart it as a new attempt under the same logical task identity: clusterflux task restart --process --yes ~~~ -`examples/recovery-build` demonstrates this without a synthetic trap. It starts -two instances of `build_lane`; one completes while the other runs a command that -exits with status 23. Edit that command to produce the recovering output, then -restart the failed task. Its original join resolves from the replacement -attempt. - ## 6. Debug in VS Code Open the project in VS Code and start `Clusterflux: Launch Virtual Process`. Set a breakpoint on a generated probe location and use Threads, Stack, Variables, Continue, Pause, and Restart. -Breakpoints remain unverified until the coordinator installs them. Thread IDs -are stable for an exact logical task instance across snapshot updates and retry -attempts. Observer reconnect diagnostics do not manufacture stopped events, and -Continue succeeds only after the coordinator acknowledges resume. - A fully frozen Debug Epoch gives a consistent all-participant view. If a participant cannot freeze within five seconds, the adapter reports a partial epoch. You may inspect frozen participants, but values across running and frozen diff --git a/docs/nodes.md b/docs/nodes.md index 3913859..f6ffd19 100644 --- a/docs/nodes.md +++ b/docs/nodes.md @@ -23,7 +23,7 @@ key pair. ~~~bash clusterflux-node \ - --coordinator https://clusterflux.lesstuff.com \ + --coordinator https://clusterflux.michelpaulissen.com \ --tenant "$TENANT" \ --project-id \ --node workstation \ diff --git a/docs/task-abi.md b/docs/task-abi.md index db64c58..20f536f 100644 --- a/docs/task-abi.md +++ b/docs/task-abi.md @@ -9,13 +9,10 @@ capability-bearing values use typed handles. A structured boundary value carries both its JSON structure and the complete typed handle table required to materialize it. -Portable handles carry the minimum stable value needed to cross the task ABI, -such as an artifact ID, digest, and size. Tenant, project, process, producer, -path, retaining location, and authorization state are coordinator-side -metadata; they are not portable authority fields supplied by task code. The SDK -encodes handles as explicit placeholders and the runtime replaces only -placeholders that match the declared type and authoritative coordinator -metadata. +Supported handles include VFS and artifact identities with full tenant, project, +process, producer, digest, size, and path provenance. The SDK encodes handles as +explicit placeholders and the runtime replaces only placeholders that match the +declared type and metadata. A malformed, missing, extra, cross-scope, or type-mismatched handle fails closed. The runtime does not reconstruct authority from a string path or caller-supplied diff --git a/examples/hello-build/Cargo.toml b/examples/hello-build/Cargo.toml deleted file mode 100644 index e8aaee7..0000000 --- a/examples/hello-build/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "hello-build" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -[lib] -crate-type = ["rlib", "cdylib"] - -[dependencies] -clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" } diff --git a/examples/hello-build/README.md b/examples/hello-build/README.md deleted file mode 100644 index 3a50299..0000000 --- a/examples/hello-build/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Hello build - -The primary Clusterflux example snapshots this project, compiles a real static C -executable in the declared network-disabled Linux execution environment, and publishes the -executable as a retained artifact. - -Run it with: - -~~~bash -clusterflux bundle inspect --project examples/hello-build -clusterflux run --project examples/hello-build build -clusterflux artifact list --process -clusterflux artifact download --to ./hello-clusterflux -chmod +x ./hello-clusterflux -./hello-clusterflux -~~~ - -The final command prints hello from a real Clusterflux build. diff --git a/examples/hello-build/src/lib.rs b/examples/hello-build/src/lib.rs deleted file mode 100644 index 90ea98a..0000000 --- a/examples/hello-build/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -use clusterflux::prelude::*; - -#[clusterflux::task(capabilities = "command")] -pub async fn compile(source: SourceSnapshot) -> Result { - let executable = fs::output("hello-clusterflux")?; - Command::new("cc") - .args([ - "-Os", - "-static", - "-s", - "fixture/hello-clusterflux.c", - "-o", - executable.as_str(), - ]) - .cwd(source.mount()?) - .env("SOURCE_DATE_EPOCH", "0") - .network_disabled() - .run() - .await?; - fs::publish(&executable).await -} - -#[clusterflux::main] -pub async fn build() -> Result { - let source = source::current_project().snapshot().await?; - let compile = clusterflux::spawn!(compile(source)) - .on(clusterflux::env!("linux")) - .await?; - compile.join().await -} diff --git a/examples/recovery-build/Cargo.toml b/examples/launch-build-demo/Cargo.toml similarity index 67% rename from examples/recovery-build/Cargo.toml rename to examples/launch-build-demo/Cargo.toml index 6fdba2d..2f6e4c0 100644 --- a/examples/recovery-build/Cargo.toml +++ b/examples/launch-build-demo/Cargo.toml @@ -1,14 +1,18 @@ [package] -name = "recovery-build" +name = "launch-build-demo" version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true -publish = false [lib] +path = "src/build.rs" crate-type = ["rlib", "cdylib"] [dependencies] clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" } +futures-executor.workspace = true serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] diff --git a/examples/launch-build-demo/README.md b/examples/launch-build-demo/README.md new file mode 100644 index 0000000..33412dc --- /dev/null +++ b/examples/launch-build-demo/README.md @@ -0,0 +1,14 @@ +# Launch Build Demo + +This build workflow is expressed as Rust source code. It uses `env!("linux")`, +recognizes `env!("windows")` when a Windows development node is attached, +spawns debugger-visible virtual tasks, and returns portable artifact and source +handles instead of moving large bytes through task arguments. + +The Linux environment is defined by `envs/linux/Containerfile`. The Windows environment is a user-attached development contract and does not claim secure managed Windows sandboxing. + +Inspect the bundle metadata: + +```bash +clusterflux bundle inspect --project examples/launch-build-demo +``` diff --git a/examples/hello-build/envs/linux/Containerfile b/examples/launch-build-demo/envs/linux/Containerfile similarity index 100% rename from examples/hello-build/envs/linux/Containerfile rename to examples/launch-build-demo/envs/linux/Containerfile diff --git a/examples/launch-build-demo/envs/windows/Dockerfile b/examples/launch-build-demo/envs/windows/Dockerfile new file mode 100644 index 0000000..2978ba6 --- /dev/null +++ b/examples/launch-build-demo/envs/windows/Dockerfile @@ -0,0 +1,2 @@ +# User-attached Windows development execution contract. +# This is not a managed untrusted Windows sandbox. diff --git a/examples/hello-build/fixture/hello-clusterflux.c b/examples/launch-build-demo/fixture/hello-clusterflux.c similarity index 100% rename from examples/hello-build/fixture/hello-clusterflux.c rename to examples/launch-build-demo/fixture/hello-clusterflux.c diff --git a/examples/launch-build-demo/src/bin/sdk-product-runtime.rs b/examples/launch-build-demo/src/bin/sdk-product-runtime.rs new file mode 100644 index 0000000..018b9e0 --- /dev/null +++ b/examples/launch-build-demo/src/bin/sdk-product-runtime.rs @@ -0,0 +1,33 @@ +use std::error::Error; + +use futures_executor::block_on; + +fn main() -> Result<(), Box> { + let handle = block_on( + clusterflux::spawn::task_with_arg(41_i32, |_| -> i32 { + panic!("product-mode spawn invoked its local Rust closure") + }) + .task_id("task_add_one") + .name("SDK product Wasmtime task") + .env(clusterflux::env!("linux")) + .start(), + )?; + let task_spec = handle + .task_spec() + .cloned() + .ok_or("CLUSTERFLUX_SDK_COORDINATOR did not enable product runtime")?; + let virtual_thread_id = handle.virtual_thread_id(); + let result = block_on(handle.join())?; + + println!( + "{}", + serde_json::to_string(&serde_json::json!({ + "kind": "clusterflux-sdk-product-runtime", + "virtual_thread_id": virtual_thread_id, + "task_spec": task_spec, + "remote_result": result, + "local_function_invoked_by_spawn": false, + }))? + ); + Ok(()) +} diff --git a/examples/launch-build-demo/src/build.rs b/examples/launch-build-demo/src/build.rs new file mode 100644 index 0000000..cedfbd1 --- /dev/null +++ b/examples/launch-build-demo/src/build.rs @@ -0,0 +1,403 @@ +use clusterflux::{Artifact, EnvRef, SourceSnapshot}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] +pub struct BuildReport { + pub linux_thread: u64, + pub linux_parallel_thread: u64, + pub package_thread: u64, + pub linux_artifact: Artifact, + pub package_artifact: Artifact, + pub source: SourceSnapshot, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)] +pub struct PackageInput { + pub release_name: String, + pub source: SourceSnapshot, + pub executable: Option, + pub inputs: Vec, +} + +pub fn linux_env() -> EnvRef { + clusterflux::env!("linux") +} + +pub fn windows_env() -> EnvRef { + clusterflux::env!("windows") +} + +#[clusterflux::task(capabilities = "source_filesystem")] +pub async fn prepare_source() -> SourceSnapshot { + #[cfg(target_arch = "wasm32")] + { + return clusterflux::source::snapshot() + .await + .expect("the source task should snapshot the node checkout it was placed on"); + } + #[cfg(not(target_arch = "wasm32"))] + SourceSnapshot { + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + } +} + +#[clusterflux::task(capabilities = "command")] +pub async fn compile_linux(source: SourceSnapshot) -> Artifact { + let _ = &source; + #[cfg(target_arch = "wasm32")] + { + let executable = clusterflux::fs::output_path("hello-clusterflux") + .expect("the executable output path should be task-local"); + let output = clusterflux::command::Command::new("cc") + .args([ + "-Os", + "-static", + "-s", + "fixture/hello-clusterflux.c", + "-o", + executable.as_str(), + ]) + .current_dir("/workspace") + .env("SOURCE_DATE_EPOCH", "0") + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the declared Linux environment should provide a real C compiler"); + assert_eq!(output.status_code, Some(0)); + return clusterflux::fs::flush(&executable) + .await + .expect("the command-created executable should flush to node artifact storage"); + } + #[cfg(not(target_arch = "wasm32"))] + Artifact { + id: "hello-clusterflux-native-test".to_owned(), + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + size_bytes: 0, + } +} + +#[clusterflux::task] +#[unsafe(no_mangle)] +pub extern "C" fn task_add_one(input: i32) -> i32 { + input + 1 +} + +#[clusterflux::task] +#[unsafe(no_mangle)] +pub extern "C" fn task_trap(_input: i32) -> i32 { + #[cfg(target_arch = "wasm32")] + core::arch::wasm32::unreachable(); + #[cfg(not(target_arch = "wasm32"))] + panic!("intentional task trap") +} + +#[clusterflux::task(capabilities = "command")] +pub async fn package_release(input: PackageInput) -> Artifact { + #[cfg(target_arch = "wasm32")] + { + assert_eq!(input.release_name, "release.tar"); + let executable = input + .executable + .as_ref() + .expect("the package input should contain the compiler artifact"); + assert!(input.inputs.iter().any(|artifact| artifact == executable)); + let executable = clusterflux::fs::materialize(&executable, "package/hello-clusterflux") + .await + .expect("the retaining node should materialize the compiler artifact locally"); + let release = clusterflux::fs::output_path("release.tar") + .expect("the release output path should be task-local"); + let output = clusterflux::command::Command::new("tar") + .args([ + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "--mode=0755", + "-cf", + release.as_str(), + "-C", + "/clusterflux/output/package", + "hello-clusterflux", + ]) + .current_dir("/workspace") + .env("SOURCE_DATE_EPOCH", "0") + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the declared Linux environment should package the materialized executable"); + assert_eq!(output.status_code, Some(0)); + assert_eq!( + executable.as_str(), + "/clusterflux/output/package/hello-clusterflux" + ); + return clusterflux::fs::flush(&release) + .await + .expect("the command-created release archive should flush as the final artifact"); + } + #[cfg(not(target_arch = "wasm32"))] + { + let _ = input; + Artifact { + id: "release-native-test".to_owned(), + digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_owned(), + size_bytes: 0, + } + } +} + +#[clusterflux::task(capabilities = "command")] +pub async fn abort_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let output = clusterflux::command::Command::new("sh") + .args(["-c", "sleep 30"]) + .output() + .await + .expect("abort probe command should either finish or be stopped by process abort"); + return output.status_code.unwrap_or(-1); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +#[clusterflux::task(capabilities = "command")] +pub async fn long_join_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let output = clusterflux::command::Command::new("sh") + .args(["-c", "sleep 125"]) + .timeout(std::time::Duration::from_secs(180)) + .network_disabled() + .output() + .await + .expect("the controlled long-running command should complete normally"); + return output.status_code.unwrap_or(-1); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +#[clusterflux::task] +pub fn cooperative_cancellation_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + loop { + if clusterflux::process::cancellation_requested() + .expect("task control should remain available while the Wasm task is running") + { + // Cancellation is a request observed and handled by task code. Returning + // normally is intentionally distinct from the runtime's forced abort path. + return 17; + } + } + #[cfg(not(target_arch = "wasm32"))] + 17 +} + +#[clusterflux::task] +pub fn debug_child_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + loop { + if clusterflux::process::cancellation_requested() + .expect("child debug participant should retain task control") + { + return 23; + } + } + #[cfg(not(target_arch = "wasm32"))] + 23 +} + +#[clusterflux::task] +pub async fn debug_parent_probe() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let child = clusterflux::spawn::task(debug_child_probe) + .name("debug child participant") + .start() + .await + .expect("parent debug participant should spawn its child"); + return child + .join() + .await + .expect("parent debug participant should join its child"); + } + #[cfg(not(target_arch = "wasm32"))] + 23 +} + +#[clusterflux::main] +pub async fn build_main() -> BuildReport { + run_build_workflow().await +} + +#[clusterflux::main(name = "fail")] +pub async fn fail_main() -> Result { + #[cfg(target_arch = "wasm32")] + { + let child = clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) + .task_id("task_trap") + .name("intentional failing child") + .start() + .await + .map_err(|error| format!("failing entrypoint could not launch its child: {error}"))?; + return child + .join() + .await + .map_err(|error| format!("intentional child failure: {error}")); + } + #[cfg(not(target_arch = "wasm32"))] + Err("intentional child failure".to_owned()) +} + +#[clusterflux::main(name = "restart")] +pub async fn restart_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + return clusterflux::spawn::task_with_arg(0_i32, |value| task_trap(value)) + .task_id("task_trap") + .name("edited restart probe") + .failure_policy(clusterflux::spawn::TaskFailurePolicy::AwaitOperator) + .start() + .await + .expect("restart probe should launch its child") + .join() + .await + .expect("restart probe child should complete"); + } + #[cfg(not(target_arch = "wasm32"))] + task_add_one(41) +} + +#[clusterflux::main(name = "park-wake")] +pub async fn park_wake_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + let mut value = 0_i32; + for _ in 0..16 { + let next = clusterflux::spawn::task_with_arg(value, |input| task_add_one(input)) + .task_id("task_add_one") + .name("park/wake fuel refill probe") + .start() + .await + .expect("park/wake probe should launch") + .join() + .await + .expect("park/wake probe should complete"); + value = i32::try_from(next).expect("park/wake result should remain in i32 range"); + } + return value; + } + #[cfg(not(target_arch = "wasm32"))] + 16 +} + +#[clusterflux::main(name = "long-join")] +pub async fn long_join_main() -> i32 { + #[cfg(target_arch = "wasm32")] + { + return clusterflux::spawn::async_task(long_join_probe) + .name("controlled task longer than two minutes") + .env(linux_env()) + .start() + .await + .expect("long join probe should launch") + .join() + .await + .expect("long join probe should remain joinable beyond two minutes"); + } + #[cfg(not(target_arch = "wasm32"))] + 0 +} + +pub async fn run_build_workflow() -> BuildReport { + let source = clusterflux::spawn::async_task(prepare_source) + .name("prepare source snapshot") + .start() + .await + .unwrap() + .join() + .await + .unwrap(); + let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) + .name("compile linux") + .env(linux_env()) + .start() + .await + .unwrap(); + let linux_parallel = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux) + .name("compile linux in parallel") + .env(linux_env()) + .start() + .await + .unwrap(); + let linux_thread = linux.virtual_thread_id(); + let linux_parallel_thread = linux_parallel.virtual_thread_id(); + let linux_artifact = linux.join().await.unwrap(); + let linux_parallel_artifact = linux_parallel.join().await.unwrap(); + assert_eq!(linux_artifact.digest, linux_parallel_artifact.digest); + let package = clusterflux::spawn::async_task_with_arg( + PackageInput { + release_name: "release.tar".to_owned(), + source: source.clone(), + executable: Some(linux_artifact.clone()), + inputs: vec![linux_artifact.clone()], + }, + package_release, + ) + .name("package artifacts") + .env(linux_env()) + .start() + .await + .unwrap(); + + let package_thread = package.virtual_thread_id(); + let package_artifact = package.join().await.unwrap(); + + BuildReport { + linux_thread, + linux_parallel_thread, + package_thread, + linux_artifact, + package_artifact, + source, + } +} + +#[cfg(test)] +mod tests { + use futures_executor::block_on; + + use super::*; + + #[test] + fn flagship_workflow_is_rust_source_with_spawned_virtual_tasks() { + let main_report = block_on(build_main()); + assert_eq!( + main_report.linux_artifact.id, + "hello-clusterflux-native-test" + ); + assert_eq!(task_add_one(41), 42); + assert_eq!(block_on(restart_main()), 42); + assert_eq!(block_on(park_wake_main()), 16); + assert_eq!(block_on(long_join_main()), 0); + assert_eq!(linux_env().name, "linux"); + assert_eq!(windows_env().name, "windows"); + + let report = block_on(run_build_workflow()); + + assert_ne!(report.linux_thread, report.package_thread); + assert_ne!(report.linux_thread, report.linux_parallel_thread); + assert_eq!(report.linux_artifact.id, "hello-clusterflux-native-test"); + assert_eq!(report.package_artifact.id, "release-native-test"); + assert_eq!( + report.source.digest, + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } +} diff --git a/examples/recovery-build/README.md b/examples/recovery-build/README.md deleted file mode 100644 index 07b5d0d..0000000 --- a/examples/recovery-build/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Recovery build - -This advanced example starts two instances of the same task definition. The -stable lane completes while the recovering lane executes a real failing command -under AwaitOperator. Edit the failing command to write -/clusterflux/output/recovering.txt, then restart that task from the debugger or -CLI. The original main join completes from the replacement attempt. diff --git a/examples/recovery-build/envs/linux/Containerfile b/examples/recovery-build/envs/linux/Containerfile deleted file mode 100644 index ba87ba3..0000000 --- a/examples/recovery-build/envs/linux/Containerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM docker.io/library/alpine:3.20 -RUN apk add --no-cache build-base tar zstd -WORKDIR /workspace diff --git a/examples/recovery-build/src/lib.rs b/examples/recovery-build/src/lib.rs deleted file mode 100644 index 3e7fb08..0000000 --- a/examples/recovery-build/src/lib.rs +++ /dev/null @@ -1,45 +0,0 @@ -use clusterflux::prelude::*; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Serialize, Deserialize, clusterflux::TaskArg)] -pub struct BuildInput { - lane: String, - source: SourceSnapshot, -} - -#[clusterflux::task(capabilities = "command")] -pub async fn build_lane(input: BuildInput) -> Result { - let source_root = input.source.mount()?; - let output = fs::output(format!("{}.txt", input.lane))?; - let command = if input.lane == "recovering" { - "exit 23".to_owned() - } else { - format!("sleep 3; printf 'stable\n' > {}", output.as_str()) - }; - Command::new("sh") - .args(["-c", command.as_str()]) - .cwd(source_root) - .network_disabled() - .run() - .await?; - fs::publish(&output).await -} - -#[clusterflux::main] -pub async fn build() -> Result> { - let source = source::current_project().snapshot().await?; - let stable = clusterflux::spawn!(build_lane(BuildInput { - lane: "stable".to_owned(), - source: source.clone(), - })) - .on(clusterflux::env!("linux")) - .await?; - let recovering = clusterflux::spawn!(build_lane(BuildInput { - lane: "recovering".to_owned(), - source, - })) - .on(clusterflux::env!("linux")) - .failure_policy(clusterflux::TaskFailurePolicy::AwaitOperator) - .await?; - Ok(vec![stable.join().await?, recovering.join().await?]) -} diff --git a/flake.nix b/flake.nix index d9ccf26..f99377a 100644 --- a/flake.nix +++ b/flake.nix @@ -9,18 +9,6 @@ forAllSystems = nixpkgs.lib.genAttrs systems; in { - packages = forAllSystems (system: - let - pkgs = import nixpkgs { inherit system; }; - publicPackages = import ./packages.nix { inherit pkgs self; }; - privatePackages = - if builtins.pathExists ./web/packages.nix then - import ./web/packages.nix { inherit pkgs self; } - else - { }; - in - publicPackages // privatePackages); - devShells = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; diff --git a/packages.nix b/packages.nix deleted file mode 100644 index 1305b67..0000000 --- a/packages.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ pkgs, self }: -let - clusterflux-tools = pkgs.rustPlatform.buildRustPackage { - pname = "clusterflux-tools"; - version = "0.1.0"; - src = self; - cargoLock.lockFile = ./Cargo.lock; - nativeBuildInputs = [ - pkgs.git - pkgs.lld - pkgs.makeWrapper - ]; - cargoBuildFlags = [ - "--package" - "clusterflux-cli" - "--package" - "clusterflux-node" - "--package" - "clusterflux-coordinator" - "--package" - "clusterflux-dap" - ]; - cargoTestFlags = [ - "--package" - "clusterflux-cli" - "--package" - "clusterflux-node" - "--package" - "clusterflux-coordinator" - "--package" - "clusterflux-dap" - ]; - NIX_BUILD_CORES = "2"; - RUST_MIN_STACK = "1073741824"; - postInstall = '' - test -x "$out/bin/clusterflux" - test -x "$out/bin/clusterflux-node" - test -x "$out/bin/clusterflux-coordinator" - test -x "$out/bin/clusterflux-debug-dap" - for command in \ - clusterflux \ - clusterflux-node \ - clusterflux-coordinator \ - clusterflux-debug-dap - do - ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --version >/dev/null - ${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --help >/dev/null - done - ''; - postFixup = - let - runtimePath = pkgs.lib.makeBinPath [ - pkgs.cargo - pkgs.git - pkgs.lld - pkgs.rustc - ]; - in - '' - wrapProgram "$out/bin/clusterflux" --prefix PATH : ${runtimePath} - wrapProgram "$out/bin/clusterflux-node" --prefix PATH : ${runtimePath} - wrapProgram "$out/bin/clusterflux-debug-dap" --prefix PATH : ${runtimePath} - ''; - meta = { - description = "Clusterflux CLI, node, coordinator, and debugger adapter"; - mainProgram = "clusterflux"; - }; - }; -in -{ - inherit clusterflux-tools; - clusterflux = clusterflux-tools; - default = clusterflux-tools; -} diff --git a/scripts/acceptance-private.sh b/scripts/acceptance-private.sh new file mode 100755 index 0000000..5a54a15 --- /dev/null +++ b/scripts/acceptance-private.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +if [[ ! -d private/hosted-policy ]]; then + exit 0 +fi + +scripts/check-old-name.sh +node scripts/check-docs.js +scripts/check-code-size.sh +cargo fmt --all --manifest-path private/hosted-policy/Cargo.toml --check +cargo clippy --manifest-path private/hosted-policy/Cargo.toml --all-targets -- -D warnings +cargo test --locked --manifest-path private/hosted-policy/Cargo.toml --all-targets +node private/hosted-policy/scripts/prepare-hosted-deployment.js +if command -v podman >/dev/null 2>&1; then + node private/hosted-policy/scripts/postgres-durable-smoke.js +elif command -v nix >/dev/null 2>&1; then + nix shell nixpkgs#podman --command node private/hosted-policy/scripts/postgres-durable-smoke.js +else + node private/hosted-policy/scripts/postgres-durable-smoke.js +fi +if [[ -n "${CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR:-}" ]]; then + node private/hosted-policy/scripts/hosted-service-live-check.js +fi diff --git a/scripts/acceptance-public.sh b/scripts/acceptance-public.sh new file mode 100755 index 0000000..d5969ad --- /dev/null +++ b/scripts/acceptance-public.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +scripts/check-old-name.sh +node scripts/check-docs.js +scripts/check-code-size.sh +scripts/release-source-scan.sh +cargo fmt --all --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace --all-targets +cargo build --workspace --all-targets +cargo build -p launch-build-demo --target wasm32-unknown-unknown +node scripts/resource-metering-contract-smoke.js +node scripts/hostile-input-contract-smoke.js +node scripts/tenant-isolation-contract-smoke.js +node scripts/self-hosted-coordinator-smoke.js +node scripts/public-local-demo-matrix-smoke.js +node scripts/cli-output-mode-smoke.js +node scripts/cli-login-smoke.js +node scripts/cli-error-exit-smoke.js +node scripts/cli-browser-login-flow-smoke.js +node scripts/cli-install-smoke.js +node scripts/user-session-token-boundary-smoke.js +node scripts/sdk-spawn-runtime-smoke.js +node scripts/node-lifecycle-contract-smoke.js +node scripts/wasmtime-node-smoke.js +node scripts/wasmtime-assignment-smoke.js +if command -v podman >/dev/null 2>&1; then + node scripts/podman-backend-smoke.js +elif command -v nix >/dev/null 2>&1; then + nix shell nixpkgs#podman --command node scripts/podman-backend-smoke.js +else + node scripts/podman-backend-smoke.js +fi +node scripts/vscode-extension-smoke.js +node scripts/vscode-f5-smoke.js +node scripts/node-attach-smoke.js +node scripts/cli-local-run-smoke.js +node scripts/artifact-download-smoke.js +node scripts/artifact-export-smoke.js +node scripts/operator-panel-smoke.js +node scripts/source-preparation-smoke.js +node scripts/scheduler-placement-smoke.js +node scripts/windows-best-effort-smoke.js +node scripts/quic-smoke.js +node scripts/dap-smoke.js +node scripts/flagship-demo-smoke.js +scripts/verify-public-split.sh diff --git a/scripts/agent-signing.js b/scripts/agent-signing.js new file mode 100644 index 0000000..71c913d --- /dev/null +++ b/scripts/agent-signing.js @@ -0,0 +1,99 @@ +const crypto = require("crypto"); + +const { nodeIdentity, signedRequestPayloadDigest } = require("./node-signing"); + +function agentIdentity(seedPrefix, agent) { + const identity = nodeIdentity(seedPrefix, agent); + return { + ...identity, + publicKeyFingerprint: `sha256:${crypto + .createHash("sha256") + .update(identity.publicKey) + .digest("hex")}`, + }; +} + +function agentWorkflowSignatureMessage({ + tenant, + project, + agent, + requestKind, + process: processId, + task = "", + payloadDigest, + nonce, + issuedAtEpochSeconds, +}) { + const parts = [ + "clusterflux-agent-workflow-signature:v2", + tenant, + project, + agent, + requestKind, + processId, + task, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedAgentWorkflowProof(identity, request, options = {}) { + const nonce = + options.nonce || + `${request.type}-${process.pid}-${Date.now()}-${crypto + .randomBytes(8) + .toString("hex")}`; + const issuedAtEpochSeconds = + options.issuedAtEpochSeconds || Math.floor(Date.now() / 1000); + const processId = + request.type === "launch_task" ? request.task_spec?.process : request.process; + const task = + request.type === "launch_task" ? request.task_spec?.task_instance : request.task || ""; + const signature = crypto.sign( + null, + agentWorkflowSignatureMessage({ + tenant: request.tenant, + project: request.project, + agent: request.actor_agent, + requestKind: request.type, + process: processId, + task, + payloadDigest: signedRequestPayloadDigest(request), + nonce, + issuedAtEpochSeconds, + }), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAtEpochSeconds, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedAgentWorkflowRequest(identity, request, options = {}) { + const unsignedRequest = { + ...request, + agent_public_key_fingerprint: + options.publicKeyFingerprint || identity.publicKeyFingerprint, + }; + return { + ...unsignedRequest, + agent_signature: signedAgentWorkflowProof(identity, unsignedRequest, options), + }; +} + +module.exports = { + agentIdentity, + agentWorkflowSignatureMessage, + signedAgentWorkflowProof, + signedAgentWorkflowRequest, +}; diff --git a/scripts/artifact-download-smoke.js b/scripts/artifact-download-smoke.js new file mode 100755 index 0000000..2f6a44a --- /dev/null +++ b/scripts/artifact-download-smoke.js @@ -0,0 +1,401 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); +const { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + repo, + runFlagshipWorker, + send, + waitForJsonLine, +} = require("./real-flagship-harness"); + +const downloadNode = "node-download"; +const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode); + +const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function downloadRetainedBytes(addr, link, artifact, expectedSize) { + const chunks = []; + let offset = 0; + for (let attempt = 0; attempt < 500; attempt += 1) { + const response = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 256 * 1024, + }); + assert.strictEqual(response.type, "artifact_download_stream"); + if (!response.content_bytes_available) { + assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending"); + await delay(10); + continue; + } + assert.strictEqual(response.content_source, "retaining_node_reverse_stream"); + assert.strictEqual(response.content_offset, offset); + const bytes = Buffer.from(response.content_base64, "base64"); + assert.strictEqual(response.streamed_bytes, bytes.length); + chunks.push(bytes); + offset += bytes.length; + if (response.content_eof) { + const content = Buffer.concat(chunks); + assert.strictEqual(content.length, expectedSize); + return { content, response }; + } + } + throw new Error("timed out waiting for retained artifact reverse stream"); +} + +function downloadNodeCapabilities() { + return flagshipNodeCapabilities(); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + let worker; + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + assert.strictEqual(workerReady.mode, "worker"); + const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr); + assert.strictEqual(compileEvent.status_code, 0); + assert.strictEqual(packageEvent.status_code, 0); + assert.deepStrictEqual(compileEvent.result, { + Artifact: { + id: compileEvent.artifact_path.slice("/vfs/artifacts/".length), + digest: compileEvent.artifact_digest, + size_bytes: compileEvent.artifact_size_bytes, + }, + }); + assert.deepStrictEqual(packageEvent.result, { + Artifact: { + id: packageEvent.artifact_path.slice("/vfs/artifacts/".length), + digest: packageEvent.artifact_digest, + size_bytes: packageEvent.artifact_size_bytes, + }, + }); + assert.ok(compileEvent.artifact_size_bytes > 0); + assert.ok(packageEvent.artifact_size_bytes > compileEvent.artifact_size_bytes); + assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); + assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/); + const artifactPath = packageEvent.artifact_path; + assert.match(artifactPath, /^\/vfs\/artifacts\/release\.tar-[0-9a-f]{64}$/); + const artifact = artifactPath.slice("/vfs/artifacts/".length); + + const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [artifact], + direct_connectivity: false, + online: true, + })); + assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded"); + + const disconnectedLink = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(disconnectedLink.type, "artifact_download_link"); + + const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [artifact], + direct_connectivity: true, + online: true, + })); + assert.strictEqual(connectedReport.type, "node_capabilities_recorded"); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(link.type, "artifact_download_link"); + assert.strictEqual(link.link.tenant, "tenant"); + assert.strictEqual(link.link.project, "project"); + assert.strictEqual(link.link.process, virtualProcess); + assert.deepStrictEqual(link.link.actor, { User: "user" }); + assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/); + assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); + assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60); + assert.ok( + link.link.url_path.endsWith( + `/artifacts/tenant/project/${virtualProcess}/${artifact}` + ) + ); + assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" }); + + const crossTenant = await send(addr, { + type: "create_artifact_download_link", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match(crossTenant.message, /tenant mismatch/); + + const crossProject = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "other-project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(crossProject.type, "error"); + assert.match(crossProject.message, /project mismatch/); + + const crossTenantOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossTenantOpen.type, "error"); + assert.match(crossTenantOpen.message, /tenant mismatch/); + + const crossProjectOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "other-project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossProjectOpen.type, "error"); + assert.match(crossProjectOpen.message, /project mismatch/); + + const guessed = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: "sha256:guessed", + chunk_bytes: 1, + }); + assert.strictEqual(guessed.type, "error"); + assert.match(guessed.message, /token is invalid/); + + const crossActorOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "other-user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(crossActorOpen.type, "error"); + assert.match(crossActorOpen.message, /token is invalid/); + + const downloaded = await downloadRetainedBytes( + addr, + link, + artifact, + packageEvent.artifact_size_bytes, + ); + const downloadedDigest = `sha256:${crypto + .createHash("sha256") + .update(downloaded.content) + .digest("hex")}`; + assert.strictEqual(downloadedDigest, packageEvent.artifact_digest); + const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-")); + try { + const archive = path.join(inspect, "release.tar"); + fs.writeFileSync(archive, downloaded.content); + const listing = cp.execFileSync("tar", ["-tf", archive], { encoding: "utf8" }); + assert.strictEqual(listing.trim(), "hello-clusterflux"); + cp.execFileSync("tar", ["-xf", archive, "-C", inspect]); + fs.chmodSync(path.join(inspect, "hello-clusterflux"), 0o755); + assert.strictEqual( + cp.execFileSync(path.join(inspect, "hello-clusterflux"), { encoding: "utf8" }), + "hello from a real Clusterflux build\n" + ); + } finally { + fs.rmSync(inspect, { recursive: true, force: true }); + } + const cliDownloadDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), "clusterflux-cli-download-") + ); + try { + const cliDownloadPath = path.join(cliDownloadDirectory, "release.tar"); + const cliDownload = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "artifact", "download", artifact, + "--to", cliDownloadPath, + "--max-bytes", "1048576", + "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, + "--tenant", "tenant", + "--project-id", "project", + "--user", "user", + "--json", + ], + { cwd: repo, env: process.env, encoding: "utf8" } + ) + ); + assert.strictEqual(cliDownload.command, "artifact download"); + assert.strictEqual(cliDownload.local_download.status, "local_bytes_written"); + assert.strictEqual( + cliDownload.local_download.verified_digest, + packageEvent.artifact_digest + ); + assert.strictEqual( + `sha256:${crypto + .createHash("sha256") + .update(fs.readFileSync(cliDownloadPath)) + .digest("hex")}`, + packageEvent.artifact_digest + ); + } finally { + fs.rmSync(cliDownloadDirectory, { recursive: true, force: true }); + } + assert.strictEqual(downloaded.response.content_eof, true); + assert.strictEqual( + downloaded.response.charged_download_bytes, + packageEvent.artifact_size_bytes + ); + assert.strictEqual(downloaded.response.link.artifact, artifact); + + const crossActorRevoke = await send(addr, { + type: "revoke_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "other-user", + artifact, + token_digest: link.link.scoped_token_digest, + }); + assert.strictEqual(crossActorRevoke.type, "error"); + assert.match(crossActorRevoke.message, /token is invalid/); + + const revoked = await send(addr, { + type: "revoke_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + token_digest: link.link.scoped_token_digest, + }); + assert.strictEqual(revoked.type, "artifact_download_link_revoked"); + assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest); + + const revokedOpen = await send(addr, { + type: "open_artifact_download_stream", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + token_digest: link.link.scoped_token_digest, + chunk_bytes: 1, + }); + assert.strictEqual(revokedOpen.type, "error"); + assert.match(revokedOpen.message, /revoked/); + + const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: downloadNode, + capabilities: downloadNodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false, + online: true, + })); + assert.strictEqual(gcReport.type, "node_capabilities_recorded"); + + const collectedLink = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + max_bytes: 1024 * 1024, + ttl_seconds: 60, + }); + assert.strictEqual(collectedLink.type, "error"); + assert.match(collectedLink.message, /unavailable from current retention/); + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Artifact download smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/artifact-export-smoke.js b/scripts/artifact-export-smoke.js new file mode 100644 index 0000000..95213ed --- /dev/null +++ b/scripts/artifact-export-smoke.js @@ -0,0 +1,269 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); +const { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + repo, + runFlagshipWorker, + send, + waitForJsonLine, + waitForNodeStatus, +} = require("./real-flagship-harness"); + +const sourceNode = "node-export-source"; +const sourceIdentity = nodeIdentity("artifact-export-smoke", sourceNode); + +function nodeCapabilities() { + return flagshipNodeCapabilities(); +} + +function runJson(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = cp.spawn(command, args, { cwd: repo, ...options }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code !== 0) { + reject( + new Error( + `${command} ${args.join(" ")} failed with code ${code}\n${stderr}\n${stdout}` + ) + ); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject(new Error(`${command} did not return JSON\n${stdout}\n${error.message}`)); + } + }); + }); +} + +async function reportNode( + addr, + node, + identity, + { directConnectivity = true, online = true, artifacts = [] } = {} +) { + const response = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node, + capabilities: nodeCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: artifacts, + direct_connectivity: directConnectivity, + online, + })); + assert.strictEqual(response.type, "node_capabilities_recorded"); + assert.strictEqual(response.node, node); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { + cwd: repo, + env: { ...process.env, CLUSTERFLUX_NODE_STALE_AFTER_SECONDS: "1" }, + } + ); + + let worker; + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + worker = await runFlagshipWorker(addr, sourceNode, sourceIdentity); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + assert.strictEqual(workerReady.mode, "worker"); + const firstNodeTaskCompletion = waitForNodeStatus(worker.child, "completed"); + const { compileEvent, process: virtualProcess } = await launchFlagship(addr); + const workerCompletion = await firstNodeTaskCompletion; + assert.strictEqual(workerCompletion.node_status, "completed"); + assert.strictEqual( + workerCompletion.task_assignment_response.task_spec.task_definition, + "prepare_source" + ); + assert.strictEqual( + workerCompletion.virtual_thread, + workerCompletion.task_assignment_response.task_spec.task_instance + ); + assert.strictEqual(compileEvent.status_code, 0); + assert.match( + compileEvent.artifact_path, + /^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/ + ); + const artifact = compileEvent.artifact_path.slice("/vfs/artifacts/".length); + + await reportNode(addr, sourceNode, sourceIdentity, { + artifacts: [artifact], + }); + + const receiverIdentity = nodeIdentity("artifact-export-smoke", "node-export-receiver"); + const attachedReceiver = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node: "node-export-receiver", + public_key: receiverIdentity.publicKey, + }); + assert.strictEqual(attachedReceiver.type, "node_attached"); + await reportNode(addr, "node-export-receiver", receiverIdentity); + + const exportPlan = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(exportPlan.type, "artifact_export_plan"); + assert.strictEqual(exportPlan.source_node, "node-export-source"); + assert.strictEqual(exportPlan.receiver_node, "node-export-receiver"); + assert.strictEqual(exportPlan.plan.transport, "NativeQuic"); + assert.strictEqual(exportPlan.plan.scope.tenant, "tenant"); + assert.strictEqual(exportPlan.plan.scope.project, "project"); + assert.strictEqual(exportPlan.plan.scope.process, virtualProcess); + assert.deepStrictEqual(exportPlan.plan.scope.object, { Artifact: artifact }); + assert.strictEqual(exportPlan.plan.source.node, "node-export-source"); + assert.strictEqual(exportPlan.plan.destination.node, "node-export-receiver"); + assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true); + assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false); + assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/); + assert.strictEqual(exportPlan.artifact_size_bytes, compileEvent.artifact_size_bytes); + + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-artifact-export-")); + const exportPath = path.join(temp, "hello-clusterflux"); + const cliExport = await runJson("cargo", [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "artifact", + "export", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--user", + "user", + "--json", + artifact, + "--receiver-node", + "node-export-receiver", + "--to", + exportPath, + ]); + assert.strictEqual(cliExport.command, "artifact export"); + assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true); + assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written"); + assert.strictEqual( + cliExport.export_plan.bytes_written, + compileEvent.artifact_size_bytes + ); + assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false); + assert.strictEqual( + cliExport.local_export.verified_digest, + compileEvent.artifact_digest + ); + fs.chmodSync(exportPath, 0o755); + assert.strictEqual( + cp.execFileSync(exportPath, { encoding: "utf8" }), + "hello from a real Clusterflux build\n" + ); + + const crossTenant = await send(addr, { + type: "export_artifact_to_node", + tenant: "other", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match(crossTenant.message, /tenant mismatch/); + + await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); + const failedDirect = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: false, + failure_reason: "nat traversal failed", + }); + assert.strictEqual(failedDirect.type, "error"); + assert.match(failedDirect.message, /nat traversal failed/); + assert.match(failedDirect.message, /coordinator bulk relay is disabled/); + + await reportNode(addr, "node-export-receiver", receiverIdentity, { online: false }); + await new Promise((resolve) => setTimeout(resolve, 2100)); + await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] }); + const offlineReceiver = await send(addr, { + type: "export_artifact_to_node", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact, + receiver_node: "node-export-receiver", + direct_connectivity: true, + failure_reason: "", + }); + assert.strictEqual(offlineReceiver.type, "error"); + assert.match(offlineReceiver.message, /offline/); + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Artifact export smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/check-code-size.sh b/scripts/check-code-size.sh new file mode 100755 index 0000000..e740b44 --- /dev/null +++ b/scripts/check-code-size.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +maximum_lines=3000 +failed=0 +while IFS= read -r -d '' file; do + case "$file" in + */tests.rs|*/tests/*) continue ;; + esac + lines="$(wc -l < "$file")" + if ((lines > maximum_lines)); then + printf '%s has %s lines; production file limit is %s\n' "$file" "$lines" "$maximum_lines" >&2 + failed=1 + fi +done < <(find crates private/hosted-policy/src -type f -name '*.rs' -print0) + +if ((failed)); then + exit 1 +fi +printf 'production source file size guard passed (%s lines maximum)\n' "$maximum_lines" diff --git a/scripts/check-docs.js b/scripts/check-docs.js new file mode 100644 index 0000000..fb6f3d8 --- /dev/null +++ b/scripts/check-docs.js @@ -0,0 +1,111 @@ +#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); + +const root = path.resolve(__dirname, ".."); +const publicDocs = [ + "README.md", + "SECURITY.md", + "docs/getting-started.md", + "docs/architecture.md", + "docs/nodes.md", + "docs/environments.md", + "docs/artifacts.md", + "docs/debugging.md", + "docs/task-abi.md", + "docs/self-hosting.md", + "docs/security.md", +]; +const privateDocs = [ + "private/docs/hosted-deployment.md", + "private/docs/authentik.md", + "private/docs/community-policy.md", + "private/docs/bandwidth-and-cost-controls.md", + "private/docs/publishing.md", +]; +const internalDocs = ["internal/finish_mvp_2.md"]; +const filteredPublicTree = + process.env.CLUSTERFLUX_FILTERED_PUBLIC_TREE === "1" || + fs.existsSync(path.join(root, "CLUSTERFLUX_PUBLIC_TREE.json")); + +const failures = []; +const expectExactMarkdownSet = (directory, expected) => { + const actual = fs + .readdirSync(path.join(root, directory), { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => path.posix.join(directory, entry.name)) + .sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + failures.push(`${directory} markdown set is ${actual.join(", ")}; expected ${wanted.join(", ")}`); + } +}; + +const requiredDocs = filteredPublicTree + ? publicDocs + : [...publicDocs, ...privateDocs, ...internalDocs]; +for (const file of requiredDocs) { + if (!fs.existsSync(path.join(root, file))) failures.push(`missing canonical document: ${file}`); +} +expectExactMarkdownSet("docs", publicDocs.filter((file) => file.startsWith("docs/"))); +if (filteredPublicTree) { + for (const directory of ["private", "internal"]) { + if (fs.existsSync(path.join(root, directory))) { + failures.push(`${directory}/ must not exist in the filtered public tree`); + } + } +} + +const forbidden = [ + [/\bmvp\b/i, "internal milestone term"], + [/acceptance criteria/i, "internal gate language"], + [/release verification/i, "internal release language"], + [/public\/private source split/i, "source split narrative"], + [/founder|business decisions/i, "business planning narrative"], + [/hacker news|\bHN\b/, "launch-channel narrative"], + [/\busers can\b/i, "indirect reader wording"], + [/node\s+scripts\//i, "developer script instruction"], + [/scripts\/[^\s)]*smoke/i, "smoke script instruction"], + [/internal\/[^\s)]*/i, "internal tooling reference"], + [/private\/[^\s)]*/i, "private source reference"], +]; +const topLevelCommands = new Set([ + "doctor", "login", "logout", "auth", "agent", "key", "project", "inspect", + "build", "bundle", "run", "node", "process", "task", "logs", "artifact", + "dap", "debug", "quota", "admin", +]); + +for (const file of publicDocs) { + const absolute = path.join(root, file); + const content = fs.readFileSync(absolute, "utf8"); + for (const [pattern, description] of forbidden) { + if (pattern.test(content)) failures.push(`${file}: contains ${description}`); + } + for (const match of content.matchAll(/\]\(([^)#]+)(?:#[^)]+)?\)/g)) { + const target = match[1]; + if (/^(?:https?:|mailto:)/.test(target)) continue; + const resolved = path.resolve(path.dirname(absolute), target); + if (!fs.existsSync(resolved)) failures.push(`${file}: broken link ${target}`); + } + for (const line of content.split(/\r?\n/)) { + const command = line.trim().match(/^clusterflux\s+([a-z][a-z-]*)\b/); + if (command && !topLevelCommands.has(command[1])) { + failures.push(`${file}: unknown top-level CLI command ${command[1]}`); + } + } +} + +const rootMarkdown = fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => entry.name) + .sort(); +if (JSON.stringify(rootMarkdown) !== JSON.stringify(["README.md", "SECURITY.md"])) { + failures.push(`top-level markdown set is ${rootMarkdown.join(", ")}; expected README.md, SECURITY.md`); +} + +if (failures.length) { + for (const failure of failures) console.error(failure); + process.exit(1); +} +console.log("documentation checks passed"); diff --git a/scripts/check-old-name.sh b/scripts/check-old-name.sh new file mode 100755 index 0000000..9a871de --- /dev/null +++ b/scripts/check-old-name.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +legacy_lower="$(printf '%s%s' 'disa' 'smer')" +legacy_title="$(printf '%s%s' 'Disa' 'smer')" +legacy_upper="$(printf '%s%s' 'DISA' 'SMER')" +pattern="${legacy_lower}|${legacy_title}|${legacy_upper}" + +allowed_content='^\./scripts/migrate-clusterflux-state\.sh:' +matches="$( + rg -n --hidden \ + --glob '!**/.git/**' \ + --glob '!**/target/**' \ + --glob '!**/node_modules/**' \ + --glob '!**/vendor/**' \ + --glob '!**/.direnv/**' \ + --glob '!**/.cache/**' \ + --glob '!**/dist/**' \ + --glob '!**/out/**' \ + "$pattern" . 2>/dev/null | rg -v "$allowed_content" || true +)" + +paths="$( + find . \ + \( -type d \( -name .git -o -name target -o -name node_modules -o -name vendor -o -name .direnv -o -name .cache -o -name dist -o -name out \) -prune \) -o \ + -iname "*${legacy_lower}*" -print | sort || true +)" + +if [[ -n "$matches" || -n "$paths" ]]; then + [[ -z "$matches" ]] || printf '%s\n' "$matches" >&2 + [[ -z "$paths" ]] || printf '%s\n' "$paths" >&2 + printf 'unexpected legacy product name remains\n' >&2 + exit 1 +fi + +printf 'old-name guard passed\n' diff --git a/scripts/cli-browser-login-flow-smoke.js b/scripts/cli-browser-login-flow-smoke.js new file mode 100644 index 0000000..0c6341b --- /dev/null +++ b/scripts/cli-browser-login-flow-smoke.js @@ -0,0 +1,222 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const tmp = path.join(repo, "target", "acceptance", "tmp", "cli-browser-login-flow"); +const project = path.join(tmp, "project"); +fs.rmSync(tmp, { recursive: true, force: true }); +fs.mkdirSync(project, { recursive: true }); + +function writeOpener() { + const opener = path.join(tmp, "browser-opener.js"); + const trace = path.join(tmp, "browser-opener.log"); + fs.writeFileSync( + opener, + `#!/usr/bin/env node +const fs = require("fs"); +const trace = ${JSON.stringify(trace)}; +const loginUrl = new URL(process.argv[2]); +const state = loginUrl.searchParams.get("state"); +const nonce = loginUrl.searchParams.get("nonce"); +const challenge = loginUrl.searchParams.get("code_challenge"); +if (loginUrl.protocol !== "https:" || !state || !nonce || !challenge) { + fs.appendFileSync(trace, "missing server-owned OIDC parameters\\n"); + process.exit(1); +} +fs.appendFileSync(trace, "server-owned browser transaction\\n"); +// Model a real browser/opener that remains alive after the CLI transaction. +// Its descriptors must not keep the invoking CLI process open. +setTimeout(() => process.exit(0), 5000); +` + ); + fs.chmodSync(opener, 0o755); + return opener; +} + +function startCoordinator() { + return new Promise((resolve, reject) => { + const requests = []; + const server = net.createServer((socket) => { + let buffered = ""; + socket.on("data", (chunk) => { + buffered += chunk.toString("utf8"); + while (buffered.includes("\n")) { + const newline = buffered.indexOf("\n"); + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const envelope = JSON.parse(line); + requests.push(envelope); + assert.strictEqual(envelope.type, "coordinator_request"); + assert.strictEqual(envelope.protocol_version, 1); + assert.strictEqual(envelope.authentication.kind, "none"); + const request = envelope.payload; + + if (requests.length === 1) { + assert.strictEqual(envelope.request_id, "cli-1"); + assert.strictEqual(envelope.operation, "begin_oidc_browser_login"); + assert.deepStrictEqual(request, { + type: "begin_oidc_browser_login", + }); + socket.write( + `${JSON.stringify({ + type: "oidc_browser_login_started", + transaction_id: "login-transaction", + polling_secret: "opaque-polling-secret", + authorization_url: + "https://auth.michelpaulissen.com/application/o/authorize/?state=server-state&nonce=server-nonce&code_challenge=server-pkce&code_challenge_method=S256&redirect_uri=https%3A%2F%2Fclusterflux.michelpaulissen.com%2Fauth%2Fcallback", + expires_at_epoch_seconds: 1800000000, + })}\n` + ); + continue; + } + + assert.strictEqual(envelope.request_id, "cli-2"); + assert.strictEqual(envelope.operation, "poll_oidc_browser_login"); + assert.deepStrictEqual(request, { + type: "poll_oidc_browser_login", + transaction_id: "login-transaction", + polling_secret: "opaque-polling-secret", + }); + socket.end( + `${JSON.stringify({ + type: "oidc_browser_session", + session: { + tenant: "tenant-smoke", + project: "project-smoke", + user: "user-smoke", + cli_session_credential_kind: "CliDeviceSession", + cli_session_secret: "scoped-cli-session-secret", + expires_at_epoch_seconds: 1800000000, + provider_tokens_sent_to_nodes: false, + }, + })}\n` + ); + server.close(); + } + }); + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resolve({ + url: `${address.address}:${address.port}`, + requests, + close: () => + new Promise((closeResolve) => { + if (!server.listening) closeResolve(); + else server.close(() => closeResolve()); + }), + }); + }); + }); +} + +function runClusterflux(args, env) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "--manifest-path", + path.join(repo, "Cargo.toml"), + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + ...args, + ], + { cwd: project, env, stdio: ["ignore", "pipe", "pipe"] } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk.toString("utf8"))); + child.stderr.on("data", (chunk) => (stderr += chunk.toString("utf8"))); + child.once("error", reject); + child.once("close", (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(`clusterflux exited ${code}\n${stderr}\n${stdout}`)); + }); + }); +} + +(async () => { + const opener = writeOpener(); + const coordinator = await startCoordinator(); + try { + const loginStarted = Date.now(); + const report = JSON.parse( + await runClusterflux( + [ + "login", + "--browser", + "--json", + "--coordinator", + coordinator.url, + "--project-id", + "project-smoke", + ], + { + ...process.env, + CLUSTERFLUX_BROWSER_OPEN_COMMAND: opener, + CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS: "5", + } + ) + ); + assert( + Date.now() - loginStarted < 3000, + "a long-lived browser opener must not keep CLI output pipes or login completion open" + ); + assert.strictEqual(report.plan.coordinator, coordinator.url); + assert.strictEqual(report.boundary.cli_contacted_coordinator, true); + assert.strictEqual(report.boundary.scoped_cli_session_received, true); + assert.strictEqual(report.boundary.local_cli_session_file_written, true); + assert.strictEqual(report.boundary.provider_tokens_persisted_locally, false); + assert.strictEqual(report.boundary.provider_tokens_exposed_to_cli, false); + assert.strictEqual(report.boundary.provider_tokens_sent_to_nodes, false); + assert.strictEqual(report.boundary.coordinator_session_requests, 2); + assert.strictEqual(coordinator.requests.length, 2); + + const sessionFile = path.join(project, ".clusterflux", "session.json"); + const sessionText = fs.readFileSync(sessionFile, "utf8"); + const session = JSON.parse(sessionText); + assert.strictEqual(session.kind, "human"); + assert.strictEqual(session.coordinator, coordinator.url); + assert.strictEqual(session.tenant, "tenant-smoke"); + assert.strictEqual(session.project, "project-smoke"); + assert.strictEqual(session.user, "user-smoke"); + assert.strictEqual(session.cli_session_credential_kind, "CliDeviceSession"); + assert.strictEqual(session.token_expiry_posture, "expires_at"); + assert.strictEqual(session.expires_at, "1800000000"); + assert.strictEqual(session.provider_tokens_exposed_to_cli, false); + assert.strictEqual(session.provider_tokens_sent_to_nodes, false); + assert.doesNotMatch( + sessionText, + /access_token|refresh_token|id_token|provider-secret|authorization_code|Bearer/ + ); + + const authStatus = JSON.parse( + await runClusterflux(["auth", "status", "--json"], process.env) + ); + assert.strictEqual(authStatus.active_coordinator, coordinator.url); + assert.strictEqual(authStatus.principal, "user-smoke"); + assert.strictEqual(authStatus.tenant, "tenant-smoke"); + assert.strictEqual(authStatus.project, "project-smoke"); + assert.strictEqual(authStatus.session.kind, "human"); + assert.strictEqual(authStatus.session.source, "session_file"); + assert.strictEqual(authStatus.session.provider_tokens_exposed_to_cli, false); + assert.strictEqual(authStatus.session.provider_tokens_exposed_to_nodes, false); + } finally { + await coordinator.close(); + } + console.log("CLI browser login flow smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-error-exit-smoke.js b/scripts/cli-error-exit-smoke.js new file mode 100755 index 0000000..76763bf --- /dev/null +++ b/scripts/cli-error-exit-smoke.js @@ -0,0 +1,365 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const http = require("http"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/launch-build-demo"); +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-error-")); + +function runClusterflux(args) { + return new Promise((resolve) => { + const child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + stdio: ["ignore", "pipe", "pipe"], + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("close", (code, signal) => { + resolve({ code, signal, stdout, stderr }); + }); + }); +} + +async function runWithOneCoordinatorResponse(buildArgs, response) { + let request = ""; + const server = http.createServer((incoming, outgoing) => { + incoming.setEncoding("utf8"); + incoming.on("data", (chunk) => { + request += chunk; + }); + incoming.on("end", () => { + outgoing.writeHead(200, { "content-type": "application/json" }); + outgoing.end(JSON.stringify(response)); + server.close(); + }); + }); + + const address = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address())); + }); + const coordinator = `http://${address.address}:${address.port}`; + const result = await runClusterflux(buildArgs(coordinator)); + return { request, result }; +} + +async function main() { + const environmentProject = path.join(tempRoot, "missing-env-project"); + fs.mkdirSync(path.join(environmentProject, "src"), { recursive: true }); + fs.writeFileSync( + path.join(environmentProject, "Cargo.toml"), + "[package]\nname = \"missing-env-project\"\nversion = \"0.1.0\"\nedition = \"2021\"\n" + ); + fs.writeFileSync( + path.join(environmentProject, "src", "main.rs"), + "fn main() { let _target = env!(\"linux\"); }\n" + ); + const environmentFailure = await runClusterflux([ + "build", + "--project", + environmentProject, + "--json", + ]); + assert.strictEqual(environmentFailure.signal, null, environmentFailure.stderr); + assert.strictEqual(environmentFailure.code, 26, environmentFailure.stderr); + const environmentReport = JSON.parse(environmentFailure.stdout); + assert.strictEqual(environmentReport.status, "blocked_before_schedule"); + assert.strictEqual(environmentReport.scheduled_work, false); + assert.strictEqual(environmentReport.machine_error.category, "environment"); + assert.strictEqual( + environmentReport.machine_error.process_exit_code_applied, + true + ); + assert( + environmentReport.machine_error.next_actions.includes("clusterflux inspect") + ); + assert.strictEqual(environmentReport.diagnostics[0].code, "missing_environment"); + + const nonInteractive = await runClusterflux([ + "run", + "build", + "--project", + project, + "--non-interactive", + "--json", + ]); + assert.strictEqual(nonInteractive.signal, null, nonInteractive.stderr); + assert.strictEqual(nonInteractive.code, 20, nonInteractive.stderr); + assert.doesNotMatch(nonInteractive.stderr, /Opening Clusterflux browser login/); + const nonInteractiveReport = JSON.parse(nonInteractive.stdout); + assert.strictEqual(nonInteractiveReport.status, "authentication_required"); + assert.strictEqual(nonInteractiveReport.non_interactive, true); + assert.strictEqual(nonInteractiveReport.browser_opened, false); + assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); + assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); + assert.strictEqual( + nonInteractiveReport.machine_error.process_exit_code_applied, + true + ); + assert( + nonInteractiveReport.machine_error.next_actions.includes( + "pass --local to run against local services" + ) + ); + + let request = ""; + const server = http.createServer((incoming, outgoing) => { + incoming.setEncoding("utf8"); + incoming.on("data", (chunk) => { + request += chunk; + }); + incoming.on("end", () => { + outgoing.writeHead(200, { "content-type": "application/json" }); + outgoing.end( + JSON.stringify({ + type: "error", + message: "quota unavailable: resource limit exceeded for api_calls", + }) + ); + server.close(); + }); + }); + + const address = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address())); + }); + const coordinator = `http://${address.address}:${address.port}`; + + const result = await runClusterflux([ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ]); + + assert.strictEqual(result.signal, null, result.stderr); + assert.strictEqual(result.code, 22, result.stderr); + assert.match(request, /"type":"start_process"/); + const report = JSON.parse(result.stdout); + assert.strictEqual(report.status, "coordinator_rejected"); + assert.strictEqual(report.run_start.machine_error.category, "quota"); + assert.strictEqual(report.run_start.machine_error.resource_category, "api_calls"); + assert.strictEqual(report.run_start.machine_error.community_tier_language, true); + assert.strictEqual( + report.run_start.machine_error.community_tier_label, + "community tier" + ); + assert.doesNotMatch(result.stdout, new RegExp(["free", "tier"].join(" "), "i")); + assert.strictEqual( + report.run_start.machine_error.private_abuse_heuristics_exposed, + false + ); + assert.strictEqual(report.run_start.machine_error.stable_exit_code, 22); + assert.strictEqual( + report.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + report.run_start.machine_error.next_actions.includes("clusterflux quota status") + ); + + const capabilityFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ], + { + type: "error", + message: + "scheduler placement failed: no capable node for placement: missing capability Command", + } + ); + assert.strictEqual(capabilityFailure.result.signal, null, capabilityFailure.result.stderr); + assert.strictEqual(capabilityFailure.result.code, 24, capabilityFailure.result.stderr); + assert.match(capabilityFailure.request, /"type":"start_process"/); + const capabilityReport = JSON.parse(capabilityFailure.result.stdout); + assert.strictEqual( + capabilityReport.run_start.machine_error.category, + "capability" + ); + assert.strictEqual( + capabilityReport.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + capabilityReport.run_start.machine_error.next_actions.includes( + "attach a node with the required capabilities" + ) + ); + + const nodePolicyFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "run", + "build", + "--project", + project, + "--coordinator", + coordinator, + "--json", + ], + { + type: "error", + message: "node policy denied native command execution", + } + ); + assert.strictEqual(nodePolicyFailure.result.signal, null, nodePolicyFailure.result.stderr); + assert.strictEqual(nodePolicyFailure.result.code, 23, nodePolicyFailure.result.stderr); + assert.match(nodePolicyFailure.request, /"type":"start_process"/); + const nodePolicyReport = JSON.parse(nodePolicyFailure.result.stdout); + assert.strictEqual( + nodePolicyReport.run_start.machine_error.category, + "policy" + ); + assert.strictEqual( + nodePolicyReport.run_start.machine_error.process_exit_code_applied, + true + ); + assert( + nodePolicyReport.run_start.machine_error.next_actions.includes( + "check coordinator policy for this action" + ) + ); + + const programFailure = await runWithOneCoordinatorResponse( + (coordinator) => [ + "task", + "list", + "--coordinator", + coordinator, + "--json", + ], + { + type: "task_events", + events: [ + { + process: "vp-current", + task: "compile", + terminal_state: "failed", + environment: "linux", + node: "node-linux", + status_code: 1, + stderr_tail: "task exited with status 1", + }, + ], + } + ); + assert.strictEqual(programFailure.result.signal, null, programFailure.result.stderr); + assert.strictEqual(programFailure.result.code, 0, programFailure.result.stderr); + assert.match(programFailure.request, /"type":"list_task_events"/); + const programReport = JSON.parse(programFailure.result.stdout); + assert.strictEqual(programReport.tasks[0].machine_error.category, "program"); + assert.strictEqual(programReport.tasks[0].machine_error.stable_exit_code, 27); + assert( + programReport.tasks[0].machine_error.next_actions.includes("clusterflux logs") + ); + + const artifactDownload = await runWithOneCoordinatorResponse( + (coordinator) => [ + "artifact", + "download", + "app.txt", + "--coordinator", + coordinator, + "--json", + ], + { + type: "artifact_download_denied", + message: "artifact download unauthorized for project", + } + ); + assert.strictEqual(artifactDownload.result.signal, null, artifactDownload.result.stderr); + assert.strictEqual(artifactDownload.result.code, 21, artifactDownload.result.stderr); + assert.match(artifactDownload.request, /"type":"create_artifact_download_link"/); + const artifactDownloadReport = JSON.parse(artifactDownload.result.stdout); + assert.strictEqual( + artifactDownloadReport.download_session.machine_error.category, + "authorization" + ); + assert.strictEqual( + artifactDownloadReport.download_session.machine_error.process_exit_code_applied, + true + ); + + const artifactExport = await runWithOneCoordinatorResponse( + (coordinator) => [ + "artifact", + "export", + "app.txt", + "--to", + path.join(project, "target", "blocked-artifact.txt"), + "--coordinator", + coordinator, + "--json", + ], + { + type: "artifact_export_unavailable", + message: "direct connectivity unavailable for artifact export", + } + ); + assert.strictEqual(artifactExport.result.signal, null, artifactExport.result.stderr); + assert.strictEqual(artifactExport.result.code, 25, artifactExport.result.stderr); + assert.match(artifactExport.request, /"type":"export_artifact_to_node"/); + const artifactExportReport = JSON.parse(artifactExport.result.stdout); + assert.strictEqual( + artifactExportReport.export_plan.machine_error.category, + "connectivity" + ); + assert.strictEqual( + artifactExportReport.export_plan.machine_error.process_exit_code_applied, + true + ); + + const confirmation = await runClusterflux([ + "process", + "cancel", + "--coordinator", + "127.0.0.1:9", + "--json", + ]); + assert.strictEqual(confirmation.signal, null, confirmation.stderr); + assert.strictEqual(confirmation.code, 23, confirmation.stderr); + const confirmationReport = JSON.parse(confirmation.stdout); + assert.strictEqual(confirmationReport.status, "confirmation_required"); + assert.strictEqual(confirmationReport.coordinator_request_sent, false); + assert.strictEqual(confirmationReport.machine_error.category, "policy"); + assert.strictEqual( + confirmationReport.machine_error.process_exit_code_applied, + true + ); + assert( + confirmationReport.next_actions.some((action) => action.includes("--yes")) + ); +} + +main() + .then(() => { + console.log("CLI error exit smoke passed"); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/scripts/cli-happy-path-live-smoke.js b/scripts/cli-happy-path-live-smoke.js new file mode 100644 index 0000000..07e16d4 --- /dev/null +++ b/scripts/cli-happy-path-live-smoke.js @@ -0,0 +1,2909 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const https = require("https"); +const os = require("os"); +const path = require("path"); +const { DapClient } = require("./dap-client"); +const { agentIdentity, signedAgentWorkflowRequest } = require("./agent-signing"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { + nodeIdentity, + nodeIdentityFromPrivateKey, + signedNodeHeartbeat, + signedNodeRequest, +} = require("./node-signing"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +configurePodmanTestEnvironment(repo); +if ( + !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && + cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && + cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 +) { + cp.execFileSync( + "nix", + ["shell", "nixpkgs#podman", "--command", "node", __filename], + { + cwd: repo, + env: { ...process.env, CLUSTERFLUX_PODMAN_NIX_SHELL: "1" }, + stdio: "inherit", + } + ); + process.exit(0); +} + +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const acceptanceRoot = path.join(repo, "target/acceptance"); +const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = path.join(acceptanceRoot, "cli-happy-path-live.json"); +const serviceEndpoint = "https://clusterflux.michelpaulissen.com"; +const serviceAddr = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR || + "clusterflux.michelpaulissen.com:443"; +const browserOpenCommand = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND; +const reuseSessionFile = + process.env.CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE; +const enabled = process.env.CLUSTERFLUX_CLI_HAPPY_PATH_LIVE === "1"; +const strictFullRelease = process.env.CLUSTERFLUX_STRICT_FULL_RELEASE === "1"; +const strictVpsRestart = process.env.CLUSTERFLUX_STRICT_VPS_RESTART === "1"; +const strictVpsHost = process.env.CLUSTERFLUX_STRICT_VPS_HOST; +const strictVpsIdentity = process.env.CLUSTERFLUX_STRICT_VPS_IDENTITY; +const strictServiceUnit = + process.env.CLUSTERFLUX_STRICT_SERVICE_UNIT || + "clusterflux-hosted.service"; +const strictSoakSeconds = Number( + process.env.CLUSTERFLUX_STRICT_SOAK_SECONDS || "300" +); +const commands = []; + +function requireEnabled() { + if (!enabled) { + throw new Error( + "CLUSTERFLUX_CLI_HAPPY_PATH_LIVE=1 is required because this runs the live CLI happy path" + ); + } + if (!browserOpenCommand && !reuseSessionFile) { + throw new Error( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND or CLUSTERFLUX_CLI_HAPPY_PATH_REUSE_SESSION_FILE is required" + ); + } + if ( + strictFullRelease && + !process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE + ) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_SECOND_TENANT_SESSION_FILE for live isolation and quota independence" + ); + } + if (strictFullRelease && !process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE" + ); + } + if (strictFullRelease && !strictVpsRestart) { + throw new Error( + "CLUSTERFLUX_STRICT_FULL_RELEASE=1 requires CLUSTERFLUX_STRICT_VPS_RESTART=1" + ); + } + if (strictVpsRestart && (!strictVpsHost || !strictVpsIdentity)) { + throw new Error( + "strict VPS restart requires CLUSTERFLUX_STRICT_VPS_HOST and CLUSTERFLUX_STRICT_VPS_IDENTITY" + ); + } + if ( + !Number.isInteger(strictSoakSeconds) || + strictSoakSeconds < 120 || + strictSoakSeconds > 1800 + ) { + throw new Error("CLUSTERFLUX_STRICT_SOAK_SECONDS must be an integer from 120 to 1800"); + } +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function sha256(bytes) { + return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`; +} + +function nonInteractiveGitEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function commandEnv(command, env) { + if (command !== "git") return env; + return nonInteractiveGitEnv(env); +} + +function recordCommand(command, args) { + const redacted = []; + let hideNext = false; + for (const argument of args) { + if (hideNext) { + redacted.push(""); + hideNext = false; + } else { + redacted.push(argument); + hideNext = ["--enrollment-grant", "--admin-token"].includes(argument); + } + } + commands.push([command, ...redacted].join(" ")); +} + +function run(command, args, options = {}) { + recordCommand(command, args); + return cp.execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 15 * 60 * 1000, + ...options, + env: commandEnv(command, options.env), + }); +} + +function parseJsonOutput(output) { + const trimmed = output.trim(); + if (!trimmed) throw new Error("expected JSON output, got empty stdout"); + try { + return JSON.parse(trimmed); + } catch (_) { + // Commands may print progress before their final JSON report. + } + const lines = trimmed.split(/\r?\n/).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + try { + return JSON.parse(lines.slice(index).join("\n")); + } catch (_) { + // Keep scanning for the trailing JSON value. + } + } + throw new Error(`could not parse JSON output:\n${trimmed}`); +} + +function runJson(command, args, options = {}) { + return parseJsonOutput(run(command, args, options)); +} + +function authenticatedRequest(sessionSecret, request) { + return { + type: "authenticated", + session_secret: sessionSecret, + request, + }; +} + +function sendHostedControl(payload) { + const body = Buffer.from( + JSON.stringify(coordinatorWireRequest(payload, "strict-live")) + ); + const url = new URL("/api/v1/control", serviceEndpoint); + return new Promise((resolve, reject) => { + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + }, + timeout: 30_000, + }, + (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => { + const responseBody = Buffer.concat(chunks).toString("utf8"); + if (response.statusCode < 200 || response.statusCode >= 300) { + reject( + new Error( + `hosted control HTTP ${response.statusCode}: ${responseBody}` + ) + ); + return; + } + try { + resolve(JSON.parse(responseBody)); + } catch (error) { + reject( + new Error(`hosted control returned invalid JSON: ${error.message}`) + ); + } + }); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted control request timed out")) + ); + request.on("error", reject); + request.end(body); + }); +} + +function sendHostedControlDroppingResponse(payload) { + const body = Buffer.from( + JSON.stringify(coordinatorWireRequest(payload, "strict-live-dropped-response")) + ); + const url = new URL("/api/v1/control", serviceEndpoint); + return new Promise((resolve, reject) => { + let settled = false; + const request = https.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": body.length, + }, + timeout: 30_000, + }, + (response) => { + settled = true; + const statusCode = response.statusCode; + response.destroy(); + resolve({ status_code: statusCode, response_body_consumed: false }); + } + ); + request.on("timeout", () => + request.destroy(new Error("hosted dropped-response request timed out")) + ); + request.on("error", (error) => { + if (!settled) reject(error); + }); + request.end(body); + }); +} + +function assertDenied(response, pattern, label) { + assert.strictEqual(response.type, "error", `${label}: ${JSON.stringify(response)}`); + assert.match(response.message, pattern, `${label}: ${JSON.stringify(response)}`); + return { + denied: true, + category: response.error?.category || null, + message: response.message, + }; +} + +function readNodeCredential(projectDir, node) { + const directory = path.join(projectDir, ".clusterflux", "nodes"); + for (const file of fs.readdirSync(directory)) { + if (!file.endsWith(".json")) continue; + const absolute = path.join(directory, file); + const credential = readJson(absolute); + if (credential.node === node) return { absolute, credential }; + } + throw new Error(`persisted credential for ${node} was not found`); +} + +function directoryStats(root) { + if (!fs.existsSync(root)) return { files: 0, bytes: 0 }; + let files = 0; + let bytes = 0; + const visit = (directory) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile()) { + files += 1; + bytes += fs.statSync(absolute).size; + } + } + }; + visit(root); + return { files, bytes }; +} + +function sshArgs(...remoteArgs) { + assert(strictVpsHost && strictVpsIdentity); + return [ + "-i", + path.resolve(strictVpsIdentity.replace(/^~(?=\/)/, os.homedir())), + "-o", + "BatchMode=yes", + strictVpsHost, + ...remoteArgs, + ]; +} + +function strictInfrastructureSample(projectDir) { + const memory = Object.fromEntries( + run( + "ssh", + sshArgs( + "systemctl", + "show", + strictServiceUnit, + "--property=MemoryCurrent", + "--property=MemoryPeak" + ) + ) + .trim() + .split(/\r?\n/) + .map((line) => line.split("=")) + ); + const serviceDisk = Number( + run( + "ssh", + sshArgs("du", "-sb", "/var/lib/clusterflux-public-release") + ) + .trim() + .split(/\s+/)[0] + ); + const database = run( + "ssh", + sshArgs( + "runuser", + "-u", + "postgres", + "--", + "psql", + "-d", + "clusterflux", + "-AtF," + ), + { + input: + "SELECT pg_database_size('clusterflux'), (SELECT count(*) FROM clusterflux_tenants), (SELECT count(*) FROM clusterflux_projects), (SELECT count(*) FROM clusterflux_node_identities), (SELECT count(*) FROM clusterflux_cli_sessions);\n", + stdio: ["pipe", "pipe", "pipe"], + } + ) + .trim() + .split(",") + .map(Number); + assert(Number.isFinite(Number(memory.MemoryCurrent))); + assert(Number.isFinite(Number(memory.MemoryPeak))); + assert.strictEqual(database.length, 5); + return { + sampled_at_epoch_ms: Date.now(), + service_memory_current_bytes: Number(memory.MemoryCurrent), + service_memory_peak_bytes: Number(memory.MemoryPeak), + service_state_disk_bytes: serviceDisk, + postgres_database_bytes: database[0], + durable_rows: { + tenants: database[1], + projects: database[2], + node_identities: database[3], + cli_sessions: database[4], + }, + local_node_state: directoryStats(path.join(projectDir, ".clusterflux")), + }; +} + +function executable(root, name) { + return path.join(root, "bin", process.platform === "win32" ? `${name}.exe` : name); +} + +function binaryArchive(manifest) { + const platform = `${os.platform()}-${os.arch()}`; + const asset = manifest.assets.find( + (candidate) => + candidate.name.startsWith("clusterflux-public-binaries-") && + candidate.name.includes(platform) + ); + assert(asset, `missing released binary archive for ${platform}`); + assert(fs.existsSync(asset.file), `missing released binary archive ${asset.file}`); + return asset.file; +} + +function publicRepoUrl(manifest) { + const url = + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + manifest.public_repo_url || + manifest.public_repo_remote; + assert(url, "public repository URL is required"); + assert(url.includes("git.michelpaulissen.com")); + return url; +} + +function stagePublicCheckout(manifest, checkout) { + const candidateTree = process.env.CLUSTERFLUX_LIVE_PUBLIC_TREE; + if (candidateTree) { + const source = path.resolve(candidateTree); + assert.strictEqual( + source, + path.resolve(manifest.public_tree), + "explicit live candidate tree must be the tree recorded by the release manifest" + ); + fs.cpSync(source, checkout, { recursive: true }); + return { + kind: "local_filtered_release_candidate", + path: source, + published: false, + }; + } + run("git", ["clone", "--depth", "1", publicRepoUrl(manifest), checkout]); + return { + kind: "published_forgejo_checkout", + url: publicRepoUrl(manifest), + published: true, + }; +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function waitForCli(label, action, predicate, timeoutMs = 10 * 60 * 1000) { + const deadline = Date.now() + timeoutMs; + let lastValue; + let lastError; + while (Date.now() < deadline) { + try { + lastValue = action(); + if (predicate(lastValue)) return lastValue; + lastError = undefined; + } catch (error) { + lastError = error; + } + await delay(250); + } + throw new Error( + `${label} timed out; last value=${JSON.stringify(lastValue)}${ + lastError ? `; last error=${lastError.message}` : "" + }` + ); +} + +function spawnJsonLines(command, args, options) { + recordCommand(command, args); + const child = cp.spawn(command, args, { + ...options, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + const values = []; + const waiters = []; + + function deliver(value) { + values.push(value); + for (let index = waiters.length - 1; index >= 0; index -= 1) { + const waiter = waiters[index]; + if (waiter.predicate(value)) { + waiters.splice(index, 1); + clearTimeout(waiter.timer); + waiter.resolve(value); + } + } + } + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + while (stdout.includes("\n")) { + const newline = stdout.indexOf("\n"); + const line = stdout.slice(0, newline).trim(); + stdout = stdout.slice(newline + 1); + if (!line) continue; + try { + deliver(JSON.parse(line)); + } catch (_) { + // Non-JSON progress stays out of the evidence report. + } + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + function waitFor(predicate, label, timeoutMs = 120000) { + const existing = values.find(predicate); + if (existing) return Promise.resolve(existing); + return new Promise((resolve, reject) => { + const waiter = { predicate, resolve, reject, timer: undefined }; + waiter.timer = setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) waiters.splice(index, 1); + reject(new Error(`${label} timed out\n${stderr}`)); + }, timeoutMs); + waiters.push(waiter); + }); + } + + child.once("exit", (code) => { + for (const waiter of waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(new Error(`${waiter.label || "child"} exited with ${code}\n${stderr}`)); + } + }); + return { child, values, waitFor, stderr: () => stderr }; +} + +async function stopChild(child) { + if (!child || child.exitCode !== null) return; + child.kill("SIGTERM"); + const exited = new Promise((resolve) => child.once("exit", resolve)); + const forced = delay(5000).then(() => { + if (child.exitCode === null) child.kill("SIGKILL"); + }); + await Promise.race([exited, forced]); + if (child.exitCode === null) await exited; +} + +function rawTaskEvents(report) { + return report?.events?.response?.events || []; +} + +function nodeDescriptor(status, node) { + return status?.response?.descriptors?.find( + (descriptor) => descriptor.id === node || descriptor.node === node + ); +} + +function completedFlagship(events) { + const completedDefinitions = new Set( + events + .filter((event) => event.terminal_state === "completed") + .map((event) => event.task_definition) + ); + return ( + completedDefinitions.has("prepare_source") && + completedDefinitions.has("compile_linux") && + completedDefinitions.has("package_release") && + events.some( + (event) => + event.executor === "coordinator_main" && event.terminal_state === "completed" + ) + ); +} + +function contentAddressedArtifactId(event, logicalName) { + const digest = event?.artifact_digest; + if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) { + return null; + } + const artifactId = `${logicalName}-${digest.slice("sha256:".length)}`; + return event.artifact_path?.endsWith(`/vfs/artifacts/${artifactId}`) + ? artifactId + : null; +} + +async function prepareLiveNodeCredentialSecurity({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, +}) { + const node = `security-node-${suffix}`; + const grant = runJson(clusterflux, ["node", "enroll", ...scope], { + cwd: projectDir, + }); + const attach = runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + node, + "--enrollment-grant", + grant.enrollment_grant.grant, + "--json", + ], + { cwd: projectDir } + ); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + const stored = readNodeCredential(projectDir, node); + const identity = nodeIdentityFromPrivateKey(stored.credential.private_key); + assert.strictEqual(identity.publicKey, stored.credential.public_key); + + const heartbeat = { + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity, { + nonce: `strict-valid-${suffix}`, + }), + }; + const valid = await sendHostedControl(heartbeat); + assert.strictEqual(valid.type, "node_heartbeat"); + const replay = assertDenied( + await sendHostedControl(heartbeat), + /nonce.*already.*used|replay/i, + "replayed node credential" + ); + + const expired = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity, { + nonce: `strict-expired-${suffix}`, + issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, + }), + }), + /expired|clock skew/i, + "expired node credential" + ); + + const forgedIdentity = nodeIdentity("strict-forged-node", node); + const forged = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, forgedIdentity, { + nonce: `strict-forged-${suffix}`, + }), + }), + /signature|public key/i, + "forged node credential" + ); + + const originalCapabilityBody = { + type: "report_node_capabilities", + tenant, + project, + node, + capabilities: { + os: "Linux", + arch: process.arch === "x64" ? "x86_64" : process.arch, + capabilities: [], + environment_backends: [], + source_providers: [], + }, + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false, + online: true, + }; + const modifiedEnvelope = signedNodeRequest( + node, + identity, + "report_node_capabilities", + originalCapabilityBody, + { nonce: `strict-modified-${suffix}` } + ); + modifiedEnvelope.request.online = false; + const bodyModified = assertDenied( + await sendHostedControl(modifiedEnvelope), + /signature/i, + "body-modified node credential" + ); + + return { + node, + identity, + credential_path: stored.absolute, + credential_digest: sha256(fs.readFileSync(stored.absolute)), + capability_body: originalCapabilityBody, + evidence: { + valid_request: valid.type, + replay, + expired, + forged, + body_modified: bodyModified, + }, + }; +} + +async function runLiveAgentCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + suffix, + bundle, + securityNode, + workerRuntime, +}) { + const agent = `security-agent-${suffix}`; + const identity = agentIdentity("strict-live-agent", agent); + const added = runJson( + clusterflux, + ["key", "add", ...scope, "--agent", agent, "--public-key", identity.publicKey], + { cwd: projectDir } + ); + assert.strictEqual(added.command, "key add"); + + const processId = `vp-agent-security-${suffix}`; + const unsigned = { + type: "start_process", + tenant, + project, + actor_agent: agent, + process: processId, + restart: false, + }; + const validRequest = signedAgentWorkflowRequest(identity, unsigned, { + nonce: `strict-agent-valid-${suffix}`, + }); + const valid = await sendHostedControl(validRequest); + assert.strictEqual(valid.type, "process_started", JSON.stringify(valid)); + const replay = assertDenied( + await sendHostedControl(validRequest), + /nonce.*already.*used|replay/i, + "replayed agent credential" + ); + + const expired = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-expired` }, + { + nonce: `strict-agent-expired-${suffix}`, + issuedAtEpochSeconds: Math.floor(Date.now() / 1000) - 3600, + } + ) + ), + /expired|clock skew/i, + "expired agent credential" + ); + + const modifiedRequest = signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-modified` }, + { nonce: `strict-agent-modified-${suffix}` } + ); + modifiedRequest.restart = true; + const bodyModified = assertDenied( + await sendHostedControl(modifiedRequest), + /signature/i, + "body-modified agent credential" + ); + + const forgedIdentity = agentIdentity("strict-live-forged-agent", agent); + const forged = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + forgedIdentity, + { ...unsigned, process: `${processId}-forged` }, + { + nonce: `strict-agent-forged-${suffix}`, + publicKeyFingerprint: identity.publicKeyFingerprint, + } + ) + ), + /signature|public key/i, + "forged agent credential" + ); + + const fakeEnvironmentDigest = sha256( + Buffer.from(`strict-debug-missing-participant-${suffix}`) + ); + const fakeCapabilityBody = { + ...securityNode.capability_body, + cached_environment_digests: [fakeEnvironmentDigest], + online: true, + }; + const mainTask = `ti:${processId}:main`; + const fakeTask = `missing-debug-participant-${suffix}`; + const mainLaunchBody = { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: { + tenant, + project, + process: processId, + task_definition: bundle.entryStableId, + task_instance: mainTask, + dispatch: { + kind: "coordinator_node_wasm", + export: bundle.entryExport, + abi: "entrypoint_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: true, + artifact_path: `/vfs/artifacts/${mainTask}-output.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + let mainLaunch; + let directTaskV1; + let fakeLaunch; + let liveParentTask; + let workerPaused = false; + try { + mainLaunch = await sendHostedControl( + signedAgentWorkflowRequest(identity, mainLaunchBody, { + nonce: `strict-agent-main-${suffix}`, + }) + ); + assert.strictEqual(mainLaunch.type, "main_launched", JSON.stringify(mainLaunch)); + assert.strictEqual(mainLaunch.task_instance, mainTask); + assert.strictEqual(mainLaunch.state, "running"); + assert.strictEqual(mainLaunch.actor.kind, "agent"); + assert.strictEqual(mainLaunch.actor.authenticated_without_browser, true); + + const liveParentAssignment = await workerRuntime.worker.waitFor( + (value) => + value.node_status === "assignment_started" && + value.node === workerRuntime.node && + value.process === processId && + value.task_assignment_response?.task_spec?.dispatch?.abi === "task_v1", + "agent main to dispatch a live child to the real worker", + 120000 + ); + liveParentTask = liveParentAssignment.virtual_thread; + assert.strictEqual(typeof liveParentTask, "string"); + assert.notStrictEqual(liveParentTask, ""); + assert.strictEqual( + workerRuntime.child.kill("SIGSTOP"), + true, + "failed to pause the real worker after observing its live child" + ); + workerPaused = true; + + const fakeCapabilities = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_node_capabilities", + fakeCapabilityBody, + { nonce: `strict-agent-fake-capabilities-${suffix}` } + ) + ); + assert.strictEqual( + fakeCapabilities.type, + "node_capabilities_recorded", + JSON.stringify(fakeCapabilities) + ); + + const fakeTaskSpec = { + tenant, + project, + process: processId, + task_definition: "task_add_one", + task_instance: fakeTask, + dispatch: { + kind: "coordinator_node_wasm", + export: null, + abi: "task_v1", + }, + environment_id: "strict-debug-missing-participant", + environment: null, + environment_digest: fakeEnvironmentDigest, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [{ SmallJson: 41 }], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }; + directTaskV1 = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { + type: "launch_task", + tenant, + project, + actor_agent: agent, + task_spec: fakeTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `strict-agent-direct-task-v1-${suffix}` } + ) + ), + /external callers.*EntrypointV1|TaskV1 requires an authenticated live parent/i, + "external direct TaskV1 launch" + ); + + const fakeLaunchBody = { + type: "launch_child_task", + tenant, + project, + process: processId, + node: workerRuntime.node, + parent_task: liveParentTask, + task_spec: fakeTaskSpec, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}.txt`, + wasm_module_base64: bundle.moduleBase64, + }; + fakeLaunch = await sendHostedControl( + signedNodeRequest( + workerRuntime.node, + workerRuntime.identity, + "launch_child_task", + fakeLaunchBody, + { nonce: `strict-parent-runtime-fake-task-${suffix}` } + ) + ); + assert.strictEqual(fakeLaunch.type, "task_launched", JSON.stringify(fakeLaunch)); + assert.strictEqual(fakeLaunch.placement.node, securityNode.node); + } finally { + if (workerPaused) { + assert.strictEqual( + workerRuntime.child.kill("SIGCONT"), + true, + "failed to resume the real worker after authenticated child launch" + ); + } + } + + const freezeStartedAt = Date.now(); + const freeze = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_debug_epoch", + process: processId, + stopped_task: fakeTask, + reason: "strict live missing debug participant injection", + }) + ); + assert.strictEqual(freeze.type, "debug_epoch", JSON.stringify(freeze)); + await delay(5500); + const partialFreeze = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "inspect_debug_epoch", + process: processId, + epoch: freeze.epoch, + }) + ); + assert.strictEqual(partialFreeze.type, "debug_epoch_status"); + assert.strictEqual(partialFreeze.partially_frozen, true, JSON.stringify(partialFreeze)); + assert.strictEqual(partialFreeze.fully_frozen, false); + assert.strictEqual(partialFreeze.failed, true); + assert.match( + partialFreeze.failure_messages.join("; "), + /did not acknowledge frozen state within \d+ ms/ + ); + + const mismatchedDigest = sha256(Buffer.from(`strict-nested-mismatch-${suffix}`)); + const mismatchedChild = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "launch_child_task", + { + type: "launch_child_task", + tenant, + project, + process: processId, + node: securityNode.node, + parent_task: fakeTask, + task_spec: { + tenant, + project, + process: processId, + task_definition: "task_add_one", + task_instance: `${fakeTask}:child:mismatched-environment`, + dispatch: { + kind: "coordinator_node_wasm", + export: null, + abi: "task_v1", + }, + environment_id: "strict-nested-mismatch", + environment: null, + environment_digest: mismatchedDigest, + required_capabilities: [], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [{ SmallJson: 41 }], + vfs_epoch: valid.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: false, + artifact_path: `/vfs/artifacts/${fakeTask}-mismatched-child.txt`, + wasm_module_base64: bundle.moduleBase64, + }, + { nonce: `strict-nested-mismatch-${suffix}` } + ) + ); + const nestedEnvironmentMismatch = assertDenied( + mismatchedChild, + /environment|digest|compatible|placement|no node/i, + "nested environment mismatch" + ); + + const resume = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "resume_debug_epoch", + process: processId, + epoch: freeze.epoch, + }) + ); + assert.strictEqual(resume.type, "debug_epoch", JSON.stringify(resume)); + await delay(1000); + const resumed = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "inspect_debug_epoch", + process: processId, + epoch: freeze.epoch, + }) + ); + assert.strictEqual(resumed.type, "debug_epoch_status"); + const resumedAcknowledgements = resumed.acknowledgements.filter( + (acknowledgement) => acknowledgement.state === "running" + ); + assert(resumedAcknowledgements.length > 0, JSON.stringify(resumed)); + + const completed = await waitForCli( + "agent-authenticated coordinator main and child workflow completion", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { + cwd: projectDir, + }), + (tasks) => completedFlagship(rawTaskEvents(tasks)), + 120000 + ); + const completedEvents = rawTaskEvents(completed); + const concurrentCompileTasks = [ + ...new Set( + completedEvents + .filter( + (event) => + event.task_definition === "compile_linux" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + assert(concurrentCompileTasks.length >= 2, JSON.stringify(completedEvents)); + + const revoked = runJson( + clusterflux, + ["key", "revoke", ...scope, "--agent", agent, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(revoked.command, "key revoke"); + const revokedUse = assertDenied( + await sendHostedControl( + signedAgentWorkflowRequest( + identity, + { ...unsigned, process: `${processId}-revoked` }, + { nonce: `strict-agent-revoked-${suffix}` } + ) + ), + /revoked/i, + "revoked agent credential" + ); + return { + agent, + valid_request: valid.type, + replay, + expired, + forged, + body_modified: bodyModified, + revoked: revokedUse, + workflow: { + process: processId, + main_launch: mainLaunch.type, + external_direct_task_v1: directTaskV1, + child_launch: fakeLaunch.type, + child_parent: liveParentTask, + actor_kind: mainLaunch.actor.kind, + authenticated_without_browser: mainLaunch.actor.authenticated_without_browser, + flagship_completed: completedFlagship(completedEvents), + concurrent_compile_tasks: concurrentCompileTasks, + }, + partial_freeze: { + epoch: freeze.epoch, + elapsed_ms: Date.now() - freezeStartedAt, + partially_frozen: partialFreeze.partially_frozen, + fully_frozen: partialFreeze.fully_frozen, + warning: partialFreeze.failure_messages, + resumed_participants: resumedAcknowledgements.map( + (acknowledgement) => acknowledgement.task + ), + }, + nested_environment_mismatch: nestedEnvironmentMismatch, + }; +} + +async function runSecondTenantIsolation({ + firstTenant, + firstProject, + firstProcess, + firstNode, + firstArtifact, + manifest, +}) { + const file = process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE; + const evidenceFile = process.env.CLUSTERFLUX_SECOND_TENANT_EVIDENCE_FILE; + if (!file && evidenceFile) { + const evidenceBytes = fs.readFileSync(path.resolve(evidenceFile)); + const previous = JSON.parse(evidenceBytes); + const isolation = previous.tenant_isolation; + assert.strictEqual(isolation?.executed, true); + assert.notStrictEqual(isolation.second_tenant, firstTenant); + assert.strictEqual(isolation.project?.denied, true); + assert.strictEqual(isolation.process_hidden, true); + assert.strictEqual(isolation.node_hidden, true); + assert.strictEqual(isolation.tasks_and_logs?.denied, true); + assert.strictEqual(isolation.debug?.denied, true); + assert.strictEqual(isolation.debug?.audited, true); + assert.strictEqual(isolation.debug?.charged_debug_read_bytes, 0); + assert.strictEqual(isolation.artifact_and_download?.denied, true); + assert.strictEqual(isolation.process_control?.denied, true); + const previousDeployment = previous.release_binding?.deployment; + const currentDeployment = deploymentProvenance(); + assert.strictEqual( + previousDeployment?.hosted_service_sha256, + currentDeployment.hosted_service_sha256, + "reused isolation evidence must target the exact deployed hosted binary" + ); + assert.deepStrictEqual( + previous.release_binding?.binary_digests, + manifest.binary_digests, + "reused isolation evidence must target the exact public binaries" + ); + return { + ...isolation, + reused_from_immutable_evidence: { + report_sha256: sha256(evidenceBytes), + source_commit: previous.source_commit, + hosted_service_sha256: currentDeployment.hosted_service_sha256, + public_binary_digests: manifest.binary_digests, + }, + }; + } + if (!file) return { executed: false, reason: "second tenant session not supplied" }; + const second = readJson(path.resolve(file)); + assert.strictEqual(second.coordinator, serviceEndpoint); + assert(second.session_secret, "second tenant session omitted its session secret"); + assert.notStrictEqual( + second.tenant, + firstTenant, + "isolation proof requires a genuinely distinct hosted tenant" + ); + const call = (request) => + sendHostedControl(authenticatedRequest(second.session_secret, request)); + + const project = assertDenied( + await call({ type: "select_project", project: firstProject }), + /outside|not visible|tenant|project/i, + "cross-tenant project selection" + ); + const processes = await call({ type: "list_processes" }); + assert.strictEqual( + processes.type, + "process_statuses", + JSON.stringify(processes) + ); + assert(!JSON.stringify(processes).includes(firstProcess)); + const nodes = await call({ type: "list_node_descriptors" }); + assert.strictEqual(nodes.type, "node_descriptors", JSON.stringify(nodes)); + assert(!JSON.stringify(nodes).includes(firstNode)); + const tasksAndLogs = assertDenied( + await call({ type: "list_task_events", process: firstProcess }), + /outside|scope|tenant|project|not active|requires an active|unknown/i, + "cross-tenant task and log listing" + ); + const debugResponse = await call({ + type: "debug_attach", + process: firstProcess, + }); + assert.strictEqual(debugResponse.type, "debug_attach", JSON.stringify(debugResponse)); + assert.strictEqual(debugResponse.authorization?.allowed, false); + assert.strictEqual(debugResponse.audit_event?.allowed, false); + assert.match( + debugResponse.authorization?.reason || "", + /outside|scope|permission|tenant|project|not active|unknown/i + ); + assert.strictEqual(debugResponse.charged_debug_read_bytes, 0); + const debug = { + denied: true, + reason: debugResponse.authorization.reason, + audited: true, + charged_debug_read_bytes: debugResponse.charged_debug_read_bytes, + }; + const artifact = assertDenied( + await call({ + type: "create_artifact_download_link", + artifact: firstArtifact, + max_bytes: 1024, + ttl_seconds: 60, + }), + /outside|scope|tenant|project|not found|unavailable/i, + "cross-tenant artifact download" + ); + const control = assertDenied( + await call({ type: "abort_process", process: firstProcess }), + /outside|scope|tenant|project|not active|requires an active|unknown/i, + "cross-tenant process control" + ); + return { + executed: true, + second_tenant: second.tenant, + project, + process_hidden: true, + node_hidden: true, + tasks_and_logs: tasksAndLogs, + debug, + artifact_and_download: artifact, + process_control: control, + }; +} + +async function runLiveSoak({ + clusterflux, + projectDir, + scope, + securityNode, +}) { + if (!strictVpsRestart) { + return { executed: false, reason: "strict VPS measurement access not supplied" }; + } + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const processId = runReport.process; + const parked = await waitForCli( + "soak coordinator main to park without a capable node", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => + status.live_process?.main_state === "running" && + status.live_process?.main_wait_state === "waiting_for_task" && + status.current_task_count === 0 && + status.live_process?.connected_nodes?.length === 0, + 120000 + ); + + const pollSecurityNode = (nonce) => + sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "poll_task_assignment", + { + type: "poll_task_assignment", + tenant: securityNode.capability_body.tenant, + project: securityNode.capability_body.project, + node: securityNode.node, + }, + { nonce } + ) + ); + const drainedPriorAssignments = []; + for (let drainIndex = 0; drainIndex < 16; drainIndex += 1) { + const poll = await pollSecurityNode( + `strict-soak-drain-${drainIndex}-${Date.now()}` + ); + assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); + if (poll.assignment === null) break; + assert.notStrictEqual( + poll.assignment.process, + processId, + "soak process unexpectedly received a task assignment" + ); + drainedPriorAssignments.push({ + process: poll.assignment.process, + task: poll.assignment.task, + }); + assert(drainIndex < 15, "pre-soak assignment drain did not quiesce"); + } + + const startedAt = Date.now(); + const deadline = startedAt + strictSoakSeconds * 1000; + const samples = []; + let sampleIndex = 0; + while (true) { + const capabilities = await sendHostedControl( + signedNodeRequest( + securityNode.node, + securityNode.identity, + "report_node_capabilities", + securityNode.capability_body, + { nonce: `strict-soak-capabilities-${sampleIndex}-${Date.now()}` } + ) + ); + assert.strictEqual( + capabilities.type, + "node_capabilities_recorded", + JSON.stringify(capabilities) + ); + const poll = await pollSecurityNode( + `strict-soak-poll-${sampleIndex}-${Date.now()}` + ); + assert.strictEqual(poll.type, "task_assignment", JSON.stringify(poll)); + assert.strictEqual(poll.assignment, null); + const status = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.strictEqual(status.live_process.main_wait_state, "waiting_for_task"); + samples.push(strictInfrastructureSample(projectDir)); + sampleIndex += 1; + if (Date.now() >= deadline) break; + await delay(Math.min(30_000, deadline - Date.now())); + } + + const currentMemory = samples.map( + (sample) => sample.service_memory_current_bytes + ); + const serviceDisk = samples.map((sample) => sample.service_state_disk_bytes); + const databaseDisk = samples.map((sample) => sample.postgres_database_bytes); + const localDisk = samples.map((sample) => sample.local_node_state.bytes); + const spread = (values) => Math.max(...values) - Math.min(...values); + assert(spread(currentMemory) <= 128 * 1024 * 1024); + assert(spread(serviceDisk) <= 16 * 1024 * 1024); + assert(spread(databaseDisk) <= 32 * 1024 * 1024); + assert(spread(localDisk) <= 16 * 1024 * 1024); + assert.deepStrictEqual( + samples.at(-1).durable_rows, + samples[0].durable_rows, + "durable object counts grew during the parked-process soak" + ); + const aborted = runJson( + clusterflux, + ["process", "abort", ...scope, "--process", processId, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(aborted.abort_request.process_slot_released, true); + return { + executed: true, + process: processId, + duration_seconds: Math.floor((Date.now() - startedAt) / 1000), + sample_interval_seconds: 30, + sample_count: samples.length, + parked_main: { + main_state: parked.live_process.main_state, + main_wait_state: parked.live_process.main_wait_state, + }, + drained_prior_assignments: drainedPriorAssignments, + signed_node_poll_cycles: samples.length, + memory_spread_bytes: spread(currentMemory), + service_disk_spread_bytes: spread(serviceDisk), + postgres_disk_spread_bytes: spread(databaseDisk), + local_node_state_spread_bytes: spread(localDisk), + samples, + }; +} + +async function revokeSecurityNode({ + clusterflux, + projectDir, + scope, + securityNode, +}) { + const revoked = runJson( + clusterflux, + ["node", "revoke", ...scope, "--node", securityNode.node, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(revoked.command, "node revoke"); + const denied = assertDenied( + await sendHostedControl({ + type: "node_heartbeat", + node: securityNode.node, + node_signature: signedNodeHeartbeat( + securityNode.node, + securityNode.identity, + { nonce: `strict-node-revoked-${Date.now()}` } + ), + }), + /revoked|unknown node|not recognized|not enrolled/i, + "revoked node credential" + ); + return { node: securityNode.node, denied }; +} + +async function runLiveQuotaPreallocation({ sessionSecret, suffix }) { + const readSpawnQuota = async () => { + const status = await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "quota_status" }) + ); + assert.strictEqual(status.type, "quota_status", JSON.stringify(status)); + const limit = status.limits?.limits?.Spawn; + const usage = status.usage?.Spawn; + const windowSeconds = status.window_seconds?.Spawn; + const windowStarted = status.window_started_epoch_seconds?.Spawn; + assert(Number.isInteger(limit) && limit > 0, JSON.stringify(status)); + assert(Number.isInteger(usage) && usage >= 0, JSON.stringify(status)); + assert( + Number.isInteger(windowSeconds) && windowSeconds > 0, + JSON.stringify(status) + ); + assert(Number.isInteger(windowStarted), JSON.stringify(status)); + return { limit, usage, windowSeconds, windowStarted }; + }; + + let spawnQuota = await readSpawnQuota(); + const windowElapsed = Math.max( + 0, + Math.floor(Date.now() / 1000) - spawnQuota.windowStarted + ); + if (spawnQuota.usage > 0 || windowElapsed > 1) { + await delay( + Math.max(1000, (spawnQuota.windowSeconds - windowElapsed + 1) * 1000) + ); + spawnQuota = await readSpawnQuota(); + } + let denial; + let deniedProcess; + let acceptedStarts = 0; + const maxAttempts = spawnQuota.limit - spawnQuota.usage + 2; + for (let index = 0; index < maxAttempts; index += 1) { + const processId = `vp-quota-${suffix}-${index}`; + const started = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "start_process", + process: processId, + restart: false, + }) + ); + if (started.type === "error") { + assert.match(started.message, /quota|spawn|limit|community tier/i); + denial = started; + deniedProcess = processId; + break; + } + assert.strictEqual(started.type, "process_started", JSON.stringify(started)); + acceptedStarts += 1; + const aborted = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "abort_process", + process: processId, + }) + ); + assert.strictEqual(aborted.type, "process_aborted", JSON.stringify(aborted)); + } + assert( + denial, + `live community-tier spawn quota was not reached in ${maxAttempts} attempts` + ); + const processes = await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "list_processes" }) + ); + assert.strictEqual( + processes.type, + "process_statuses", + JSON.stringify(processes) + ); + assert( + !JSON.stringify(processes).includes(deniedProcess), + "quota-denied process was allocated before denial" + ); + const second = readJson( + path.resolve(process.env.CLUSTERFLUX_SECOND_TENANT_SESSION_FILE) + ); + assert.notStrictEqual(second.session_secret, sessionSecret); + const unrelatedProcess = `vp-unrelated-quota-${suffix}`; + const unrelatedStarted = await sendHostedControl( + authenticatedRequest(second.session_secret, { + type: "start_process", + process: unrelatedProcess, + restart: false, + }) + ); + assert.strictEqual( + unrelatedStarted.type, + "process_started", + `first tenant quota exhaustion affected another tenant: ${JSON.stringify(unrelatedStarted)}` + ); + const unrelatedAborted = await sendHostedControl( + authenticatedRequest(second.session_secret, { + type: "abort_process", + process: unrelatedProcess, + }) + ); + assert.strictEqual( + unrelatedAborted.type, + "process_aborted", + JSON.stringify(unrelatedAborted) + ); + return { + limit: spawnQuota.limit, + initial_usage: spawnQuota.usage, + window_seconds: spawnQuota.windowSeconds, + accepted_starts_before_denial: acceptedStarts, + denied_process: deniedProcess, + denial: { + type: denial.type, + category: denial.error?.category || null, + message: denial.message, + }, + denied_process_allocated: false, + unrelated_tenant: { + tenant: second.tenant, + process: unrelatedProcess, + start: unrelatedStarted.type, + abort: unrelatedAborted.type, + unaffected_by_first_tenant_quota: true, + }, + }; +} + +async function runLongJoinProof({ clusterflux, projectDir, scope }) { + const startedAt = Date.now(); + const runReport = runJson( + clusterflux, + ["run", "long-join", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const terminal = await waitForCli( + "controlled task longer than two minutes", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (tasks) => + rawTaskEvents(tasks).some( + (event) => + event.task_definition === "long_join_probe" && + event.terminal_state === "completed" + ), + 5 * 60 * 1000 + ); + const completed = rawTaskEvents(terminal).find( + (event) => + event.task_definition === "long_join_probe" && + event.terminal_state === "completed" + ); + const durationSeconds = Math.floor((Date.now() - startedAt) / 1000); + assert(durationSeconds > 120); + const released = await waitForCli( + "long join process automatic slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + return { + process: runReport.process, + task: completed.task, + terminal_state: completed.terminal_state, + result: completed.result, + duration_seconds: durationSeconds, + process_state: released.state, + }; +} + +async function runRepeatedParkWakeProof({ clusterflux, projectDir, scope }) { + const runReport = runJson( + clusterflux, + ["run", "park-wake", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.status, "main_launched"); + const terminal = await waitForCli( + "repeated coordinator-main park/wake completion", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (tasks) => { + const completed = new Set( + rawTaskEvents(tasks) + .filter( + (event) => + event.task_definition === "task_add_one" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ); + return completed.size >= 16; + }, + 120000 + ); + const completedTasks = [ + ...new Set( + rawTaskEvents(terminal) + .filter( + (event) => + event.task_definition === "task_add_one" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + const released = await waitForCli( + "park/wake process automatic slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", runReport.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + return { + process: runReport.process, + completed_cycles: completedTasks.length, + task_instances: completedTasks, + terminal_state: released.state, + }; +} + +async function runDroppedConnectionRollback({ + clusterflux, + projectDir, + scope, + sessionSecret, + suffix, +}) { + const processId = `vp-dropped-response-${suffix}`; + const dropped = await sendHostedControlDroppingResponse( + authenticatedRequest(sessionSecret, { + type: "start_process", + process: processId, + restart: false, + }) + ); + assert.strictEqual(dropped.response_body_consumed, false); + const committed = await waitForCli( + "process creation committed after dropped response", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state !== "not_active" && Boolean(status.live_process), + 30000 + ); + const rollback = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "abort_process", + process: processId, + }) + ); + assert.strictEqual(rollback.type, "process_aborted", JSON.stringify(rollback)); + const released = await waitForCli( + "dropped-response process rollback release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 30000 + ); + return { + process: processId, + response_body_consumed: dropped.response_body_consumed, + committed_state: committed.state, + rollback: rollback.type, + terminal_state: released.state, + }; +} + +async function runLiveBandwidthPreallocation({ + sessionSecret, + artifact, + maxBytes, +}) { + let denial; + let acceptedReservations = 0; + for (let index = 0; index < 64; index += 1) { + const response = await sendHostedControl( + authenticatedRequest(sessionSecret, { + type: "create_artifact_download_link", + artifact, + max_bytes: maxBytes, + ttl_seconds: 300, + }) + ); + if (response.type === "error") { + assert.match( + response.message, + /artifact relay.*(?:concurrency|period byte budget)|bandwidth|reservation/i + ); + denial = response; + break; + } + assert.strictEqual(response.type, "artifact_download_link", JSON.stringify(response)); + acceptedReservations += 1; + } + assert(denial, "artifact relay preallocation was not denied in 64 reservations"); + return { + artifact, + accepted_reservations_before_denial: acceptedReservations, + bytes_served_before_denial: 0, + denial: { + type: denial.type, + category: denial.error?.category || null, + message: denial.message, + }, + }; +} + +function deploymentProvenance() { + if (!strictVpsRestart) return null; + const systemGeneration = run( + "ssh", + sshArgs("readlink", "-f", "/run/current-system") + ).trim(); + const mainPid = run( + "ssh", + sshArgs( + "systemctl", + "show", + strictServiceUnit, + "--property=MainPID", + "--value" + ) + ).trim(); + assert.match(mainPid, /^[1-9][0-9]*$/, "hosted service has no live MainPID"); + const executable = run( + "ssh", + sshArgs("readlink", "-f", `/proc/${mainPid}/exe`) + ).trim(); + const binarySha256 = run( + "ssh", + sshArgs("sha256sum", `/proc/${mainPid}/exe`) + ) + .trim() + .split(/\s+/)[0]; + const fragment = run( + "ssh", + sshArgs("systemctl", "show", strictServiceUnit, "--property=FragmentPath", "--value") + ).trim(); + const serviceUnitSha256 = run( + "ssh", + sshArgs("sha256sum", fragment) + ) + .trim() + .split(/\s+/)[0]; + return { + system_generation: systemGeneration, + hosted_service_executable: executable, + hosted_service_sha256: `sha256:${binarySha256}`, + service_unit: fragment, + service_unit_sha256: `sha256:${serviceUnitSha256}`, + }; +} + +function configurationProvenance() { + const configRepo = path.resolve( + process.env.CLUSTERFLUX_DEPLOYMENT_CONFIG_REPO || + path.join(repo, "..", "michelpaulissen.com") + ); + const status = run("git", ["status", "--short"], { cwd: configRepo }).trim(); + return { + repository: configRepo, + revision: run("git", ["rev-parse", "HEAD"], { cwd: configRepo }).trim(), + clean: status === "", + status: status || null, + }; +} + +async function restartHostedServiceAndResume({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + workerNode, + credentialPath, + credentialDigest, +}) { + if (!strictVpsRestart) { + return { + executed: false, + reason: "strict VPS restart access not supplied", + worker: null, + }; + } + const before = deploymentProvenance(); + run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); + const deadline = Date.now() + 120000; + let ping; + let lastError; + while (Date.now() < deadline) { + try { + ping = await sendHostedControl({ type: "ping" }); + if (ping.type === "pong") break; + } catch (error) { + lastError = error; + } + await delay(500); + } + assert.strictEqual( + ping?.type, + "pong", + `hosted service did not recover after restart: ${lastError?.message || "no pong"}` + ); + const afterFirstRestart = deploymentProvenance(); + assert.strictEqual(afterFirstRestart.system_generation, before.system_generation); + assert.strictEqual( + afterFirstRestart.hosted_service_sha256, + before.hosted_service_sha256 + ); + + const projects = runJson(clusterflux, ["project", "list", ...scope], { + cwd: projectDir, + }); + assert(projects.project_count >= 1, "CLI session/project did not survive restart"); + const ephemeralRun = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(ephemeralRun.status, "main_launched"); + const ephemeralProcess = ephemeralRun.process; + await waitForCli( + "pre-restart ephemeral main to park", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", ephemeralProcess], + { cwd: projectDir } + ), + (status) => status.live_process?.main_wait_state === "waiting_for_node", + 120000 + ); + + run("ssh", sshArgs("systemctl", "restart", strictServiceUnit)); + const secondDeadline = Date.now() + 120000; + ping = undefined; + lastError = undefined; + while (Date.now() < secondDeadline) { + try { + ping = await sendHostedControl({ type: "ping" }); + if (ping.type === "pong") break; + } catch (error) { + lastError = error; + } + await delay(500); + } + assert.strictEqual( + ping?.type, + "pong", + `hosted service did not recover after ephemeral-state restart: ${ + lastError?.message || "no pong" + }` + ); + const after = deploymentProvenance(); + assert.strictEqual(after.system_generation, before.system_generation); + assert.strictEqual(after.hosted_service_sha256, before.hosted_service_sha256); + const oldStatus = runJson( + clusterflux, + ["process", "status", ...scope, "--process", ephemeralProcess], + { cwd: projectDir } + ); + assert.strictEqual(oldStatus.state, "not_active"); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const worker = spawnJsonLines(clusterfluxNode, workerArgs, { + cwd: projectDir, + env: { ...process.env }, + }); + const ready = await worker.waitFor( + (value) => value.node_status === "ready", + "persisted worker ready after hosted service restart" + ); + assert.strictEqual(ready.node, workerNode); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const restartedRun = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(restartedRun.status, "main_launched"); + const restartedProcess = restartedRun.process; + const completed = await waitForCli( + "new process completion after hosted service restart", + () => + runJson( + clusterflux, + ["task", "list", ...scope, "--process", restartedProcess], + { cwd: projectDir } + ), + (tasks) => completedFlagship(rawTaskEvents(tasks)), + 5 * 60 * 1000 + ); + const completedEvent = rawTaskEvents(completed).find( + (event) => + event.executor === "coordinator_main" && + event.terminal_state === "completed" + ); + assert(completedEvent, "post-restart flagship omitted its completed main event"); + return { + executed: true, + account_project_session_preserved: true, + node_identity_preserved: true, + old_process_terminated_honestly: true, + terminated_ephemeral_process: ephemeralProcess, + new_process: restartedProcess, + new_process_result: completedEvent.result, + before, + after_first_restart: afterFirstRestart, + after, + worker, + }; +} + +async function finishUserCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, +}) { + const forged = assertDenied( + await sendHostedControl( + authenticatedRequest( + `forged-session-${crypto.randomBytes(24).toString("hex")}`, + { type: "list_projects" } + ) + ), + /not recognized|invalid|authentication|session/i, + "forged user session" + ); + + let expired = null; + const expiredFile = process.env.CLUSTERFLUX_EXPIRED_USER_SESSION_FILE; + if (expiredFile) { + const oldSession = readJson(path.resolve(expiredFile)); + assert(oldSession.session_secret, "expired session evidence omitted its secret"); + expired = assertDenied( + await sendHostedControl( + authenticatedRequest(oldSession.session_secret, { type: "list_projects" }) + ), + /expired|not recognized/i, + "expired user session" + ); + } else if (strictFullRelease) { + throw new Error( + "strict full release requires CLUSTERFLUX_EXPIRED_USER_SESSION_FILE for a genuinely expired hosted session" + ); + } + + const logout = runJson(clusterflux, ["logout", ...scope, "--yes"], { + cwd: projectDir, + }); + assert.strictEqual(logout.server_session_revocation.revoked, true); + const revoked = assertDenied( + await sendHostedControl( + authenticatedRequest(sessionSecret, { type: "list_projects" }) + ), + /revoked|not recognized/i, + "revoked user session" + ); + return { forged, expired, revoked }; +} + +async function dapVariables(client, variablesReference) { + const request = client.send("variables", { variablesReference }); + return (await client.response(request, "variables")).body.variables; +} + +async function runLiveDapEditRestart({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + worker, +}) { + const sourcePath = fs.realpathSync(path.join(projectDir, "src/build.rs")); + const originalSource = fs.readFileSync(sourcePath, "utf8"); + const sourceLines = originalSource.split(/\r?\n/); + const lineFor = (needle) => { + const line = sourceLines.findIndex((candidate) => candidate.includes(needle)) + 1; + assert(line > 0, `flagship source omitted ${needle}`); + return line; + }; + const mainLine = lineFor("pub async fn restart_main()"); + const taskLine = lineFor("fn task_trap("); + const client = new DapClient({ + cwd: projectDir, + command: clusterfluxDap, + args: [], + env: { ...process.env }, + }); + let sourceRestored = false; + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "restart", + project: projectDir, + runtimeBackend: "live-services", + coordinatorEndpoint: serviceEndpoint, + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const setBreakpoints = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: mainLine }, { line: taskLine }], + }); + const breakpointResponse = await client.response( + setBreakpoints, + "setBreakpoints" + ); + assert.deepStrictEqual( + breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), + [true, true] + ); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const mainStop = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(mainStop.body.allThreadsStopped, true); + + const mainContinue = client.send("continue", { threadId: mainStop.body.threadId }); + await client.response(mainContinue, "continue"); + const taskStop = await client.waitFor( + (message) => + message.seq > mainStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(taskStop.body.allThreadsStopped, true); + assert.notStrictEqual(taskStop.body.threadId, mainStop.body.threadId); + + const stackTrace = client.send("stackTrace", { + threadId: taskStop.body.threadId, + startFrame: 0, + levels: 1, + }); + const taskFrames = (await client.response(stackTrace, "stackTrace")).body + .stackFrames; + assert.strictEqual(taskFrames[0].line, taskLine); + + const scopesRequest = client.send("scopes", { frameId: taskFrames[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const argsScope = scopes.find( + (candidate) => candidate.name === "Task Args and Handles" + ); + const runtimeScope = scopes.find( + (candidate) => candidate.name === "Clusterflux Runtime" + ); + assert(argsScope && runtimeScope); + const taskArgs = await dapVariables(client, argsScope.variablesReference); + assert( + taskArgs.some( + (variable) => + variable.name === "arg_0" && String(variable.value).includes("0") + ) + ); + const runtime = await dapVariables(client, runtimeScope.variablesReference); + assert( + runtime.some( + (variable) => + variable.name === "runtime_backend" && variable.value === "LiveServices" + ) + ); + assert( + runtime.some( + (variable) => variable.name === "state" && variable.value === "Frozen" + ) + ); + const dapProcess = runtime.find( + (variable) => variable.name === "virtual_process_id" + )?.value; + assert.match(dapProcess || "", /^vp-[0-9a-f]{12}$/); + + const taskContinue = client.send("continue", { threadId: taskStop.body.threadId }); + await client.response(taskContinue, "continue"); + const failedStop = await client.waitFor( + (message) => + message.seq > taskStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "exception", + 5 * 60 * 1000 + ); + assert.strictEqual(failedStop.body.allThreadsStopped, false); + + const beforeEdit = runJson( + clusterflux, + ["task", "list", ...scope, "--process", dapProcess], + { cwd: projectDir } + ); + const originalTaskEvent = rawTaskEvents(beforeEdit).find( + (event) => + event.task_definition === "task_trap" && + event.terminal_state === "failed" + ); + assert(originalTaskEvent, "DAP restart entry omitted its original child event"); + assert(originalTaskEvent.attempt_id, "failed attempt omitted its identity"); + + const originalTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { + #[cfg(target_arch = "wasm32")] + core::arch::wasm32::unreachable(); + #[cfg(not(target_arch = "wasm32"))] + panic!("intentional task trap") +}`; + const replacementTaskBody = `pub extern "C" fn task_trap(_input: i32) -> i32 { + _input + 42 // strict hosted compatible restart probe +}`; + const editedSource = originalSource.replace(originalTaskBody, replacementTaskBody); + assert.notStrictEqual(editedSource, originalSource); + fs.writeFileSync(sourcePath, editedSource); + + const restartFrame = client.send("restartFrame", { + frameId: taskFrames[0].id, + }); + const restartResponse = await client.response(restartFrame, "restartFrame"); + const restartedStop = await client.waitFor( + (message) => + message.seq > restartResponse.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(restartedStop.body.allThreadsStopped, true); + const restartedContinue = client.send("continue", { + threadId: restartedStop.body.threadId, + }); + await client.response(restartedContinue, "continue"); + const restartedNodeReport = await worker.waitFor( + (value) => + value.node_status === "completed" && + value.virtual_thread === originalTaskEvent.task && + value.task_assignment_response?.task_spec?.task_definition === + "task_trap", + "edited task to execute on the live restarted node", + 5 * 60 * 1000 + ); + assert.strictEqual( + restartedNodeReport.node_status, + "completed", + `edited task restart failed on the live node: ${JSON.stringify(restartedNodeReport)}` + ); + assert.match(restartedNodeReport.stdout_tail, /"SmallJson":42/); + const restartedTask = restartedNodeReport.virtual_thread; + + const afterEdit = await waitForCli( + "edited live task event", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", dapProcess], { + cwd: projectDir, + }), + (tasks) => + rawTaskEvents(tasks).some( + (event) => + event.task === restartedTask && + event.terminal_state === "completed" && + JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) + ), + 5 * 60 * 1000 + ); + const restartedEvent = rawTaskEvents(afterEdit).find( + (event) => + event.task === restartedTask && + event.terminal_state === "completed" && + JSON.stringify(event.result) === JSON.stringify({ SmallJson: 42 }) + ); + assert(restartedEvent, "replacement attempt omitted its completed event"); + assert(restartedEvent.attempt_id, "replacement attempt omitted its identity"); + assert.notStrictEqual(restartedEvent.attempt_id, originalTaskEvent.attempt_id); + await client.waitFor( + (message) => message.type === "event" && message.event === "terminated", + 5 * 60 * 1000 + ); + fs.writeFileSync(sourcePath, originalSource); + sourceRestored = true; + + return { + process: dapProcess, + main_breakpoint_line: mainLine, + task_breakpoint_line: taskLine, + main_all_threads_stopped: mainStop.body.allThreadsStopped, + task_all_threads_stopped: taskStop.body.allThreadsStopped, + original_task_instance: originalTaskEvent.task, + original_attempt: originalTaskEvent.attempt_id, + restarted_task_instance: restartedTask, + restarted_attempt: restartedEvent.attempt_id, + restarted_result: restartedEvent.result, + compatible_source_edit: "task_trap: trap -> input + 42", + original_arguments_preserved: true, + clean_vfs_boundary_used: true, + }; + } finally { + if (!sourceRestored) fs.writeFileSync(sourcePath, originalSource); + await client.close().catch(() => { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + }); + } +} + +async function main() { + requireEnabled(); + const manifest = readJson(manifestPath); + assert.strictEqual(manifest.kind, "clusterflux-public-release"); + assert.strictEqual(manifest.default_hosted_coordinator_endpoint, serviceEndpoint); + assert.strictEqual(serviceAddr, "clusterflux.michelpaulissen.com:443"); + + const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-happy-")); + const installDir = path.join(workRoot, "install"); + const checkout = path.join(workRoot, "public-repo"); + const downloadPath = path.join(workRoot, "release.tar"); + const extractedArtifact = path.join(workRoot, "artifact"); + ensureDir(installDir); + run("tar", ["-xzf", binaryArchive(manifest), "-C", installDir]); + const publicSource = stagePublicCheckout(manifest, checkout); + + const clusterflux = executable(installDir, "clusterflux"); + const clusterfluxNode = executable(installDir, "clusterflux-node"); + const clusterfluxDap = executable(installDir, "clusterflux-debug-dap"); + const projectDir = path.join(checkout, "examples/launch-build-demo"); + const suffix = String(Date.now()); + const workerNode = `worker-${suffix}`; + + let loginSession; + let receivedDefaultProject; + if (reuseSessionFile) { + const sourceSessionPath = path.resolve(reuseSessionFile); + const sourceProjectPath = path.join(path.dirname(sourceSessionPath), "project.json"); + const sourceSession = readJson(sourceSessionPath); + const sourceProject = readJson(sourceProjectPath); + assert.strictEqual(sourceSession.kind, "human"); + assert.strictEqual(sourceSession.coordinator, serviceEndpoint); + assert.strictEqual(sourceProject.coordinator, serviceEndpoint); + assert.strictEqual(sourceProject.tenant, sourceSession.tenant); + assert.strictEqual(sourceProject.project, sourceSession.project); + assert.strictEqual(sourceProject.user, sourceSession.user); + assert( + Number(sourceSession.expires_at) > Math.floor(Date.now() / 1000) + 300, + "reused CLI session expires too soon for the live journey" + ); + const destinationConfig = path.join(projectDir, ".clusterflux"); + ensureDir(destinationConfig); + fs.copyFileSync(sourceSessionPath, path.join(destinationConfig, "session.json")); + fs.copyFileSync(sourceProjectPath, path.join(destinationConfig, "project.json")); + fs.chmodSync(path.join(destinationConfig, "session.json"), 0o600); + fs.chmodSync(path.join(destinationConfig, "project.json"), 0o644); + loginSession = sourceSession; + receivedDefaultProject = true; + } else { + const login = runJson( + clusterflux, + ["login", "--browser", "--json"], + { + cwd: projectDir, + env: { ...process.env, CLUSTERFLUX_BROWSER_OPEN_COMMAND: browserOpenCommand }, + } + ); + assert.strictEqual(login.plan.coordinator, serviceEndpoint); + assert.strictEqual(login.boundary.scoped_cli_session_received, true); + assert.strictEqual(login.boundary.provider_tokens_exposed_to_cli, false); + assert.strictEqual(login.boundary.provider_tokens_sent_to_nodes, false); + loginSession = login.coordinator_response.session; + assert(loginSession, "hosted browser login omitted its scoped session"); + receivedDefaultProject = + typeof loginSession.project === "string" && loginSession.project.length > 0; + assert(receivedDefaultProject, "login did not return its server-owned project"); + } + const sessionSecret = + loginSession.cli_session_secret || loginSession.session_secret; + assert(sessionSecret, "hosted browser login omitted the CLI session credential"); + const tenant = loginSession.tenant; + const project = loginSession.project; + const user = loginSession.user; + for (const [name, value] of Object.entries({ tenant, project, user })) { + assert.strictEqual(typeof value, "string", `hosted session omitted ${name}`); + assert.notStrictEqual(value, "", `hosted session returned an empty ${name}`); + } + const scope = [ + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--user", + user, + "--json", + ]; + + if (!reuseSessionFile) { + const projectInit = runJson( + clusterflux, + [ + "project", + "init", + ...scope, + "--new-project", + project, + "--name", + "CLI Happy Path", + "--yes", + ], + { cwd: projectDir } + ); + assert.strictEqual(projectInit.command, "project init"); + assert.strictEqual(projectInit.project_config_written, true); + assert.strictEqual(projectInit.coordinator_response.type, "project_created"); + } + + const projectList = runJson(clusterflux, ["project", "list", ...scope], { + cwd: projectDir, + }); + assert(projectList.project_count >= 1); + const projectSelect = runJson( + clusterflux, + ["project", "select", ...scope, project], + { cwd: projectDir } + ); + assert.strictEqual(projectSelect.command, "project select"); + + const inspection = runJson(clusterflux, ["inspect", "--project", ".", "--json"], { + cwd: projectDir, + }); + assert( + inspection.metadata.environments.some((environment) => environment.name === "linux") + ); + assert(Array.isArray(inspection.source_provider_statuses)); + assert(inspection.source_provider_manifest); + + const runReport = runJson( + clusterflux, + ["run", "build", "--project", ".", "--json"], + { cwd: projectDir } + ); + assert.strictEqual(runReport.command, "run"); + assert.strictEqual(runReport.status, "main_launched"); + assert.strictEqual(runReport.task_launch.type, "main_launched"); + const processId = runReport.process; + + const parkedStatus = await waitForCli( + "hosted coordinator main to park before node enrollment", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => + status.live_process?.main_state === "running" && + status.live_process?.main_wait_state === "waiting_for_node" && + status.live_process?.connected_nodes?.length === 0, + 120000 + ); + + const grant = runJson(clusterflux, ["node", "enroll", ...scope], { + cwd: projectDir, + }); + assert.strictEqual(grant.command, "node enroll"); + const attach = runJson( + clusterflux, + [ + "node", + "attach", + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + workerNode, + "--enrollment-grant", + grant.enrollment_grant.grant, + "--json", + ], + { cwd: projectDir } + ); + assert.strictEqual(attach.command, "node attach"); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + + const credentialDir = path.join(projectDir, ".clusterflux", "nodes"); + const credentialFiles = fs + .readdirSync(credentialDir) + .filter((file) => file.endsWith(".json")); + assert.strictEqual(credentialFiles.length, 1, "expected one persisted node credential"); + const credentialPath = path.join(credentialDir, credentialFiles[0]); + const credentialBefore = fs.readFileSync(credentialPath); + const credentialDigest = sha256(credentialBefore); + if (process.platform !== "win32") { + assert.strictEqual(fs.statSync(credentialPath).mode & 0o777, 0o600); + } + + const workerArgs = [ + "--coordinator", + serviceEndpoint, + "--tenant", + tenant, + "--project-id", + project, + "--node", + workerNode, + "--worker", + "--project-root", + projectDir, + "--assignment-poll-ms", + "500", + "--emit-ready", + ]; + const spawnWorker = () => + spawnJsonLines(clusterfluxNode, workerArgs, { + cwd: projectDir, + env: { ...process.env }, + }); + + let worker = spawnWorker(); + try { + const firstReady = await worker.waitFor( + (value) => value.node_status === "ready", + "first persisted worker ready" + ); + assert.strictEqual(firstReady.node, workerNode); + const initiallyOnline = runJson( + clusterflux, + ["node", "status", ...scope, "--node", workerNode], + { cwd: projectDir } + ); + assert.strictEqual(nodeDescriptor(initiallyOnline, workerNode)?.online, true); + + const completedTasks = await waitForCli( + "real hosted flagship completion", + () => + runJson(clusterflux, ["task", "list", ...scope, "--process", processId], { + cwd: projectDir, + }), + (tasks) => completedFlagship(rawTaskEvents(tasks)) + ); + const firstEvents = rawTaskEvents(completedTasks); + assert( + firstEvents + .filter((event) => event.executor === "node") + .every((event) => event.node === workerNode) + ); + assert( + firstEvents.some( + (event) => + event.task_definition === "package_release" && + event.terminal_state === "completed" && + contentAddressedArtifactId(event, "release.tar") + ) + ); + + const releasedFlagshipStatus = await waitForCli( + "completed flagship process to release its active slot", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + + const offlineStartedAt = Date.now(); + await stopChild(worker.child); + const observedOffline = await waitForCli( + "server-derived worker stale/offline transition", + () => + runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { + cwd: projectDir, + }), + (status) => nodeDescriptor(status, workerNode)?.online === false, + 120000 + ); + worker = spawnWorker(); + const secondReady = await worker.waitFor( + (value) => value.node_status === "ready", + "restarted persisted worker ready" + ); + assert.strictEqual(secondReady.node, workerNode); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + + const nodeStatus = await waitForCli( + "server-derived worker online recovery", + () => + runJson(clusterflux, ["node", "status", ...scope, "--node", workerNode], { + cwd: projectDir, + }), + (status) => nodeDescriptor(status, workerNode)?.online === true, + 30000 + ); + assert(JSON.stringify(nodeStatus.response).includes(workerNode)); + assert.strictEqual(sha256(fs.readFileSync(credentialPath)), credentialDigest); + const nodeLivenessTransition = { + initial_online: nodeDescriptor(initiallyOnline, workerNode).online, + stale_offline: nodeDescriptor(observedOffline, workerNode).online, + recovered_online: nodeDescriptor(nodeStatus, workerNode).online, + offline_detection_ms: Date.now() - offlineStartedAt, + persisted_identity_reused: true, + }; + + const processStatus = runJson( + clusterflux, + ["process", "status", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert.strictEqual(processStatus.command, "process status"); + assert(processStatus.current_task_count >= firstEvents.length); + + const logs = runJson( + clusterflux, + ["logs", ...scope, "--process", processId], + { cwd: projectDir } + ); + assert(logs.log_entries.length >= 4); + + const artifacts = runJson( + clusterflux, + ["artifact", "list", ...scope, "--process", processId], + { cwd: projectDir } + ); + const releaseArtifact = artifacts.artifacts.find((artifact) => { + const digest = artifact.digest; + return ( + typeof digest === "string" && + /^sha256:[0-9a-f]{64}$/.test(digest) && + artifact.artifact === `release.tar-${digest.slice("sha256:".length)}` + ); + }); + assert(releaseArtifact, "hosted flagship did not publish release.tar"); + assert.strictEqual(releaseArtifact.state, "metadata_flushed"); + + const download = runJson( + clusterflux, + [ + "artifact", + "download", + ...scope, + releaseArtifact.artifact, + "--to", + downloadPath, + ], + { cwd: projectDir } + ); + assert.strictEqual(download.download_session.link_issued, true); + assert.strictEqual(download.local_download.local_bytes_written_by_cli, true); + assert.strictEqual(download.local_download.verified_digest, releaseArtifact.digest); + assert.strictEqual(sha256(fs.readFileSync(downloadPath)), releaseArtifact.digest); + + ensureDir(extractedArtifact); + const archiveEntries = run("tar", ["-tf", downloadPath]); + assert.match(archiveEntries, /(?:^|\n)hello-clusterflux(?:\n|$)/); + run("tar", ["-xf", downloadPath, "-C", extractedArtifact]); + const builtExecutable = path.join(extractedArtifact, "hello-clusterflux"); + assert(fs.existsSync(builtExecutable)); + const builtOutput = run(builtExecutable, []).trim(); + assert.strictEqual(builtOutput, "hello from a real Clusterflux build"); + + const dapEditRestart = await runLiveDapEditRestart({ + clusterfluxDap, + clusterflux, + projectDir, + scope, + worker, + }); + + const restart = runJson( + clusterflux, + ["process", "restart", ...scope, "--process", dapEditRestart.process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(restart.restart_request.accepted, true); + const cancel = runJson( + clusterflux, + ["process", "cancel", ...scope, "--process", dapEditRestart.process, "--yes"], + { cwd: projectDir } + ); + assert.strictEqual(cancel.cancel_request.accepted, true); + const releasedDap = await waitForCli( + "cancelled DAP process slot release", + () => + runJson( + clusterflux, + ["process", "status", ...scope, "--process", dapEditRestart.process], + { cwd: projectDir } + ), + (status) => status.state === "not_active", + 120000 + ); + assert.strictEqual(releasedDap.state, "not_active"); + + const repeatedParkWake = await runRepeatedParkWakeProof({ + clusterflux, + projectDir, + scope, + }); + const longJoin = await runLongJoinProof({ + clusterflux, + projectDir, + scope, + }); + + const tenantIsolation = await runSecondTenantIsolation({ + firstTenant: tenant, + firstProject: project, + firstProcess: dapEditRestart.process, + firstNode: workerNode, + firstArtifact: releaseArtifact.artifact, + manifest, + }); + + const securityNode = await prepareLiveNodeCredentialSecurity({ + clusterflux, + projectDir, + scope, + tenant, + project, + suffix, + bundle: { + digest: runReport.bundle_digest, + moduleBase64: fs + .readFileSync( + path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) + ) + .toString("base64"), + entryExport: runReport.entry_export, + entryStableId: runReport.entry_stable_id, + }, + }); + const droppedConnectionRollback = await runDroppedConnectionRollback({ + clusterflux, + projectDir, + scope, + sessionSecret, + suffix, + }); + const bandwidthPreallocation = await runLiveBandwidthPreallocation({ + sessionSecret, + artifact: releaseArtifact.artifact, + maxBytes: download.local_download.bytes_written, + }); + const agentCredentialSecurity = await runLiveAgentCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, + tenant, + project, + suffix, + bundle: { + digest: runReport.bundle_digest, + moduleBase64: fs + .readFileSync( + path.resolve(projectDir, runReport.bundle_build.bundle_artifact.module) + ) + .toString("base64"), + entryExport: runReport.entry_export, + entryStableId: runReport.entry_stable_id, + }, + securityNode, + workerRuntime: { + node: workerNode, + child: worker.child, + worker, + identity: nodeIdentityFromPrivateKey( + readNodeCredential(projectDir, workerNode).credential.private_key + ), + }, + }); + + await stopChild(worker.child); + const liveSoak = await runLiveSoak({ + clusterflux, + projectDir, + scope, + securityNode, + }); + const revokedNodeCredential = await revokeSecurityNode({ + clusterflux, + projectDir, + scope, + securityNode, + }); + const quotaPreallocation = await runLiveQuotaPreallocation({ + sessionSecret, + suffix, + }); + const serviceRestart = await restartHostedServiceAndResume({ + clusterflux, + clusterfluxNode, + projectDir, + scope, + workerArgs, + workerNode, + credentialPath, + credentialDigest, + }); + if (serviceRestart.worker) worker = serviceRestart.worker; + const userCredentialSecurity = await finishUserCredentialSecurity({ + clusterflux, + projectDir, + scope, + sessionSecret, + }); + const configProvenance = configurationProvenance(); + const concurrentCompileTasks = [ + ...new Set( + firstEvents + .filter( + (event) => + event.task_definition === "compile_linux" && + event.terminal_state === "completed" + ) + .map((event) => event.task) + ), + ]; + assert(concurrentCompileTasks.length >= 2); + const strictRequirementLedger = [ + { id: "01_authenticated_project", passed: Boolean(sessionSecret && tenant && project) }, + { id: "02_project_commands", passed: projectList.project_count >= 1 && projectSelect.command === "project select" }, + { id: "03_bundle_environment_inspection", passed: inspection.metadata.environments.some((environment) => environment.name === "linux") }, + { id: "04_main_parks_before_node", passed: parkedStatus.live_process.main_wait_state === "waiting_for_node" }, + { id: "05_enrollment_and_persisted_credential", passed: attach.boundary.used_enrollment_exchange === true && secondReady.node === workerNode }, + { id: "06_server_derived_node_liveness", passed: nodeLivenessTransition.initial_online === true && nodeLivenessTransition.stale_offline === false && nodeLivenessTransition.recovered_online === true }, + { id: "07_real_flagship_and_artifact", passed: completedFlagship(firstEvents) && Boolean(releaseArtifact) && builtOutput === "hello from a real Clusterflux build" }, + { id: "08_same_definition_concurrency", passed: concurrentCompileTasks.length >= 2 }, + { id: "09_repeated_park_wake", passed: repeatedParkWake.completed_cycles >= 16 && repeatedParkWake.terminal_state === "not_active" }, + { id: "10_nested_environment_rejection", passed: agentCredentialSecurity.nested_environment_mismatch.denied === true }, + { id: "11_partial_freeze_warning_resume", passed: agentCredentialSecurity.partial_freeze.partially_frozen === true && agentCredentialSecurity.partial_freeze.resumed_participants.length > 0 }, + { id: "12_live_dap_edit_restart", passed: dapEditRestart.clean_vfs_boundary_used === true }, + { id: "13_long_join_and_process_lifecycle", passed: longJoin.duration_seconds > 120 && longJoin.terminal_state === "completed" && releasedFlagshipStatus.state === "not_active" && restart.restart_request.accepted === true && cancel.cancel_request.accepted === true }, + { id: "14_tenant_isolation", passed: tenantIsolation.executed === true && tenantIsolation.process_hidden === true && tenantIsolation.node_hidden === true }, + { id: "15_user_credential_hostility", passed: Boolean(userCredentialSecurity.expired && userCredentialSecurity.forged && userCredentialSecurity.revoked) }, + { id: "16_node_credential_hostility", passed: securityNode.evidence.replay.denied === true && securityNode.evidence.expired.denied === true && securityNode.evidence.forged.denied === true && revokedNodeCredential.denied.denied === true }, + { id: "17_agent_workflow_and_credentials", passed: agentCredentialSecurity.workflow.flagship_completed === true && agentCredentialSecurity.workflow.authenticated_without_browser === true && agentCredentialSecurity.workflow.external_direct_task_v1.denied === true && agentCredentialSecurity.workflow.child_launch === "task_launched" && agentCredentialSecurity.revoked.denied === true }, + { id: "18_dropped_response_rollback", passed: droppedConnectionRollback.response_body_consumed === false && droppedConnectionRollback.terminal_state === "not_active" }, + { id: "19_quota_and_bandwidth_preallocation", passed: quotaPreallocation.denied_process_allocated === false && quotaPreallocation.unrelated_tenant.unaffected_by_first_tenant_quota === true && bandwidthPreallocation.bytes_served_before_denial === 0 }, + { id: "20_soak_restart_and_provenance", passed: liveSoak.executed === true && serviceRestart.executed === true && manifest.source_tree_clean === true && configProvenance.clean === true }, + ]; + const fullReleasePassed = + strictFullRelease && + strictRequirementLedger.every((requirement) => requirement.passed === true); + + const report = { + kind: "clusterflux-cli-happy-path-live", + release_name: manifest.release_name, + source_commit: manifest.source_commit, + public_tree_identity: manifest.public_tree_identity, + service_addr: serviceAddr, + default_hosted_coordinator_endpoint: serviceEndpoint, + public_repository_url: publicSource.published ? publicSource.url : null, + public_source: publicSource, + tenant, + project, + worker_node: workerNode, + process: processId, + signed_in: true, + fresh_hosted_login: !reuseSessionFile, + reused_scoped_cli_session: Boolean(reuseSessionFile), + received_default_project: receivedDefaultProject, + process_started_before_node: true, + parked_main_observed: { + main_state: parkedStatus.live_process.main_state, + main_wait_state: parkedStatus.live_process.main_wait_state, + connected_nodes: parkedStatus.live_process.connected_nodes, + }, + node_enrollment_grants_used: 2, + persisted_node_credential_mode: + process.platform === "win32" ? null : fs.statSync(credentialPath).mode & 0o777, + persisted_node_credential_digest: credentialDigest, + node_restarted_without_grant: true, + restarted_node_ready: secondReady.node, + node_liveness_transition: nodeLivenessTransition, + flagship_terminal_state: releasedFlagshipStatus.state, + flagship_task_definitions: [ + "prepare_source", + "compile_linux", + "package_release", + ], + rootless_podman_flagship_completed: true, + downstream_release_artifact: releaseArtifact, + artifact_download: { + bytes_written: download.local_download.bytes_written, + verified_digest: download.local_download.verified_digest, + executable_output: builtOutput, + }, + flagship_process_released_automatically: true, + concurrent_same_definition_tasks: concurrentCompileTasks, + repeated_park_wake: repeatedParkWake, + long_join: longJoin, + live_dap_edit_restart: dapEditRestart, + process_restart_requested: true, + process_cancel_requested: true, + dap_process_aborted: true, + tenant_isolation: tenantIsolation, + credential_security: { + user: userCredentialSecurity, + node: { + node: securityNode.node, + credential_digest: securityNode.credential_digest, + ...securityNode.evidence, + revoked: revokedNodeCredential.denied, + }, + agent: agentCredentialSecurity, + }, + quota_preallocation: quotaPreallocation, + bandwidth_preallocation: bandwidthPreallocation, + dropped_connection_rollback: droppedConnectionRollback, + live_soak: liveSoak, + hosted_service_restart: { + ...serviceRestart, + worker: undefined, + }, + release_binding: { + manifest: manifestPath, + source_commit: manifest.source_commit, + source_tree_clean: manifest.source_tree_clean, + public_tree_identity: manifest.public_tree_identity, + binary_digests: manifest.binary_digests, + deployment: serviceRestart.after || deploymentProvenance(), + configuration: configProvenance, + }, + commands, + strict_requirement_ledger: strictRequirementLedger, + acceptance_result: fullReleasePassed ? "passed" : "partial", + }; + ensureDir(path.dirname(reportPath)); + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + if (strictFullRelease && !fullReleasePassed) { + const failed = strictRequirementLedger + .filter((requirement) => requirement.passed !== true) + .map((requirement) => requirement.id); + throw new Error(`strict full release failed: ${failed.join(", ")}`); + } + console.log(`CLI happy-path live smoke passed: ${reportPath}`); + } finally { + await stopChild(worker?.child); + } +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-install-smoke.js b/scripts/cli-install-smoke.js new file mode 100755 index 0000000..6c72342 --- /dev/null +++ b/scripts/cli-install-smoke.js @@ -0,0 +1,61 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-install-")); +const installRoot = path.join(temp, "install"); +const targetDir = + process.env.CLUSTERFLUX_CLI_INSTALL_CARGO_TARGET_DIR || + process.env.CARGO_TARGET_DIR || + path.join(repo, "target"); +const project = path.join(repo, "examples/launch-build-demo"); +const binName = process.platform === "win32" ? "clusterflux.exe" : "clusterflux"; +const installedBin = path.join(installRoot, "bin", binName); + +try { + cp.execFileSync( + "cargo", + [ + "install", + "--path", + "crates/clusterflux-cli", + "--bin", + "clusterflux", + "--root", + installRoot, + "--debug" + ], + { + cwd: repo, + env: { + ...process.env, + CARGO_TARGET_DIR: targetDir + }, + stdio: "inherit" + } + ); + + assert(fs.existsSync(installedBin), "installed clusterflux binary must exist"); + + const inspection = JSON.parse( + cp.execFileSync( + installedBin, + ["bundle", "inspect", "--project", project, "--json"], + { cwd: repo, encoding: "utf8" } + ) + ); + + assert.strictEqual(inspection.project, project); + assert.strictEqual(inspection.metadata.embeds_full_container_images, false); + assert(inspection.metadata.environments.some((env) => env.name === "linux")); + assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/build.rs")); +} finally { + fs.rmSync(temp, { recursive: true, force: true }); +} + +console.log("CLI install smoke passed"); diff --git a/scripts/cli-local-run-smoke.js b/scripts/cli-local-run-smoke.js new file mode 100755 index 0000000..45d9c8f --- /dev/null +++ b/scripts/cli-local-run-smoke.js @@ -0,0 +1,269 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +configurePodmanTestEnvironment(repo); +if ( + !process.env.CLUSTERFLUX_PODMAN_NIX_SHELL && + cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 && + cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0 +) { + cp.execFileSync( + "nix", + ["shell", "nixpkgs#podman", "--command", "node", __filename], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_PODMAN_NIX_SHELL: "1", + }, + stdio: "inherit", + } + ); + process.exit(0); +} +const project = path.join(repo, "examples/launch-build-demo"); + +cp.execFileSync( + "cargo", + ["build", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node"], + { cwd: repo, stdio: "inherit" } +); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function runCli(args, env = {}) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + env: { + ...process.env, + ...env + } + } + ); + const cliPid = child.pid; + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`CLI run failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve({ pid: cliPid, report: JSON.parse(stdout) }); + } catch (error) { + reject(new Error(`CLI output was not JSON: ${stdout}\n${error.stack || error.message}`)); + } + }); + }); +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + assert(Number.isInteger(coordinator.pid)); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const { pid: cliPid, report } = await runCli([ + "run", + "--coordinator", + `${addr.host}:${addr.port}`, + "--project", + project, + "--json", + ]); + assert(Number.isInteger(cliPid)); + assert.notStrictEqual(cliPid, coordinator.pid); + assert.strictEqual(report.plan.entry, "build"); + assert.deepStrictEqual(report.plan.session, "Anonymous"); + assert.strictEqual(report.boundary.cli_process_started_node_process, true); + assert.strictEqual(report.boundary.cli_process_started_coordinator_process, false); + assert(Number.isInteger(report.boundary.spawned_node_process_id)); + assert.notStrictEqual(report.boundary.spawned_node_process_id, cliPid); + assert.notStrictEqual(report.boundary.spawned_node_process_id, coordinator.pid); + assert.strictEqual(report.boundary.node_session_requests, 0); + assert.strictEqual(report.node_report.node_status, "completed"); + assert.strictEqual(report.node_report.execution_substrate, "wasm"); + assert.strictEqual(report.node_report.task_spawn_host_import, true); + assert.strictEqual( + report.node_report.pre_node_process_status.processes.length, + 1 + ); + assert.deepStrictEqual( + report.node_report.pre_node_process_status.processes[0].connected_nodes, + [] + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_state, + "running" + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_wait_state, + "waiting_for_node", + "the coordinator must expose that the capless main is parked on placement before a node exists" + ); + assert.strictEqual( + report.node_report.pre_node_process_status.processes[0].main_task_instance, + report.node_report.run.task_instance + ); + assert.strictEqual(report.node_report.run.status, "main_launched"); + assert.strictEqual(report.node_report.join.type, "task_joined"); + const process = report.node_report.run.process; + assert.strictEqual(process, "vp-current"); + + const events = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process + }); + assert.strictEqual(events.type, "task_events"); + assert(events.events.length >= 4); + assert( + events.events + .filter((event) => event.executor === "node") + .every((event) => event.node === "node-cli-local") + ); + assert( + events.events.some( + (event) => + event.executor === "coordinator_main" && + event.node === "coordinator-main" + ) + ); + assert(events.events.every((event) => event.process === process)); + assert.deepStrictEqual( + new Set(events.events.map((event) => event.task_definition)), + new Set([ + report.node_report.run.task_definition, + "prepare_source", + "compile_linux", + "package_release", + ]) + ); + assert.strictEqual( + new Set(events.events.map((event) => event.task)).size, + events.events.length, + "every live task event must retain its unique instance identity" + ); + assert( + events.events.some( + (event) => event.task === report.node_report.run.task_instance + ) + ); + assert(events.events.some((event) => event.task.endsWith(":child:1"))); + assert(events.events.some((event) => event.task.endsWith(":child:2"))); + assert(events.events.some((event) => event.task.endsWith(":child:3"))); + assert(events.events.some((event) => event.artifact_path)); + } finally { + coordinator.kill("SIGTERM"); + } + + const { pid: autoCliPid, report: autoReport } = await runCli([ + "run", + "--local", + "--project", + project, + "--json", + ]); + assert(Number.isInteger(autoCliPid)); + assert.strictEqual(autoReport.plan.entry, "build"); + assert.deepStrictEqual(autoReport.plan.coordinator, "LocalOnly"); + assert.deepStrictEqual(autoReport.plan.session, "Anonymous"); + assert.strictEqual(autoReport.boundary.cli_process_started_node_process, true); + assert.strictEqual(autoReport.boundary.cli_process_started_coordinator_process, true); + assert.match(autoReport.boundary.coordinator_address, /^127\.0\.0\.1:\d+$/); + assert(Number.isInteger(autoReport.boundary.coordinator_process_id)); + assert(Number.isInteger(autoReport.boundary.spawned_node_process_id)); + assert.notStrictEqual(autoReport.boundary.coordinator_process_id, autoCliPid); + assert.notStrictEqual(autoReport.boundary.spawned_node_process_id, autoCliPid); + assert.notStrictEqual( + autoReport.boundary.spawned_node_process_id, + autoReport.boundary.coordinator_process_id + ); + assert.strictEqual(autoReport.boundary.node_session_requests, 0); + assert.strictEqual(autoReport.node_report.node_status, "completed"); + assert.strictEqual(autoReport.node_report.execution_substrate, "wasm"); + assert.strictEqual(autoReport.node_report.task_spawn_host_import, true); + assert.strictEqual(autoReport.node_report.run.status, "main_launched"); + assert.strictEqual(autoReport.node_report.join.type, "task_joined"); + + console.log("CLI local run smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/cli-login-smoke.js b/scripts/cli-login-smoke.js new file mode 100644 index 0000000..61b053b --- /dev/null +++ b/scripts/cli-login-smoke.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const coordinator = "https://coord.example.test"; +const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; + +function clusterflux(args) { + return JSON.parse( + cp.execFileSync( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { cwd: repo, encoding: "utf8" } + ) + ); +} + +function clusterfluxRaw(args, env = {}) { + return cp.spawnSync( + "cargo", + ["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args], + { + cwd: repo, + encoding: "utf8", + env: { + ...process.env, + ...env, + }, + } + ); +} + +const browser = clusterflux(["login", "--plan", "--coordinator", coordinator, "--json"]); +assert.strictEqual(browser.coordinator, coordinator); +assert(browser.human_flow.Browser, "browser login should be available for human users"); +assert.strictEqual(browser.human_flow.Browser.authorization_url, null); +assert.strictEqual(browser.human_flow.Browser.server_owns_state, true); +assert.strictEqual(browser.human_flow.Browser.server_owns_nonce, true); +assert.strictEqual(browser.human_flow.Browser.pkce_required, true); +assert.strictEqual(browser.human_flow.Browser.hosted_callback, true); +assert.strictEqual(browser.human_flow.Browser.cli_receives_provider_authorization_code, false); +assert.strictEqual(browser.human_flow.Browser.cli_submits_identity_claims, false); + +const defaultBrowser = clusterflux(["login", "--plan", "--json"]); +assert.strictEqual(defaultBrowser.coordinator, defaultHostedCoordinatorEndpoint); +assert(defaultBrowser.human_flow.Browser); + +const nonInteractiveBrowser = clusterfluxRaw( + ["login", "--browser", "--non-interactive", "--coordinator", coordinator, "--json"], + { + CLUSTERFLUX_BROWSER_OPEN_COMMAND: + "node -e 'require(\"fs\").writeFileSync(\"/tmp/clusterflux-browser-should-not-open\", \"opened\")'", + } +); +assert.strictEqual(nonInteractiveBrowser.status, 20, nonInteractiveBrowser.stderr); +assert.doesNotMatch(nonInteractiveBrowser.stderr, /Opening Clusterflux browser login/); +const nonInteractiveReport = JSON.parse(nonInteractiveBrowser.stdout); +assert.strictEqual(nonInteractiveReport.status, "authentication_required"); +assert.strictEqual(nonInteractiveReport.non_interactive, true); +assert.strictEqual(nonInteractiveReport.browser_opened, false); +assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication"); +assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20); +assert( + nonInteractiveReport.machine_error.next_actions.includes( + "rerun without --non-interactive to open the browser" + ) +); + +console.log("CLI login smoke passed"); diff --git a/scripts/cli-output-mode-smoke.js b/scripts/cli-output-mode-smoke.js new file mode 100644 index 0000000..c3bd2b8 --- /dev/null +++ b/scripts/cli-output-mode-smoke.js @@ -0,0 +1,190 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/launch-build-demo"); +const isolatedCwd = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-output-")); +const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-home-")); + +function clusterflux(args, env = {}, cwd = isolatedCwd) { + return cp.execFileSync( + "cargo", + [ + "run", + "-q", + "--manifest-path", + path.join(repo, "Cargo.toml"), + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + ...args, + ], + { + cwd, + encoding: "utf8", + env: { + ...process.env, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_DATA_HOME: path.join(isolatedHome, ".local", "share"), + XDG_STATE_HOME: path.join(isolatedHome, ".local", "state"), + ...env, + }, + } + ); +} + +function json(args, env, cwd) { + return JSON.parse(clusterflux(args, env, cwd)); +} + +function assertHuman(name, output, requiredPatterns) { + assert( + !output.trimStart().startsWith("{"), + `${name} default output should be human-readable text, not JSON` + ); + for (const pattern of requiredPatterns) { + assert.match(output, pattern, `${name} human output missing ${pattern}`); + } +} + +const helpHuman = clusterflux(["help"]); +assertHuman("help", helpHuman, [ + /Primary workflow:/, + /clusterflux login --browser/, + /clusterflux project init/, + /clusterflux node attach; clusterflux-node --worker/, + /Clusterflux: Launch Virtual Process/, + /Hosted account creation happens in the browser login flow/, + /--json/, +]); + +const loginHuman = clusterflux([ + "login", + "--plan", + "--coordinator", + "https://coord.example.test", +]); +assertHuman("login", loginHuman, [ + /Clusterflux login/, + /flow: browser/, +]); + +const loginJson = json([ + "login", + "--plan", + "--coordinator", + "https://coord.example.test", + "--json", +]); +assert.strictEqual(loginJson.coordinator, "https://coord.example.test"); +assert(loginJson.human_flow.Browser); +assert.strictEqual(loginJson.human_flow.Browser.hosted_callback, true); +assert.strictEqual(loginJson.human_flow.Browser.cli_submits_identity_claims, false); + +const doctorHuman = clusterflux(["doctor"]); +assertHuman("doctor", doctorHuman, [ + /Clusterflux doctor/, + /coordinator reachability: not_configured/, + /dependencies:/, + /auth:/, + /node capabilities:/, + /node readiness: (ready_to_attach|local_dependencies_missing|limited_capabilities)/, + /node next:/, +]); + +const doctorJson = json(["doctor", "--json"]); +assert.strictEqual(doctorJson.coordinator_reachability.checked, false); +assert.strictEqual(doctorJson.coordinator_reachability.status, "not_configured"); +assert( + ["ready_to_attach", "local_dependencies_missing", "limited_capabilities"].includes( + doctorJson.node_readiness_summary.status + ) +); +assert.strictEqual(doctorJson.node_readiness_summary.explicit_attach_required, true); +assert.strictEqual( + doctorJson.node_readiness_summary.command_execution_capability, + true +); +assert(Array.isArray(doctorJson.node_readiness_summary.missing_local_dependencies)); +assert(doctorJson.node_readiness_summary.next_actions.length >= 2); + +const authJson = json(["auth", "status", "--json"], { + CLUSTERFLUX_TOKEN: "token", + CLUSTERFLUX_TOKEN_EXPIRES_AT: "2026-07-04T00:00:00Z", +}, isolatedCwd); +assert.strictEqual(authJson.session.kind, "human"); +assert.strictEqual(authJson.session.token_expiry_posture, "expires_at"); +assert.strictEqual(authJson.session.expires_at, "2026-07-04T00:00:00Z"); +assert.strictEqual(authJson.coordinator_account_status.checked, false); +assert.strictEqual(authJson.coordinator_account_status.account_status, "unknown"); +assert.strictEqual( + authJson.coordinator_account_status.private_moderation_details_exposed, + false +); + +const inspectHuman = clusterflux(["bundle", "inspect", "--project", project]); +assertHuman("bundle inspect", inspectHuman, [ + /Clusterflux bundle inspect/, + /bundle: sha256:/, + /environments:/, +]); + +const inspectJson = json(["bundle", "inspect", "--project", project, "--json"]); +assert.strictEqual(inspectJson.project, project); +assert.match(inspectJson.metadata.identity, /^sha256:/); +assert.match(inspectJson.metadata.wasm_code, /^sha256:/); +assert.strictEqual(inspectJson.metadata.task_metadata.default_entrypoint, "build"); +assert.deepStrictEqual(inspectJson.metadata.task_metadata.entrypoints, [ + "build", + "fail", + "long-join", + "park-wake", + "restart", +]); +assert.strictEqual( + inspectJson.metadata.source_metadata.transfer_policy.coordinator_receives_source_bytes_by_default, + false +); +assert.strictEqual( + inspectJson.metadata.source_metadata.transfer_policy.default_full_repo_tarball, + false +); +assert.strictEqual(inspectJson.metadata.debug_metadata.dap_virtual_process, true); +assert.strictEqual( + inspectJson.metadata.large_input_policy.selected_inputs_are_content_digests, + true +); +assert.strictEqual(inspectJson.metadata.large_input_policy.selected_input_bytes_included, false); +assert.strictEqual(inspectJson.metadata.large_input_policy.full_repository_bytes_included, false); +assert.strictEqual( + inspectJson.metadata.large_input_policy.silent_task_argument_serialization, + false +); +assert(inspectJson.metadata.large_input_policy.supported_handle_types.includes("SourceSnapshot")); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.source_edits_can_restart_from_clean_task_boundary, + true +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.requires_clean_checkpoint_boundary, + true +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.compares_task_abi, + inspectJson.metadata.task_metadata.task_abi +); +assert.strictEqual( + inspectJson.metadata.restart_compatibility.incompatible_changes_require_whole_process_restart, + true +); + +console.log("CLI output mode smoke passed"); diff --git a/scripts/coordinator-wire.js b/scripts/coordinator-wire.js new file mode 100644 index 0000000..7761736 --- /dev/null +++ b/scripts/coordinator-wire.js @@ -0,0 +1,43 @@ +let requestId = 0; + +function coordinatorWireRequest(payload, prefix = "acceptance") { + if (payload && payload.type === "coordinator_request") return payload; + if (!payload || typeof payload.type !== "string" || !payload.type.trim()) { + throw new Error("coordinator payload must have a non-empty type"); + } + requestId += 1; + return { + type: "coordinator_request", + protocol_version: 1, + request_id: `${prefix}-${process.pid}-${requestId}`, + operation: payload.type, + authentication: authenticationMetadata(payload), + payload, + }; +} + +function authenticationMetadata(payload) { + if (payload.type === "authenticated") { + return { + kind: "cli_session", + session: true, + request_operation: payload.request?.type || "unknown", + }; + } + if (payload.type === "signed_node" || payload.node_signature) { + return { kind: "node_signature", node: payload.node || null }; + } + if (payload.agent_signature) { + return { + kind: "agent_signature", + agent: payload.actor_agent || null, + fingerprint: payload.agent_public_key_fingerprint || null, + }; + } + if (payload.admin_token) { + return { kind: "admin_credential" }; + } + return { kind: "none" }; +} + +module.exports = { coordinatorWireRequest }; diff --git a/scripts/dap-client.js b/scripts/dap-client.js new file mode 100644 index 0000000..bd3d004 --- /dev/null +++ b/scripts/dap-client.js @@ -0,0 +1,132 @@ +const cp = require("child_process"); + +class DapClient { + constructor({ + cwd = process.cwd(), + env = process.env, + command = "cargo", + args = [ + "run", + "-q", + "-p", + "clusterflux-dap", + "--bin", + "clusterflux-debug-dap", + ], + } = {}) { + this.child = cp.spawn( + command, + args, + { cwd, env } + ); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(message)); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error( + `DAP ${command} failed: ${message.message || JSON.stringify(message)}\nAdapter stderr:\n${this.stderr}` + ); + } + return message; + } + + async failure(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (message.success) { + throw new Error(`DAP ${command} unexpectedly succeeded`); + } + return message; + } + + waitFor(predicate, timeoutMs = 120000) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.child.kill("SIGKILL"); + const recent = this.messages + .slice(-10) + .map((message) => JSON.stringify(message)) + .join("\n"); + reject( + new Error( + `timed out waiting for DAP message\nRecent DAP messages:\n${recent}\nAdapter stderr:\n${this.stderr}` + ) + ); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +module.exports = { DapClient }; diff --git a/scripts/dap-smoke.js b/scripts/dap-smoke.js new file mode 100644 index 0000000..f72cd89 --- /dev/null +++ b/scripts/dap-smoke.js @@ -0,0 +1,363 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); +const { DapClient } = require("./dap-client"); + +(async () => { + const repo = path.resolve(__dirname, ".."); + const project = path.join(repo, "examples/launch-build-demo"); + const sourcePath = fs.realpathSync(path.join(project, "src/build.rs")); + const sourceLines = fs.readFileSync(sourcePath, "utf8").split(/\r?\n/); + const buildMainLine = + sourceLines.findIndex((line) => line.includes("pub async fn build_main()")) + 1; + assert(buildMainLine > 0, "flagship source must contain build_main"); + + const client = new DapClient(); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "build", + project, + runtimeBackend: "local-services", + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const breakpoints = client.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: buildMainLine }], + }); + const breakpointResponse = await client.response( + breakpoints, + "setBreakpoints" + ); + assert.strictEqual(breakpointResponse.body.breakpoints.length, 1); + assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, true); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const stopped = await client.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(stopped.body.allThreadsStopped, true); + assert.match(stopped.body.description, /confirmed by every active participant/i); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body + .threads; + const mainThread = threads.find((thread) => + thread.name.includes("build coordinator main") + ); + assert(mainThread, "the running Wasm entrypoint must be the DAP coordinator-main thread"); + assert.strictEqual(stopped.body.threadId, mainThread.id); + + const stackRequest = client.send("stackTrace", { + threadId: mainThread.id, + startFrame: 0, + levels: 1, + }); + const stack = (await client.response(stackRequest, "stackTrace")).body + .stackFrames; + assert.strictEqual(stack.length, 1); + assert.strictEqual(stack[0].line, buildMainLine); + assert.strictEqual(stack[0].source.path, sourcePath); + assert.match(stack[0].name, /build_main::wasm/); + assert.doesNotMatch(stack[0].name, /podman|cmd\.exe|powershell|pid|native child/i); + + const sourceRequest = client.send("source", { source: stack[0].source }); + const source = (await client.response(sourceRequest, "source")).body; + assert.match(source.content, /build_main/); + assert.match(source.mimeType, /rust/); + + const scopesRequest = client.send("scopes", { frameId: stack[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const localsScope = scopes.find((scope) => scope.name === "Source Locals"); + const wasmScope = scopes.find((scope) => scope.name === "Wasm Frame Locals"); + const argsScope = scopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + const runtimeScope = scopes.find( + (scope) => scope.name === "Clusterflux Runtime" + ); + assert(localsScope && wasmScope && argsScope && runtimeScope); + + const localsRequest = client.send("variables", { + variablesReference: localsScope.variablesReference, + }); + const locals = (await client.response(localsRequest, "variables")).body + .variables; + assert( + locals.some( + (variable) => + variable.name === "unavailable-local-diagnostic" && + String(variable.value).includes("cannot be inspected") + ) + ); + + const wasmRequest = client.send("variables", { + variablesReference: wasmScope.variablesReference, + }); + const wasmLocals = (await client.response(wasmRequest, "variables")).body + .variables; + assert.deepStrictEqual( + wasmLocals.map((variable) => variable.name), + ["wasm-local-diagnostic"] + ); + assert.match(wasmLocals[0].value, /did not report inspectable Wasm frame locals/); + + const argsRequest = client.send("variables", { + variablesReference: argsScope.variablesReference, + }); + const args = (await client.response(argsRequest, "variables")).body.variables; + assert.deepStrictEqual( + args.map((variable) => variable.name), + ["runtime-boundary-diagnostic"] + ); + assert.match(args[0].value, /reported no task arguments or handles/); + + const runtimeRequest = client.send("variables", { + variablesReference: runtimeScope.variablesReference, + }); + const runtime = (await client.response(runtimeRequest, "variables")).body + .variables; + const value = (name) => runtime.find((variable) => variable.name === name)?.value; + assert.strictEqual(value("runtime_backend"), "LocalServices"); + assert.strictEqual(value("state"), "Frozen"); + assert.strictEqual(value("debug_epoch"), 1); + assert.strictEqual(value("coordinator_task_events"), 0); + assert.match( + String(value("command_status")), + /frozen through local services at executing Wasm probe/ + ); + + const step = client.send("next", { threadId: mainThread.id }); + const stepFailure = await client.failure(step, "next"); + assert.match(stepFailure.message, /source stepping is not yet available/i); + assert.match(stepFailure.message, /synthetic step/i); + + const restart = client.send("restartFrame", { frameId: stack[0].id }); + const restartFailure = await client.failure(restart, "restartFrame"); + assert.match(restartFailure.message, /checkpoint boundary|still active/i); + + const incompatibleRestart = client.send("restartFrame", { + frameId: stack[0].id, + sourceCompatibility: "incompatible", + }); + const incompatibleFailure = await client.failure( + incompatibleRestart, + "restartFrame" + ); + assert.match(incompatibleFailure.message, /incompatible source edit/i); + assert.match(incompatibleFailure.message, /whole virtual-process restart/i); + + await client.close(); + } catch (error) { + if (client.child.exitCode === null) client.child.kill("SIGKILL"); + throw error; + } + + const failMainLine = + sourceLines.findIndex((line) => line.includes("pub async fn fail_main()")) + 1; + assert(failMainLine > 0, "flagship source must contain fail_main"); + const taskTrapLine = + sourceLines.findIndex((line) => line.includes("fn task_trap(")) + 1; + assert(taskTrapLine > 0, "flagship source must contain task_trap"); + const restartClient = new DapClient(); + try { + const initialize = restartClient.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true, + }); + await restartClient.response(initialize, "initialize"); + const launch = restartClient.send("launch", { + entry: "fail", + project, + runtimeBackend: "local-services", + }); + await restartClient.response(launch, "launch"); + await restartClient.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + const breakpoints = restartClient.send("setBreakpoints", { + source: { path: sourcePath }, + breakpoints: [{ line: failMainLine }, { line: taskTrapLine }], + }); + const breakpointResponse = await restartClient.response( + breakpoints, + "setBreakpoints" + ); + assert.deepStrictEqual( + breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified), + [true, true] + ); + const configurationDone = restartClient.send("configurationDone"); + await restartClient.response(configurationDone, "configurationDone"); + const initialStop = await restartClient.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint" + ); + assert.strictEqual(initialStop.body.allThreadsStopped, true); + const threadsRequest = restartClient.send("threads"); + const threads = ( + await restartClient.response(threadsRequest, "threads") + ).body.threads; + const failThread = threads.find( + (thread) => thread.id === initialStop.body.threadId + ); + assert(failThread, "failed entrypoint must remain a virtual task thread"); + const stackRequest = restartClient.send("stackTrace", { + threadId: failThread.id, + startFrame: 0, + levels: 1, + }); + const failedStack = ( + await restartClient.response(stackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual(failedStack[0].line, failMainLine); + + const continueRequest = restartClient.send("continue", { + threadId: failThread.id, + }); + await restartClient.response(continueRequest, "continue"); + const childStop = await restartClient.waitFor( + (message) => + message.seq > initialStop.seq && + message.type === "event" && + message.event === "stopped" && + message.body.reason === "breakpoint", + 70000 + ); + assert.strictEqual(childStop.body.allThreadsStopped, true); + + const childThreadsRequest = restartClient.send("threads"); + const childThreads = ( + await restartClient.response(childThreadsRequest, "threads") + ).body.threads; + const childThread = childThreads.find( + (thread) => thread.id === childStop.body.threadId + ); + assert(childThread, "executing child task must become a DAP virtual thread"); + assert.notStrictEqual(childThread.id, failThread.id); + assert.match(childThread.name, /task trap/i); + + const childStackRequest = restartClient.send("stackTrace", { + threadId: childThread.id, + startFrame: 0, + levels: 1, + }); + const childStack = ( + await restartClient.response(childStackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual(childStack[0].line, taskTrapLine); + assert.match(childStack[0].name, /task_trap::wasm/); + + const childScopesRequest = restartClient.send("scopes", { + frameId: childStack[0].id, + }); + const childScopes = ( + await restartClient.response(childScopesRequest, "scopes") + ).body.scopes; + const childArgsScope = childScopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + assert(childArgsScope, "child task argument scope must be present"); + const childArgsRequest = restartClient.send("variables", { + variablesReference: childArgsScope.variablesReference, + }); + const childArgs = ( + await restartClient.response(childArgsRequest, "variables") + ).body.variables; + assert( + childArgs.some( + (variable) => + variable.name === "arg_0" && + /SmallJson\(Number\(0\)\)/.test(String(variable.value)) + ), + "child task argument must come from the frozen node participant" + ); + + const parentScopesRequest = restartClient.send("scopes", { + frameId: failedStack[0].id, + }); + const parentScopes = ( + await restartClient.response(parentScopesRequest, "scopes") + ).body.scopes; + const parentArgsScope = parentScopes.find( + (scope) => scope.name === "Task Args and Handles" + ); + const parentArgsRequest = restartClient.send("variables", { + variablesReference: parentArgsScope.variablesReference, + }); + const parentArgs = ( + await restartClient.response(parentArgsRequest, "variables") + ).body.variables; + assert( + parentArgs.some( + (variable) => + /^task_handle_\d+$/.test(variable.name) && + /definition=task_trap instance=ti:.*:child:\d+ state=active/.test( + variable.value + ) && + variable.type === "runtime-handle" + ), + "parent task handle must come from its live Wasm host registry" + ); + + const continueChildRequest = restartClient.send("continue", { + threadId: childThread.id, + }); + await restartClient.response(continueChildRequest, "continue"); + await restartClient.waitFor( + (message) => + message.seq > childStop.seq && + message.type === "event" && + message.event === "terminated" + ); + + const restartRequest = restartClient.send("restartFrame", { + frameId: failedStack[0].id, + }); + await restartClient.response(restartRequest, "restartFrame"); + const restartedStop = await restartClient.waitFor( + (message) => + message.type === "event" && + message.event === "stopped" && + /Restarted main from the rebuilt bundle/.test(message.body.description) + ); + assert.strictEqual(restartedStop.body.allThreadsStopped, true); + const restartedStackRequest = restartClient.send("stackTrace", { + threadId: restartedStop.body.threadId, + startFrame: 0, + levels: 1, + }); + const restartedStack = ( + await restartClient.response(restartedStackRequest, "stackTrace") + ).body.stackFrames; + assert.strictEqual(restartedStack[0].line, failMainLine); + await restartClient.close(); + } catch (error) { + if (restartClient.child.exitCode === null) restartClient.child.kill("SIGKILL"); + throw error; + } + + console.log("DAP smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/flagship-demo-smoke.js b/scripts/flagship-demo-smoke.js new file mode 100644 index 0000000..28ee190 --- /dev/null +++ b/scripts/flagship-demo-smoke.js @@ -0,0 +1,148 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); + +const extension = require("../vscode-extension/extension"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/launch-build-demo"); +const source = fs.readFileSync(path.join(project, "src/build.rs"), "utf8"); +const forbiddenSourceAssumptions = + /\b(?:std::fs|std::process|git|podman|docker|localhost|127\.0\.0\.1)|\/home\/|\/Users\/|C:\\Users\\/i; + +const envs = extension.discoverEnvironmentNames(project); +assert.deepStrictEqual(envs, ["linux", "windows"]); +assert.deepStrictEqual(extension.diagnoseEnvReferences(source, envs), []); +assert.doesNotMatch( + source, + forbiddenSourceAssumptions, + "flagship build source must not bypass Clusterflux host capabilities or rely on coordinator-side filesystem, Git, container, or machine-local assumptions" +); + +const inspection = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "bundle", + "inspect", + "--project", + project, + "--json" + ], + { cwd: repo, encoding: "utf8" } + ) +); + +assert.strictEqual(inspection.project, project); +assert.strictEqual( + inspection.source_provider_manifest.coordinator_requires_checkout_access, + false +); +assert.strictEqual( + inspection.source_provider_manifest.transfer_policy.local_source_bytes_remain_node_local, + true +); +assert.strictEqual( + inspection.source_provider_manifest.transfer_policy.coordinator_receives_source_bytes_by_default, + false +); +assert.strictEqual( + inspection.source_provider_manifest.transfer_policy.default_full_repo_tarball, + false +); +assert.deepStrictEqual( + inspection.source_provider_manifest.transfer_policy.allowed_remote_transfer.sort(), + ["ExplicitSnapshotChunks", "RequiredContent"] +); +assert.strictEqual(inspection.metadata.embeds_full_container_images, false); +assert(inspection.metadata.environments.some((env) => env.name === "linux")); +assert(inspection.metadata.environments.some((env) => env.name === "windows")); +assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/build.rs")); + +const bundleDirectory = path.join(repo, "target/acceptance/flagship-bundle"); +const build = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "build", + "--project", + project, + "--output", + bundleDirectory, + "--json", + ], + { cwd: repo, encoding: "utf8" } + ) +); +assert.strictEqual(build.status, "built"); +assert.strictEqual(build.bundle_artifact.task_descriptor_count, 10); +assert.strictEqual(build.bundle_artifact.entrypoint_count, 5); +assert.deepStrictEqual(build.bundle_artifact.files.sort(), [ + "debug-metadata.json", + "entrypoints.json", + "environments.json", + "manifest.json", + "module.wasm", + "source-provider.json", + "task-descriptors.json", + "vfs-seed.json", +]); +const bundleManifest = JSON.parse( + fs.readFileSync(path.join(bundleDirectory, "manifest.json"), "utf8") +); +const bundleEntrypoints = JSON.parse( + fs.readFileSync(path.join(bundleDirectory, "entrypoints.json"), "utf8") +); +assert.deepStrictEqual( + bundleEntrypoints.map((entrypoint) => entrypoint.name).sort(), + ["build", "fail", "long-join", "park-wake", "restart"] +); +const bundleModule = fs.readFileSync(path.join(bundleDirectory, "module.wasm")); +assert.strictEqual(bundleManifest.kind, "clusterflux-bundle"); +assert.strictEqual( + bundleManifest.bundle_digest, + `sha256:${crypto.createHash("sha256").update(bundleModule).digest("hex")}` +); +assert.strictEqual(bundleManifest.embeds_full_repository, false); +assert.strictEqual( + bundleManifest.coordinator_receives_source_bytes_by_default, + false +); +const taskDescriptors = JSON.parse( + fs.readFileSync(path.join(bundleDirectory, "task-descriptors.json"), "utf8") +); +assert( + taskDescriptors.some( + (task) => + task.name === "task_add_one" && + task.argument_schema === "input : i32" && + task.result_schema === "i32" && + task.restart_compatibility_hash.startsWith("sha256:") && + task.probe_symbol === "clusterflux.probe.task_add_one" + ) +); + +cp.execFileSync("cargo", ["test", "-p", "launch-build-demo"], { + cwd: repo, + stdio: "inherit" +}); + +console.log("Flagship demo smoke passed"); diff --git a/scripts/hostile-input-contract-smoke.js b/scripts/hostile-input-contract-smoke.js new file mode 100644 index 0000000..0be6153 --- /dev/null +++ b/scripts/hostile-input-contract-smoke.js @@ -0,0 +1,184 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing hostile-input evidence: ${name}`); +} + +const coreSource = read("crates/clusterflux-core/src/source.rs"); +const coreCapabilities = read("crates/clusterflux-core/src/capability.rs"); +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/signed_nodes.rs"), + read("crates/clusterflux-coordinator/src/service/logs.rs"), + read("crates/clusterflux-coordinator/src/service/tests.rs"), +].join("\n"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); +const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); + +for (const [name, pattern] of [ + ["source manifests validate shape", /pub fn validate_public_mvp\(&self\)[\s\S]*self\.validate_shape\(\)\?/], + ["source manifests reject invalid digests", /SourceManifestError::InvalidDigest/], + ["source manifests reject invalid custom providers", /SourceManifestError::InvalidProviderId/], + ["source manifests reject control characters", /DescriptionControlCharacter/], + ["source manifests reject coordinator checkout access", /CoordinatorCheckoutAccess/], + ["source manifests reject default source-byte upload", /CoordinatorReceivesSourceBytes/], +]) { + expect(coreSource, name, pattern); +} + +for (const [name, pattern] of [ + ["capability reports validate public shape", /pub fn validate_public_report\(&self\)/], + ["capability reports validate architecture labels", /InvalidArchitecture/], + ["capability reports validate OS labels", /InvalidOsLabel/], + ["capability reports validate source providers", /InvalidSourceProvider/], + ["source provider ids reject path traversal", /valid_source_provider_id/], +]) { + expect(coreCapabilities, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator task log tails are bounded", /MAX_TASK_LOG_TAIL_BYTES: usize = 256 \* 1024/], + ["coordinator validates reported stdout tails", /ReportTaskLog[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], + ["coordinator validates completed task stdout tails", /TaskCompleted[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/], + ["coordinator rejects oversized log tail in unit coverage", /"x"\.repeat\(MAX_TASK_LOG_TAIL_BYTES \+ 1\)/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, pattern] of [ + ["service rejects malformed node capability report", /fn service_rejects_malformed_node_capability_report\(\)/], + ["capability report rejection leaves descriptors empty", /assert!\(service\.node_descriptors\.is_empty\(\)\)/], + ["node capability report rejects cross-scope writes", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], + ["task completion rejects cross-scope writes", /task completion outside node scope|outside/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, source, patterns] of [ + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /const crossTenant = await send/, + /const crossProject = await send/, + /const guessed = await send/, + /const crossActorOpen = await send/, + /token is invalid/, + /tenant mismatch/, + /project mismatch/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [ + /render_operator_panel/, + /submit_panel_event/, + /assert\(!JSON\.stringify\(panel\)\.includes\(" max_bytes[\s\S]*contains unsupported characters/], + ["tokens are bounded", /fn validate_token[\s\S]*value\.len\(\) > max_bytes[\s\S]*contains unsupported characters/], + ["control request bodies are bounded", /MAX_CONTROL_FRAME_BYTES[\s\S]*control request too large/], + ["identity protocol rejects unknown authority fields", /deny_unknown_fields/], + ]) { + expect(hostedService, name, pattern); + } + + for (const [name, pattern] of [ + ["old client identity protocol is rejected", /hosted_login_protocol_rejects_client_identity_and_provider_configuration/], + ["raw operator action is rejected", /rawOperatorDenied[\s\S]*hosted_operator_request envelope/], + ["unsigned client identity is rejected", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], + ["cross-tenant process inspection is rejected", /crossTenantTaskEventsDenied[\s\S]*scope\|denied\|unauthorized/], + ]) { + expect(hostedTests, name, pattern); + } +} + +console.log("Hostile input contract smoke passed"); diff --git a/scripts/migrate-clusterflux-state.sh b/scripts/migrate-clusterflux-state.sh new file mode 100755 index 0000000..356091d --- /dev/null +++ b/scripts/migrate-clusterflux-state.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +migrate_path() { + local source="$1" + local destination="$2" + if [[ ! -e "$source" ]]; then + return + fi + if [[ -e "$destination" ]]; then + printf 'refusing to overwrite existing destination: %s\n' "$destination" >&2 + exit 1 + fi + mv -- "$source" "$destination" + printf 'migrated %s -> %s\n' "$source" "$destination" +} + +migrate_path "${HOME}/.disasmer" "${HOME}/.clusterflux" +migrate_path "${XDG_CONFIG_HOME:-${HOME}/.config}/disasmer" \ + "${XDG_CONFIG_HOME:-${HOME}/.config}/clusterflux" +migrate_path "${PWD}/.disasmer" "${PWD}/.clusterflux" +migrate_path "${PWD}/disasmer.toml" "${PWD}/clusterflux.toml" diff --git a/scripts/node-attach-smoke.js b/scripts/node-attach-smoke.js new file mode 100755 index 0000000..d6c4e01 --- /dev/null +++ b/scripts/node-attach-smoke.js @@ -0,0 +1,300 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); +const identities = new Map(); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function nodeIdentity(node) { + const existing = identities.get(node); + if (existing) return existing; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + const privateSeed = Buffer.from(privateDer).subarray(-32); + const identity = { + privateKey: `ed25519:${privateSeed.toString("base64")}`, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; + identities.set(node, identity); + return identity; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeHeartbeat(node, identity) { + const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const payloadDigest = `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify({ node, type: "node_heartbeat" })) + .digest("hex")}`; + const signature = crypto.sign( + null, + nodeSignatureMessage(node, "node_heartbeat", payloadDigest, nonce, issuedAt), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function runAttach(addr, grant) { + const identity = nodeIdentity("node-attach"); + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "node", + "attach", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + "node-attach", + "--public-key", + identity.publicKey, + "--enrollment-grant", + grant, + "--cap", + "quic-direct", + "--json", + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`node attach failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject( + new Error(`node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) + ); + } + }); + }); +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const grant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(grant.type, "node_enrollment_grant_created"); + assert.strictEqual(grant.tenant, "tenant"); + assert.strictEqual(grant.project, "project"); + assert.match(grant.grant, /^node_grant_[A-Za-z0-9_-]+$/); + assert.strictEqual(grant.scope, "node:attach"); + assert(grant.expires_at_epoch_seconds > Math.floor(Date.now() / 1000)); + assert(grant.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 900); + + const report = await runAttach(addr, grant.grant); + assert.strictEqual(report.plan.node, "node-attach"); + assert.strictEqual(report.plan.coordinator, `${addr.host}:${addr.port}`); + assert.strictEqual(report.plan.enrollment.grant, grant.grant); + assert.match(report.plan.enrollment.public_key_fingerprint, /^sha256:[0-9a-f]{64}$/); + assert.strictEqual( + report.plan.enrollment.exchanges_short_lived_grant_for_long_lived_node_identity, + true + ); + assert.ok(report.plan.capabilities.arch.length > 0); + assert.ok(report.plan.capabilities.source_providers.includes("filesystem")); + assert.ok(report.plan.capabilities.capabilities.includes("QuicDirect")); + assert.strictEqual(report.plan.detection.auto_detected, true); + assert.strictEqual(report.plan.detection.arch, report.plan.capabilities.arch); + assert.deepStrictEqual(report.plan.detection.manual_capability_overrides, ["quic-direct"]); + assert( + report.plan.detection.recognized_capability_overrides.includes("QuicDirect") + ); + assert.strictEqual( + report.plan.detection.os_arch_capabilities_require_manual_flags, + false + ); + assert.strictEqual(report.plan.detection.command_backend, "native-command"); + assert.strictEqual(report.plan.detection.command_backend_available, true); + assert( + report.plan.detection.source_provider_backends.some( + (provider) => provider.provider === "filesystem" && provider.detected + ) + ); + assert( + report.grant_disclosures.length > 0, + "node attach should disclose capability grants before reporting capabilities" + ); + assert( + report.grant_disclosures.every( + (disclosure) => disclosure.coordinator_policy_limited === true + ), + "node attach should mark all capability grants as coordinator-policy-limited" + ); + assert( + report.grant_disclosures.some( + (disclosure) => disclosure.grant === "native_command_execution" + ), + "node attach should disclose native command execution when detected" + ); + assert( + report.grant_disclosures.some( + (disclosure) => disclosure.grant === "source_access" + ), + "node attach should disclose source access when detected" + ); + assert.strictEqual(report.boundary.cli_contacted_coordinator, true); + assert.strictEqual(report.boundary.used_enrollment_exchange, true); + assert.strictEqual(report.boundary.coordinator_session_requests, 3); + assert.strictEqual(report.coordinator_response.type, "node_enrollment_exchanged"); + assert.strictEqual(report.coordinator_response.node, "node-attach"); + assert.strictEqual(report.coordinator_response.credential.node, "node-attach"); + assert.strictEqual(report.coordinator_response.credential.scope, "node:attach"); + assert.strictEqual(report.coordinator_response.credential.credential_kind, "NodeCredential"); + assert.match( + report.coordinator_response.credential.capability_policy_digest, + /^sha256:[0-9a-f]{64}$/ + ); + assert.strictEqual(report.heartbeat_response.type, "node_heartbeat"); + assert.strictEqual(report.capability_response.type, "node_capabilities_recorded"); + + const heartbeat = await send(addr, { + type: "node_heartbeat", + node: "node-attach", + node_signature: signedNodeHeartbeat("node-attach", nodeIdentity("node-attach")), + }); + assert.strictEqual(heartbeat.type, "node_heartbeat"); + assert.strictEqual(heartbeat.node, "node-attach"); + + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Node attach smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/node-lifecycle-contract-smoke.js b/scripts/node-lifecycle-contract-smoke.js new file mode 100755 index 0000000..b597da9 --- /dev/null +++ b/scripts/node-lifecycle-contract-smoke.js @@ -0,0 +1,151 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing node lifecycle evidence: ${name}`); +} + +const nodeMain = read("crates/clusterflux-node/src/daemon.rs"); +const nodeIdentity = read("crates/clusterflux-node/src/node_identity.rs"); +const cliNode = read("crates/clusterflux-cli/src/node.rs"); +const nodeTaskReports = read("crates/clusterflux-node/src/task_reports.rs"); +const nodeDebugAgent = read("crates/clusterflux-node/src/debug_agent.rs"); +const nodeLib = read("crates/clusterflux-node/src/lib.rs"); +const sharedWasmtimeRuntime = `${read("crates/clusterflux-wasm-runtime/src/lib.rs")}\n${read("crates/clusterflux-wasm-runtime/src/task_host_linker.rs")}`; +const nodeRuntimeSurface = `${nodeLib}\n${sharedWasmtimeRuntime}`; +const nodeLifecycleSurface = `${nodeMain}\n${nodeIdentity}\n${nodeTaskReports}\n${nodeDebugAgent}`; +const nodeAssignmentRunner = `${read("crates/clusterflux-node/src/assignment_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/control_watcher.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/process_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/validation.rs")}`; +const coordinatorCore = read("crates/clusterflux-coordinator/src/lib.rs"); +const coordinatorService = `${read("crates/clusterflux-coordinator/src/service.rs")}\n${read("crates/clusterflux-coordinator/src/service/routing.rs")}`; +const coordinatorServiceTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); +const coordinatorServiceSurface = `${coordinatorService}\n${coordinatorServiceTests}`; +const cliLocalRunSmoke = read("scripts/cli-local-run-smoke.js"); +const liveSmoke = read("scripts/cli-happy-path-live-smoke.js"); +const wasmtimeSmoke = read("scripts/wasmtime-node-smoke.js"); +const debugCore = read("crates/clusterflux-core/src/debug.rs"); + +assert.strictEqual( + (nodeMain.match(/CoordinatorSession::connect/g) || []).length, + 1, + "node runtime should open one coordinator session in the local process-boundary runtime" +); + +for (const [name, pattern] of [ + ["enrollment exchange over session", /"type": "exchange_node_enrollment_grant"/], + ["persisted node identity is reused locally", /"type": "node_identity_reused"/], + ["heartbeat over session", /"type": "node_heartbeat"/], + ["node-originated requests use signed envelope", /"type": "signed_node"/], + ["capability report over session", /"type": "report_node_capabilities"/], + ["task assignment polling over session", /"type": "poll_task_assignment"/], + ["process start over session", /"type": "start_process"/], + ["reconnect over session", /"type": "reconnect_node"/], + ["debug command polling over session", /"type": "poll_debug_command"/], + ["log event over session", /"type": "report_task_log"/], + ["VFS metadata over session", /"type": "report_vfs_metadata"/], + ["task control polling over session", /"type": "poll_task_control"/], + ["completion over session", /"type": "task_completed"/], + ["cancellation uses same session", /poll_task_cancellation\(session, args, &task, node_private_key\)/], + ["request count is reported", /session\.requests\(\)/], +]) { + expect(nodeLifecycleSurface, name, pattern); +} + +expect(cliNode, "user-authorized node attach over Client session", /"type": "attach_node"/); +assert.doesNotMatch( + nodeIdentity, + /"type": "attach_node"/, + "a persisted node must authenticate with its signed identity instead of replaying Client attach" +); + +const runtimeAcceptanceSurface = `${cliLocalRunSmoke}\n${liveSmoke}\n${coordinatorServiceTests}\n${nodeLib}\n${nodeAssignmentRunner}`; +for (const [name, pattern] of [ + ["real CLI launches a coordinator main", /node_report\.run\.status, "main_launched"/], + ["real CLI observes coordinator-main task spawning", /task_spawn_host_import, true/], + ["real CLI keeps child task instances distinct", /every live task event must retain its unique instance identity/], + ["signed active Wasm parent spawns and joins child", /fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only\(\)/], + ["controlled native process abort is exercised", /fn abort_requested[\s\S]*poll_task_control/], + ["native lifecycle freeze and resume are exercised", /linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume/], + ["strict live run requires a usable partial freeze", /partial_freeze\.partially_frozen/], + ["strict live run requires hosted restart evidence", /serviceRestart\.executed === true/], +]) { + expect(runtimeAcceptanceSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["cooperative cancellation and abort are distinct", /cancel_requested: false,[\s\S]*abort_requested: true/], + ["controlled runner polls abort while command runs", /fn abort_requested[\s\S]*poll_task_control/], + ["controlled runner creates a process group", /process\.process_group\(0\)/], + ["controlled runner freezes the native process group", /libc::kill\(process_group, libc::SIGSTOP\)/], + ["controlled runner resumes the native process group", /libc::kill\(process_group, libc::SIGCONT\)/], + ["controlled runner kills the process group", /libc::kill\(process_group, libc::SIGKILL\)/], + ["Wasm code can poll cooperative cancellation", /task_control_v1/], + ["matched Wasm probes remain at a quiescent boundary", /TaskHostOperation::DebugProbe[\s\S]*enter_quiescent_host_boundary[\s\S]*leave_quiescent_host_boundary/], + ["debug snapshots use the live Wasm task handle registry", /debug_handle_snapshot[\s\S]*task_handle_\{handle_id\}[\s\S]*state=active/], + ["native command status comes from the controlled runner", /set_command_status[\s\S]*frozen command pid[\s\S]*native command exited with status/], +]) { + expect(`${coordinatorServiceSurface}\n${nodeLifecycleSurface}\n${sharedWasmtimeRuntime}\n${nodeAssignmentRunner}`, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/], + ["reconnect preserves enrolled node identity", /reconnect_node\(&NodeId::from\("node"\), None\)/], + ["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/], +]) { + expect(coordinatorCore, name, pattern); +} + +for (const [name, pattern] of [ + ["coordinator delivers cancellation to connected node", /fn service_delivers_cancellation_to_connected_node_and_records_terminal_state\(\)/], + ["node polls task control", /CoordinatorRequest::PollTaskControl/], + ["cancelled terminal state is recorded", /TaskTerminalState::Cancelled/], +]) { + expect(coordinatorServiceSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["native lifecycle test exists", /fn linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume\(\)/], + ["native freeze succeeds when supported", /lifecycle\.freeze_for_debug_epoch\(\)\.unwrap\(\)/], + ["native resume succeeds", /lifecycle\.resume_after_debug_epoch\(\)/], + ["native cancel reaches lifecycle", /lifecycle\.cancel\(\)/], + ["unsupported freeze errors", /BackendError::DebugFreezeUnsupported/], + ["wasmtime runtime exposes freeze resume probe", /pub fn freeze_resume_i32_export_probe/], + ["wasmtime runtime captures Wasm frame locals", /debug_i32_export_snapshot[\s\S]*local_values/], + ["wasmtime runtime creates Wasm debug participant", /kind: DebugParticipantKind::WasmTask/], + ["wasmtime debug participant carries local values", /local_values: snapshot\.local_values\.clone\(\)/], + ["wasmtime runtime resumes after freeze", /epoch\.continue_all\(\)/], +]) { + expect(nodeRuntimeSurface, name, pattern); +} + +for (const [name, pattern] of [ + ["wasmtime smoke runs debug freeze resume mode", /--debug-freeze-resume/], + ["wasmtime smoke verifies frozen state", /debugReport\.frozen_state, "Frozen"/], + ["wasmtime smoke verifies resumed state", /debugReport\.resumed_state, "Running"/], + ["wasmtime smoke verifies frame local values", /debugReport\.local_values[\s\S]*wasm_local_0/], + ["wasmtime smoke proves node runtime reached wasm task", /node_runtime_reached_wasm_task/], + ["wasmtime smoke proves node captured locals", /node_runtime_captured_wasm_locals/], +]) { + expect(wasmtimeSmoke, name, pattern); +} + +for (const [name, pattern] of [ + ["debug model freezes wasm and command participants", /fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks\(\)/], + ["debug model rejects unsupported freeze", /fn debug_epoch_reports_failure_when_no_participant_can_freeze\(\)/], + ["debug model resumes frozen participants", /fn continue_resumes_every_frozen_participant\(\)/], + ["debug model includes captured locals", /local_values/], + ["wasm participants are modeled", /DebugParticipantKind::WasmTask/], + ["controlled native command participants are modeled", /DebugParticipantKind::ControlledNativeCommand/], +]) { + expect(debugCore, name, pattern); +} + +console.log("Node lifecycle contract smoke passed"); diff --git a/scripts/node-signing.js b/scripts/node-signing.js new file mode 100644 index 0000000..e50a27a --- /dev/null +++ b/scripts/node-signing.js @@ -0,0 +1,181 @@ +const crypto = require("crypto"); +const identities = new Map(); + +function nodeIdentity(identityPurpose, node) { + const identityKey = `${identityPurpose}:${node}`; + const existing = identities.get(identityKey); + if (existing) return existing; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" }); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + const privateSeed = Buffer.from(privateDer).subarray(-32); + const identity = { + privateKey: `ed25519:${privateSeed.toString("base64")}`, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; + identities.set(identityKey, identity); + return identity; +} + +function nodeIdentityFromPrivateKey(privateKey) { + if (typeof privateKey !== "string" || !privateKey.startsWith("ed25519:")) { + throw new Error("node private key must use ed25519: encoding"); + } + const seed = Buffer.from(privateKey.slice("ed25519:".length), "base64"); + if (seed.length !== 32) throw new Error("node private key must contain 32 bytes"); + const privateKeyObject = crypto.createPrivateKey({ + key: Buffer.concat([ + Buffer.from("302e020100300506032b657004220420", "hex"), + seed, + ]), + format: "der", + type: "pkcs8", + }); + const publicKeyObject = crypto.createPublicKey(privateKeyObject); + const publicDer = publicKeyObject.export({ format: "der", type: "spki" }); + return { + privateKey, + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; +} + +function canonicalSignedRequest(value, topLevel = true) { + if (Array.isArray(value)) { + return value.map((entry) => canonicalSignedRequest(entry, false)); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter( + ([key, entry]) => + entry !== null && + (!topLevel || !["agent_signature", "node_signature"].includes(key)) + ) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalSignedRequest(entry, false)]) + ); + } + return value; +} + +function withWireDefaults(request) { + const value = { ...request }; + if (value.type === "report_node_capabilities") { + value.dependency_cache_digests ??= []; + } else if (value.type === "launch_task" || value.type === "launch_child_task") { + value.wait_for_node ??= false; + if (value.task_spec && typeof value.task_spec === "object") { + value.task_spec = { ...value.task_spec }; + value.task_spec.failure_policy ??= "fail_fast"; + } + } else if (value.type === "start_process") { + value.restart ??= false; + } else if (value.type === "report_debug_state") { + value.stack_frames ??= []; + value.local_values ??= []; + value.task_args ??= []; + value.handles ??= []; + value.recent_output ??= []; + } else if (value.type === "report_task_log") { + value.stdout_tail ??= ""; + value.stderr_tail ??= ""; + } else if (value.type === "task_completed") { + value.stdout_tail ??= ""; + value.stderr_tail ??= ""; + value.stdout_truncated ??= false; + value.stderr_truncated ??= false; + } + return value; +} + +function signedRequestPayloadDigest(request) { + return `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify(canonicalSignedRequest(withWireDefaults(request)))) + .digest("hex")}`; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeProof(node, identity, requestKind, request, options = {}) { + const nonce = + options.nonce || + `${requestKind}-${process.pid}-${Date.now()}-${crypto + .randomBytes(8) + .toString("hex")}`; + const issuedAt = + options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + requestKind, + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedNodeHeartbeat(node, identity, options = {}) { + const request = { type: "node_heartbeat", node }; + return signedNodeProof(node, identity, "node_heartbeat", request, options); +} + +function signedNodeRequest(node, identity, requestKind, request, options = {}) { + return { + type: "signed_node", + node, + node_signature: signedNodeProof( + node, + identity, + requestKind, + request, + options + ), + request, + }; +} + +module.exports = { + nodeIdentity, + nodeIdentityFromPrivateKey, + signedNodeProof, + signedRequestPayloadDigest, + signedNodeHeartbeat, + signedNodeRequest, +}; diff --git a/scripts/operator-panel-smoke.js b/scripts/operator-panel-smoke.js new file mode 100755 index 0000000..a45d694 --- /dev/null +++ b/scripts/operator-panel-smoke.js @@ -0,0 +1,367 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const { nodeIdentity } = require("./node-signing"); +const { + ensureRootlessPodman, + repo, + runFlagshipWorker, + send, + startFlagship, + waitForTaskEvent, + waitForJsonLine, + waitForNodeStatus, +} = require("./real-flagship-harness"); + +const panelNode = "panel-node"; +const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode); + +function widget(panel, id) { + const item = panel.widgets[id]; + assert(item, `missing panel widget ${id}`); + return item; +} + +const delay = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForBreakpointHit(addr, process) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const status = await send(addr, { + type: "inspect_debug_breakpoints", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + assert.strictEqual(status.type, "debug_breakpoints", JSON.stringify(status)); + if (status.hit_epoch != null) return status; + await delay(25); + } + throw new Error(`timed out waiting for package breakpoint in ${process}`); +} + +async function waitForDebugEpochFrozen(addr, process, epoch) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const status = await send(addr, { + type: "inspect_debug_epoch", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + epoch, + }); + assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status)); + if (status.failed) { + throw new Error(status.failure_messages.join("; ")); + } + if (status.fully_frozen) return status; + await delay(25); + } + throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`); +} + +(async () => { + ensureRootlessPodman(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + let worker; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + const projectCreated = await send(addr, { + type: "create_project", + tenant: "tenant", + actor_user: "user", + project: "project", + name: "Operator panel smoke", + }); + assert.strictEqual(projectCreated.type, "project_created"); + + worker = await runFlagshipWorker(addr, panelNode, panelNodeIdentity); + const workerReady = await worker.ready; + assert.strictEqual(workerReady.node_status, "ready"); + const workerCompletion = waitForNodeStatus(worker.child, "completed"); + const flagship = startFlagship(addr); + await waitForTaskEvent( + addr, + flagship.process, + (event) => event.task_definition === "prepare_source", + "prepare_source before breakpoint configuration" + ); + const configuredBreakpoints = await send(addr, { + type: "set_debug_breakpoints", + tenant: "tenant", + project: "project", + actor_user: "user", + process: flagship.process, + probe_symbols: ["clusterflux.probe.package_release"], + }); + assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints"); + const breakpointHit = await waitForBreakpointHit(addr, flagship.process); + assert.strictEqual( + breakpointHit.hit_probe_symbol, + "clusterflux.probe.package_release" + ); + const frozenEpoch = await waitForDebugEpochFrozen( + addr, + flagship.process, + breakpointHit.hit_epoch + ); + assert(frozenEpoch.acknowledgements.length >= 2); + const compileEvent = await waitForTaskEvent( + addr, + flagship.process, + (event) => event.task_definition === "compile_linux", + "compile_linux before the package breakpoint" + ); + const report = await workerCompletion; + assert.strictEqual(report.node_status, "completed"); + assert.strictEqual(report.coordinator_response.type, "task_recorded"); + const process = flagship.process; + + const rendered = await send(addr, { + type: "render_operator_panel", + tenant: "tenant", + project: "project", + process, + actor_user: "user", + max_download_bytes: 1024 * 1024, + stopped: false + }); + assert.strictEqual(rendered.type, "operator_panel"); + const panel = rendered.panel; + assert.strictEqual(panel.tenant, "tenant"); + assert.strictEqual(panel.project, "project"); + assert.strictEqual(panel.process, process); + assert.strictEqual(panel.program_ui_events_enabled, true); + + assert.deepStrictEqual(widget(panel, "process-status").kind, { + Text: { value: "running" } + }); + const taskProgress = widget(panel, "task-progress").kind.Progress; + assert(taskProgress.current >= 2); + assert.strictEqual(taskProgress.current, taskProgress.total); + const taskSummary = widget(panel, "task-summary").kind.Text.value; + assert.match( + taskSummary, + new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`) + ); + assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/); + const downloadWidget = widget(panel, "download-artifact").kind; + const artifact = downloadWidget.ArtifactDownload.artifact; + assert( + artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length), + "panel download must point at a real flagship artifact" + ); + assert(!JSON.stringify(downloadWidget).includes("url_path")); + assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest")); + assert.deepStrictEqual(widget(panel, "debug-process").kind, { + Button: { action: "debug-process" } + }); + assert.deepStrictEqual(widget(panel, "cancel-process").kind, { + Button: { action: "cancel-process" } + }); + assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, { + Button: { action: "restart-task" } + }); + assert(panel.control_plane_actions.includes("DebugProcess")); + assert(panel.control_plane_actions.includes("CancelProcess")); + const restartTarget = panel.control_plane_actions.find( + (action) => action.RestartTask + )?.RestartTask; + assert( + restartTarget && taskSummary.includes(`[${restartTarget}]`), + "panel restart must target the real flagship task instance" + ); + assert( + panel.control_plane_actions.some( + (action) => action.DownloadArtifact === artifact + ) + ); + assert(!JSON.stringify(panel).includes(" action.DownloadArtifact === artifact + ) + ); + + const frozenEvent = await send(addr, { + type: "submit_panel_event", + tenant: "tenant", + project: "project", + process, + widget_id: "debug-process", + kind: "ButtonClicked", + max_events: 10 + }); + assert.strictEqual(frozenEvent.type, "error"); + assert.match(frozenEvent.message, /program UI events are disabled/i); + + const crossTenant = await send(addr, { + type: "render_operator_panel", + tenant: "other", + project: "project", + process, + actor_user: "user", + max_download_bytes: 1024 * 1024, + stopped: false + }); + assert.strictEqual(crossTenant.type, "error"); + assert.match( + crossTenant.message, + /scope|tenant|project|requires an active virtual process/i + ); + assert(!crossTenant.message.includes(process)); + + const resumed = await send(addr, { + type: "resume_debug_epoch", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + epoch: breakpointHit.hit_epoch, + }); + assert.strictEqual(resumed.type, "debug_epoch"); + const cleanup = await send(addr, { + type: "abort_process", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + assert.strictEqual(cleanup.type, "process_aborted"); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + worker?.child.kill("SIGTERM"); + coordinator.kill("SIGTERM"); + } + + console.log("Operator panel smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/podman-backend-smoke.js b/scripts/podman-backend-smoke.js new file mode 100755 index 0000000..438c2df --- /dev/null +++ b/scripts/podman-backend-smoke.js @@ -0,0 +1,86 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const path = require("path"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +const baseImage = "docker.io/library/alpine:3.20"; + +// Nix's standalone Podman package may not install the distribution-level +// containers/image policy normally found under /etc. Use an isolated test HOME +// without replacing a policy supplied by the host. +configurePodmanTestEnvironment(repo); + +function run(command, args, options = {}) { + return cp.execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: options.stdio || ["ignore", "pipe", "pipe"], + }); +} + +function incomplete(reason) { + const error = new Error(`Linux Podman backend incomplete: ${reason}`); + error.code = "CLUSTERFLUX_PODMAN_INCOMPLETE"; + throw error; +} + +function ensurePodmanBaseImage() { + try { + run("podman", ["--version"]); + } catch (error) { + incomplete(`podman command is unavailable (${error.message})`); + } + + let rootless; + try { + rootless = run("podman", ["info", "--format", "{{.Host.Security.Rootless}}"]).trim(); + } catch (error) { + incomplete(`podman info did not report rootless status (${error.message})`); + } + if (rootless !== "true") { + incomplete(`podman is not running in rootless mode (reported ${JSON.stringify(rootless)})`); + } + + try { + run("podman", ["image", "exists", baseImage]); + } catch (_) { + try { + run("podman", ["pull", baseImage], { stdio: "inherit" }); + } catch (error) { + incomplete(`unable to make ${baseImage} available (${error.message})`); + } + } +} + +try { + ensurePodmanBaseImage(); + + const stdout = run("cargo", [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-podman-smoke" + ]); + const report = JSON.parse(stdout.trim().split("\n").at(-1)); + + assert.strictEqual(report.podman_status, "completed"); + assert.strictEqual(report.status_code, 0); + assert.strictEqual(report.stdout, "podman-ok:node-local source\n"); + assert.strictEqual(report.large_bytes_uploaded, false); + assert.strictEqual(report.uses_full_repo_tarball, false); + assert.strictEqual(report.coordinator_routed_file_reads, false); + assert.strictEqual(report.staged_artifact.path, "/vfs/artifacts/podman-smoke.txt"); + + console.log("Podman backend smoke passed"); +} catch (error) { + if (error.code === "CLUSTERFLUX_PODMAN_INCOMPLETE") { + console.error(error.message); + process.exit(2); + } + throw error; +} diff --git a/scripts/podman-test-env.js b/scripts/podman-test-env.js new file mode 100644 index 0000000..2c0dc3b --- /dev/null +++ b/scripts/podman-test-env.js @@ -0,0 +1,41 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +function configurePodmanTestEnvironment(repo) { + const originalHome = os.homedir(); + if ( + fs.existsSync(path.join(originalHome, ".config/containers/policy.json")) || + fs.existsSync("/etc/containers/policy.json") + ) { + return; + } + + const isolatedHome = path.join(repo, "scripts/containers-home"); + const policy = path.join(isolatedHome, ".config/containers/policy.json"); + fs.mkdirSync(path.dirname(policy), { recursive: true }); + if (!fs.existsSync(policy)) { + fs.writeFileSync( + policy, + `${JSON.stringify( + { default: [{ type: "insecureAcceptAnything" }] }, + null, + 2 + )}\n` + ); + } + + process.env.CARGO_HOME ||= path.join(originalHome, ".cargo"); + process.env.RUSTUP_HOME ||= path.join(originalHome, ".rustup"); + process.env.HOME = isolatedHome; + process.env.XDG_DATA_HOME = path.join( + os.tmpdir(), + `clusterflux-containers-data-${process.getuid?.() ?? "user"}` + ); + process.env.XDG_CACHE_HOME = path.join( + os.tmpdir(), + `clusterflux-containers-cache-${process.getuid?.() ?? "user"}` + ); +} + +module.exports = { configurePodmanTestEnvironment }; diff --git a/scripts/prepare-public-release.js b/scripts/prepare-public-release.js new file mode 100755 index 0000000..0094ca2 --- /dev/null +++ b/scripts/prepare-public-release.js @@ -0,0 +1,738 @@ +#!/usr/bin/env node + +const crypto = require("crypto"); +const cp = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const outputRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const publicTree = path.join(outputRoot, "public-tree"); +const assetsDir = path.join(outputRoot, "assets"); +const stagingDir = path.join(outputRoot, "staging"); +const publicBuildTarget = path.join(outputRoot, "cargo-target"); +const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com"; +const forgejoHost = "git.michelpaulissen.com"; +const args = new Set(process.argv.slice(2)); +const includeForgejoWorkflows = + args.has("--include-forgejo-workflows") || + /^(1|true|yes)$/i.test(process.env.CLUSTERFLUX_INCLUDE_FORGEJO_WORKFLOWS || ""); +const filteredTopLevel = ["private", "internal", "experiments", ".git", "target"]; +const filteredDirectoryNames = [".clusterflux"]; +const archiveIgnoredPathFallbacks = [ + "target", + ".clusterflux", + "vscode-extension/node_modules", + "scripts/containers-home", +]; +const publicRepoBranch = process.env.CLUSTERFLUX_PUBLIC_REPO_BRANCH || "main"; +const publicBinaries = [ + "clusterflux", + "clusterflux-coordinator", + "clusterflux-node", + "clusterflux-debug-dap", +]; + +function commandOutput(command, args, options = {}) { + try { + const execOptions = { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + ...options, + }; + execOptions.env = commandEnv(command, options.env); + return cp + .execFileSync(command, args, execOptions) + .trim(); + } catch (_) { + return null; + } +} + +function run(command, args, options = {}) { + const execOptions = { + cwd: repo, + stdio: "inherit", + ...options, + }; + execOptions.env = commandEnv(command, options.env); + cp.execFileSync(command, args, execOptions); +} + +function nonInteractiveGitEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function commandEnv(command, env) { + if (command !== "git") { + return env; + } + return nonInteractiveGitEnv(env); +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function filteredOutPatterns() { + return [ + "private/**", + "internal/**", + "experiments/**", + ".git", + "target", + "git-ignored source paths", + "root/*.md except README.md", + "**/.clusterflux/**", + ...(includeForgejoWorkflows ? [] : [".forgejo/**"]), + ]; +} + +function gitIgnoredSourcePaths() { + const output = commandOutput("git", [ + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + ]); + if (output === null) { + return archiveIgnoredPathFallbacks; + } + return [ + ...new Set([ + ...archiveIgnoredPathFallbacks, + ...output + .split("\0") + .filter(Boolean) + .map((relativePath) => + relativePath.replaceAll("\\", "/").replace(/\/$/, "") + ), + ]), + ]; +} + +function isGitIgnored(relativePath, ignoredSourcePaths) { + const normalized = relativePath.replaceAll(path.sep, "/"); + return ignoredSourcePaths.some( + (ignored) => normalized === ignored || normalized.startsWith(`${ignored}/`) + ); +} + +function isFilteredRootMarkdown(relativePath, entry) { + return ( + !relativePath.includes(path.sep) && + (entry.isFile() || entry.isSymbolicLink()) && + path.extname(entry.name).toLowerCase() === ".md" && + !["README.md", "SECURITY.md"].includes(entry.name) + ); +} + +function shouldFilter(entry, relativePath) { + const parts = relativePath.split(path.sep).filter(Boolean); + const topLevel = parts[0]; + if (filteredTopLevel.includes(topLevel)) return true; + if (parts.some((part) => filteredDirectoryNames.includes(part))) return true; + if (topLevel === ".forgejo" && !includeForgejoWorkflows) return true; + if (isFilteredRootMarkdown(relativePath, entry)) return true; + return false; +} + +function copyFilteredTree(src, dest, ignoredSourcePaths, relative = "") { + ensureDir(dest); + const entries = fs + .readdirSync(src, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)); + + for (const entry of entries) { + const childRelative = relative ? path.join(relative, entry.name) : entry.name; + if ( + shouldFilter(entry, childRelative) || + isGitIgnored(childRelative, ignoredSourcePaths) + ) { + continue; + } + + const from = path.join(src, entry.name); + const to = path.join(dest, entry.name); + if (entry.isDirectory()) { + copyFilteredTree(from, to, ignoredSourcePaths, childRelative); + } else if (entry.isSymbolicLink()) { + fs.symlinkSync(fs.readlinkSync(from), to); + } else if (entry.isFile()) { + fs.copyFileSync(from, to); + fs.chmodSync(to, fs.statSync(from).mode & 0o777); + } + } +} + +function assertFilteredTree(ignoredSourcePaths) { + for (const excluded of ["private", "internal", "experiments"]) { + if (fs.existsSync(path.join(publicTree, excluded))) { + throw new Error(`${excluded}/ leaked into the public tree`); + } + } + for (const file of walkFiles(publicTree)) { + if (file.split(path.sep).includes(".clusterflux")) { + throw new Error(`generated .clusterflux view state leaked into the public tree: ${file}`); + } + } + if (!includeForgejoWorkflows && fs.existsSync(path.join(publicTree, ".forgejo"))) { + throw new Error(".forgejo/ leaked into the host-neutral public tree"); + } + if (!fs.existsSync(path.join(publicTree, "README.md"))) { + throw new Error("product README.md is missing from the public tree"); + } + for (const ignored of ignoredSourcePaths) { + if (fs.existsSync(path.join(publicTree, ...ignored.split("/")))) { + throw new Error(`Git-ignored source path leaked into the public tree: ${ignored}`); + } + } + for (const entry of fs.readdirSync(publicTree, { withFileTypes: true })) { + if (isFilteredRootMarkdown(entry.name, entry)) { + throw new Error(`internal root Markdown file leaked into the public tree: ${entry.name}`); + } + } +} + +function walkFiles(root, relative = "") { + const dir = path.join(root, relative); + const entries = fs + .readdirSync(dir, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)); + const files = []; + for (const entry of entries) { + const childRelative = relative ? path.join(relative, entry.name) : entry.name; + if (entry.isDirectory()) { + files.push(...walkFiles(root, childRelative)); + } else if (entry.isFile()) { + files.push(childRelative); + } else if (entry.isSymbolicLink()) { + files.push(childRelative); + } + } + return files; +} + +function hashTree(root) { + const hash = crypto.createHash("sha256"); + for (const file of walkFiles(root)) { + const absolute = path.join(root, file); + const stat = fs.lstatSync(absolute); + hash.update(file.replaceAll(path.sep, "/")); + hash.update("\0"); + hash.update(String(stat.mode & 0o777)); + hash.update("\0"); + if (stat.isSymbolicLink()) { + hash.update("symlink"); + hash.update("\0"); + hash.update(fs.readlinkSync(absolute)); + } else { + hash.update(fs.readFileSync(absolute)); + } + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function sha256File(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function tarGz(output, cwd, inputs) { + run("tar", ["-czf", output, "-C", cwd, ...inputs]); +} + +function platformName() { + return `${os.platform()}-${os.arch()}`; +} + +function binaryName(name) { + return process.platform === "win32" ? `${name}.exe` : name; +} + +function buildPublicBinaries() { + run("cargo", ["build", "--locked", "--workspace", "--bins", "--release", "--jobs", "2"], { + cwd: publicTree, + env: { ...process.env, CARGO_TARGET_DIR: publicBuildTarget }, + }); +} + +function stageBinaryAssets(releaseName) { + const stageRoot = path.join(stagingDir, "binaries"); + const binDir = path.join(stageRoot, "bin"); + fs.rmSync(stageRoot, { recursive: true, force: true }); + ensureDir(binDir); + + for (const binary of publicBinaries) { + const fileName = binaryName(binary); + const built = path.join(publicBuildTarget, "release", fileName); + if (!fs.existsSync(built)) { + throw new Error(`expected release binary ${built}`); + } + const staged = path.join(binDir, fileName); + fs.copyFileSync(built, staged); + fs.chmodSync(staged, 0o755); + } + + const archive = path.join( + assetsDir, + `clusterflux-public-binaries-${releaseName}-${platformName()}.tar.gz` + ); + tarGz(archive, stageRoot, ["."]); + return archive; +} + +function publicBinaryDigests() { + return Object.fromEntries( + publicBinaries.map((binary) => { + const fileName = binaryName(binary); + const file = path.join(publicBuildTarget, "release", fileName); + if (!fs.existsSync(file)) throw new Error(`missing built release binary ${file}`); + return [fileName, `sha256:${sha256File(file)}`]; + }) + ); +} + +function stageEvidenceAsset( + releaseName, + sourceCommit, + publicTreeIdentity, + binaryDigests +) { + const stage = path.join(stagingDir, "evidence"); + fs.rmSync(stage, { recursive: true, force: true }); + ensureDir(stage); + const evidenceRoot = path.join(repo, "target/acceptance"); + const included = []; + for (const name of ["public-environment.json", "wasmtime-assignment.json"]) { + const source = path.join(evidenceRoot, name); + if (!fs.existsSync(source)) continue; + fs.copyFileSync(source, path.join(stage, name)); + included.push(name); + } + const binding = { + kind: "clusterflux-release-evidence-binding", + source_commit: sourceCommit, + public_tree_identity: publicTreeIdentity, + binary_digests: binaryDigests, + configuration_generation: + process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, + included_evidence: included, + }; + fs.writeFileSync( + path.join(stage, "EVIDENCE_BINDING.json"), + `${JSON.stringify(binding, null, 2)}\n` + ); + const archive = path.join( + assetsDir, + `clusterflux-public-evidence-${releaseName}.tar.gz` + ); + tarGz(archive, stage, ["."]); + return archive; +} + +function stageSourceAsset(releaseName) { + const archive = path.join(assetsDir, `clusterflux-public-source-${releaseName}.tar.gz`); + tarGz(archive, publicTree, ["."]); + return archive; +} + +function stageExtensionAsset() { + const packageJson = JSON.parse( + fs.readFileSync(path.join(publicTree, "vscode-extension/package.json"), "utf8") + ); + const archive = path.join( + assetsDir, + `${packageJson.name}-${packageJson.version}.vsix` + ); + run( + "npx", + [ + "--yes", + "@vscode/vsce", + "package", + "--allow-missing-repository", + "--skip-license", + "--out", + archive, + ], + { cwd: path.join(publicTree, "vscode-extension") } + ); + return archive; +} + +function resolverInstructions() { + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS) { + return process.env.CLUSTERFLUX_PUBLIC_RELEASE_RESOLVER_INSTRUCTIONS.trim(); + } + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY) { + return [ + "Add the controlled hosts entry supplied for this release:", + "", + "```", + process.env.CLUSTERFLUX_PUBLIC_RELEASE_HOSTS_ENTRY.trim(), + "```", + ].join("\n"); + } + if (process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP) { + return [ + "Add this controlled hosts entry for the release:", + "", + "```", + `${process.env.CLUSTERFLUX_PUBLIC_RELEASE_IP} clusterflux.michelpaulissen.com`, + "```", + ].join("\n"); + } + return [ + "`clusterflux.michelpaulissen.com` should resolve through public DNS. If it", + "does not resolve yet, wait for DNS propagation or use the fallback hosts", + "entry supplied with the invitation.", + ].join("\n"); +} + +function writeGettingStartedAsset(releaseName, publicTreeIdentity, resolution) { + const file = path.join(assetsDir, `CLUSTERFLUX_GETTING_STARTED-${releaseName}.md`); + fs.writeFileSync( + file, + `# Clusterflux Getting Started + +Use the public repository and release downloads with the hosted coordinator at +\`https://clusterflux.michelpaulissen.com\`. + +## DNS + +${resolution} + +## Install + +1. Download the binary archive for your platform from the Forgejo release. +2. Check the archive against \`SHA256SUMS\`. +3. Extract it and put \`bin/\` on your \`PATH\`. +4. Install \`clusterflux-vscode-*.vsix\` when you want the VS Code debugger: + +\`\`\`bash +code --install-extension clusterflux-vscode-*.vsix +\`\`\` + +## Sign in and run + +\`\`\`bash +clusterflux login --browser +clusterflux auth status +clusterflux project list +clusterflux bundle inspect --project examples/launch-build-demo +\`\`\` + +The CLI opens the server-provided Authentik authorization URL. State, nonce, +PKCE, provider code exchange, and userinfo remain on the hosted service. The CLI +stores only the resulting scoped Clusterflux session. + +Create a node enrollment grant, attach the node, and start the worker: + +\`\`\`bash +clusterflux node enroll --project-id --json +clusterflux node attach --project-id --node workstation \\ + --enrollment-grant "$ENROLLMENT_GRANT" +clusterflux-node --coordinator https://clusterflux.michelpaulissen.com \\ + --tenant --project-id --node workstation \\ + --project-root "$PWD" --worker --emit-ready +\`\`\` + +Then run the workflow: + +\`\`\`bash +clusterflux run --project examples/launch-build-demo build +\`\`\` + +Public tree identity: \`${publicTreeIdentity}\` +Release name: \`${releaseName}\` +`, + "utf8" + ); + return file; +} + +function writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution) { + const file = path.join(assetsDir, `CLUSTERFLUX_RELEASE_NOTES-${releaseName}.md`); + const publicRepo = + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + "https://git.michelpaulissen.com/michel/clusterflux-public"; + const releaseUrl = + process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || + "the Forgejo release attached to the public repository"; + fs.writeFileSync( + file, + `# Clusterflux Release Notes + +Use this release with the public Clusterflux repository and hosted coordinator. + +## Links + +- Public repository: ${publicRepo} +- Release downloads: ${releaseUrl} +- Hosted coordinator: ${defaultHostedCoordinatorEndpoint} +- Public tree identity: ${publicTreeIdentity} +- Release name: ${releaseName} + +## DNS + +${resolution} + +## First run + +1. Download the archive for your platform and \`SHA256SUMS\`. +2. Extract the archive and put \`bin/\` on your \`PATH\`. +3. Optionally install the VS Code extension archive. +4. Run \`clusterflux login --browser\`. +5. Run \`clusterflux node enroll\`, attach your node, and start the worker. +6. Run \`clusterflux run --project examples/launch-build-demo build\`. +`, + "utf8" + ); + return file; +} + +function writeSha256Sums(assets) { + const sumsPath = path.join(assetsDir, "SHA256SUMS"); + const lines = assets + .map((asset) => `${sha256File(asset)} ${path.basename(asset)}`) + .sort() + .join("\n"); + fs.writeFileSync(sumsPath, `${lines}\n`); + return sumsPath; +} + +function boolEnv(name) { + return /^(1|true|yes)$/i.test(process.env[name] || ""); +} + +function writePublicTreeProvenance(sourceCommit, releaseName) { + const provenance = { + kind: "clusterflux-filtered-public-tree", + source_commit: sourceCommit, + release_name: releaseName, + filtered_out: filteredOutPatterns(), + public_export: { + host_neutral: !includeForgejoWorkflows, + include_forgejo_workflows: includeForgejoWorkflows, + }, + forgejo_host: forgejoHost, + default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, + }; + fs.writeFileSync( + path.join(publicTree, "CLUSTERFLUX_PUBLIC_TREE.json"), + `${JSON.stringify(provenance, null, 2)}\n` + ); +} + +function publishPublicTree(releaseName, sourceCommit, publicTreeIdentity) { + const remote = + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + null; + const enabled = boolEnv("CLUSTERFLUX_PUBLISH_PUBLIC_TREE"); + const result = { + enabled, + remote, + branch: publicRepoBranch, + commit: null, + pushed: false, + }; + + if (!enabled) { + return result; + } + if (!remote) { + throw new Error( + "CLUSTERFLUX_PUBLISH_PUBLIC_TREE requires CLUSTERFLUX_PUBLIC_REPO_REMOTE or CLUSTERFLUX_PUBLIC_REPO_URL" + ); + } + if (!remote.includes(forgejoHost)) { + throw new Error(`public repo remote must point at ${forgejoHost}: ${remote}`); + } + + run("git", ["init"], { cwd: publicTree }); + run("git", ["checkout", "-B", publicRepoBranch], { cwd: publicTree }); + run("git", ["config", "user.name", "Clusterflux release"], { + cwd: publicTree, + }); + run("git", ["config", "user.email", "release-dryrun@clusterflux.invalid"], { + cwd: publicTree, + }); + run("git", ["add", "."], { cwd: publicTree }); + run( + "git", + [ + "commit", + "-m", + `Public release ${releaseName}`, + "-m", + `Source commit: ${sourceCommit}`, + "-m", + `Public tree identity: ${publicTreeIdentity}`, + ], + { cwd: publicTree } + ); + result.commit = commandOutput("git", ["rev-parse", "HEAD"], { cwd: publicTree }); + run("git", ["remote", "add", "public", remote], { cwd: publicTree }); + const pushArgs = ["push", "public", `HEAD:${publicRepoBranch}`]; + if (boolEnv("CLUSTERFLUX_PUBLIC_REPO_PUSH_FORCE_WITH_LEASE")) { + run( + "git", + [ + "fetch", + "public", + `refs/heads/${publicRepoBranch}:refs/remotes/public/${publicRepoBranch}`, + ], + { cwd: publicTree } + ); + pushArgs.splice(1, 0, "--force-with-lease"); + } + run("git", pushArgs, { cwd: publicTree }); + result.pushed = true; + return result; +} + +function main() { + const sourceCommit = commandOutput("git", ["rev-parse", "HEAD"]); + if (!sourceCommit || !/^[0-9a-f]{40}$/u.test(sourceCommit)) { + throw new Error("the public release requires an exact Git HEAD object id"); + } + const acceptanceCommit = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT?.trim(); + if (acceptanceCommit && acceptanceCommit !== sourceCommit) { + throw new Error( + `CLUSTERFLUX_ACCEPTANCE_COMMIT ${acceptanceCommit} does not match source HEAD ${sourceCommit}` + ); + } + const shortCommit = sourceCommit.slice(0, 12); + const releaseName = process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || `release-${shortCommit}`; + const sourceStatus = commandOutput("git", ["status", "--short"]); + if (sourceStatus === null) { + throw new Error("the public release must be prepared from a Git checkout"); + } + if (sourceStatus !== "") { + throw new Error("the public release must be prepared from a clean source tree"); + } + const sourceTreeClean = true; + + fs.rmSync(outputRoot, { recursive: true, force: true }); + ensureDir(publicTree); + ensureDir(assetsDir); + ensureDir(stagingDir); + + const ignoredSourcePaths = gitIgnoredSourcePaths(); + copyFilteredTree(repo, publicTree, ignoredSourcePaths); + assertFilteredTree(ignoredSourcePaths); + writePublicTreeProvenance(sourceCommit, releaseName); + const publicTreeIdentity = hashTree(publicTree); + const sourceArchive = stageSourceAsset(releaseName); + + run("scripts/check-old-name.sh", [], { cwd: publicTree }); + run("node", ["scripts/check-docs.js"], { cwd: publicTree }); + run("scripts/check-code-size.sh", [], { cwd: publicTree }); + const publicTreePublish = publishPublicTree( + releaseName, + sourceCommit, + publicTreeIdentity + ); + buildPublicBinaries(); + const binaryArchive = stageBinaryAssets(releaseName); + const binaryDigests = publicBinaryDigests(); + const evidenceArchive = stageEvidenceAsset( + releaseName, + sourceCommit, + publicTreeIdentity, + binaryDigests + ); + const extensionArchive = stageExtensionAsset(); + const resolution = resolverInstructions(); + const gettingStarted = writeGettingStartedAsset( + releaseName, + publicTreeIdentity, + resolution + ); + const invite = writeReleaseNotesAsset(releaseName, publicTreeIdentity, resolution); + const assets = [ + sourceArchive, + binaryArchive, + evidenceArchive, + extensionArchive, + gettingStarted, + invite, + ]; + const sha256Sums = writeSha256Sums(assets); + + const manifest = { + kind: "clusterflux-public-release", + release_name: releaseName, + source_commit: sourceCommit, + source_tree_clean: sourceTreeClean, + public_tree_identity: publicTreeIdentity, + public_tree: publicTree, + filtered_out: filteredOutPatterns(), + public_export: { + host_neutral: !includeForgejoWorkflows, + include_forgejo_workflows: includeForgejoWorkflows, + }, + forgejo_host: forgejoHost, + public_repo_url: process.env.CLUSTERFLUX_PUBLIC_REPO_URL || publicTreePublish.remote, + public_repo_remote: process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || null, + public_tree_publish: publicTreePublish, + forgejo_release_url: process.env.CLUSTERFLUX_FORGEJO_RELEASE_URL || null, + default_hosted_coordinator_endpoint: defaultHostedCoordinatorEndpoint, + dns_publication_state: + process.env.CLUSTERFLUX_DNS_PUBLICATION_STATE || "published", + resolver_override: + process.env.CLUSTERFLUX_RESOLVER_OVERRIDE || "none-required-public-dns", + platform: platformName(), + binary_digests: binaryDigests, + configuration_generation: + process.env.CLUSTERFLUX_DEPLOYMENT_SYSTEM_GENERATION || null, + tool_versions: { + node: process.version, + rustc: commandOutput("rustc", ["--version"]) || null, + cargo: commandOutput("cargo", ["--version"]) || null, + tar: commandOutput("tar", ["--version"]) || null, + }, + commands: [ + "scripts/check-old-name.sh", + "node scripts/check-docs.js", + "scripts/check-code-size.sh", + ...(publicTreePublish.enabled + ? [`git push public HEAD:${publicRepoBranch}`] + : []), + "cargo build --locked --workspace --bins --release --jobs 2", + ], + assets: [...assets, sha256Sums].map((asset) => ({ + file: asset, + name: path.basename(asset), + sha256: sha256File(asset), + })), + notes: [ + "Upload the assets to the Forgejo Release for the filtered public repository.", + "The real service deployment and full e2e release remain separate acceptance evidence.", + ], + }; + + const manifestPath = path.join(outputRoot, "public-release-manifest.json"); + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + console.log(JSON.stringify({ manifest: manifestPath, assets: assetsDir }, null, 2)); +} + +main(); diff --git a/scripts/public-local-demo-matrix-smoke.js b/scripts/public-local-demo-matrix-smoke.js new file mode 100755 index 0000000..2aac30d --- /dev/null +++ b/scripts/public-local-demo-matrix-smoke.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing public local demo evidence: ${name}`); +} + +const publicAcceptance = read("scripts/acceptance-public.sh"); +const publicSplit = read("scripts/verify-public-split.sh"); +const cliInstall = read("scripts/cli-install-smoke.js"); +const flagship = read("scripts/flagship-demo-smoke.js"); +const cliLocalRun = read("scripts/cli-local-run-smoke.js"); +const nodeAttach = read("scripts/node-attach-smoke.js"); +const vscodeExtension = read("scripts/vscode-extension-smoke.js"); +const vscodeF5 = read("scripts/vscode-f5-smoke.js"); +const artifactDownload = read("scripts/artifact-download-smoke.js"); +const artifactExport = read("scripts/artifact-export-smoke.js"); + +for (const script of [publicAcceptance, publicSplit]) { + for (const smoke of [ + "scripts/cli-install-smoke.js", + "scripts/flagship-demo-smoke.js", + "scripts/cli-local-run-smoke.js", + "scripts/node-attach-smoke.js", + "scripts/vscode-extension-smoke.js", + "scripts/vscode-f5-smoke.js", + "scripts/artifact-download-smoke.js", + "scripts/artifact-export-smoke.js", + ]) { + assert(script.includes(`node ${smoke}`), `public demo gate must run ${smoke}`); + } +} + +expect(cliInstall, "CLI install from project path", /cargo[\s\S]*install[\s\S]*crates\/clusterflux-cli/); +expect(cliInstall, "CLI install smoke targets flagship project", /const project = path\.join\(repo, "examples\/launch-build-demo"\)/); +expect(cliInstall, "installed CLI inspects flagship project", /installedBin[\s\S]*\["bundle", "inspect", "--project", project, "--json"\]/); + +expect(flagship, "flagship project source is Rust workflow", /examples\/launch-build-demo[\s\S]*src\/build\.rs/); +expect(flagship, "flagship source avoids local machine assumptions", /forbiddenSourceAssumptions/); +expect(flagship, "flagship cargo test runs", /cargo[\s\S]*test[\s\S]*launch-build-demo/); + +expect(cliLocalRun, "local run starts node process", /cli_process_started_node_process[\s\S]*true/); +expect(cliLocalRun, "local run records real task events", /events\.events\.length >= 4[\s\S]*prepare_source[\s\S]*compile_linux[\s\S]*package_release/); +expect(cliLocalRun, "local run records artifact metadata", /events\.events\.some\(\(event\) => event\.artifact_path\)/); + +expect(nodeAttach, "Linux node attach creates enrollment grant", /create_node_enrollment_grant/); +expect(nodeAttach, "Linux node attach uses enrollment exchange", /used_enrollment_exchange[\s\S]*true/); +expect(nodeAttach, "attached Linux node proves signed enrollment", /used_enrollment_exchange[\s\S]*signedNodeHeartbeat/); + +expect(vscodeExtension, "extension contributes debugger", /contributes\.debuggers[\s\S]*type === "clusterflux"/); +expect(vscodeF5, "F5 uses local-services backend", /runtimeBackend[\s\S]*local-services/); +expect(vscodeF5, "F5 exposes real coordinator main thread", /threads\.find\(\(thread\) => thread\.name\.includes\("build coordinator main"\)\)[\s\S]*real coordinator-main entrypoint thread/); +expect(vscodeF5, "F5 does not fabricate a terminal event at the entry probe", /coordinator_task_events[\s\S]*value === 0[\s\S]*must not fabricate a terminal task event/); + +expect(artifactDownload, "artifact download creates scoped link", /create_artifact_download_link/); +expect(artifactDownload, "artifact download opens stream", /open_artifact_download_stream/); +expect(artifactExport, "artifact export targets receiver node", /export_artifact_to_node[\s\S]*node-export-receiver/); +expect(artifactExport, "artifact export rejects coordinator bulk relay", /coordinator_bulk_relay_allowed[\s\S]*false/); + +console.log("Public local demo matrix smoke passed"); diff --git a/scripts/public-release-preflight.js b/scripts/public-release-preflight.js new file mode 100755 index 0000000..b6c00af --- /dev/null +++ b/scripts/public-release-preflight.js @@ -0,0 +1,294 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const acceptanceRoot = path.join(repo, "target/acceptance"); +const manifestPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || + path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_PREFLIGHT_REPORT || + path.join(acceptanceRoot, "public-release-preflight.json"); + +function commandOutput(command, args, options = {}) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + ...options, + }) + .trim(); + } catch (_) { + return null; + } +} + +function nonInteractiveEnv(extra = {}) { + return { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: process.env.GIT_ASKPASS || "/bin/false", + SSH_ASKPASS: process.env.SSH_ASKPASS || "/bin/false", + GIT_SSH_COMMAND: + process.env.GIT_SSH_COMMAND || + "ssh -o BatchMode=yes -o NumberOfPasswordPrompts=0", + ...extra, + }; +} + +function expectedSourceCommit() { + const head = commandOutput("git", ["rev-parse", "HEAD"]); + assert(head && /^[0-9a-f]{40}$/u.test(head), "preflight requires an exact Git HEAD"); + const asserted = process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT; + if (asserted) { + assert.strictEqual(asserted, head, "acceptance commit must match Git HEAD"); + } + return head; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function sha256File(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function parseSha256Sums(file) { + const sums = new Map(); + for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) { + if (!line.trim()) continue; + const match = /^([0-9a-f]{64})\s+(.+)$/.exec(line.trim()); + assert(match, `malformed SHA256SUMS line: ${line}`); + sums.set(path.basename(match[2]), match[1]); + } + return sums; +} + +function remoteHead(remote) { + const output = commandOutput("git", ["ls-remote", remote, "HEAD", "refs/heads/main"], { + env: nonInteractiveEnv(), + timeout: Number(process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE_TIMEOUT_MS || 30000), + }); + if (!output) return null; + const lines = output.split(/\r?\n/).filter(Boolean); + const head = lines.find((line) => line.endsWith("\tHEAD")) || lines[0]; + return head && head.split(/\s+/)[0]; +} + +function publicTreeAlreadyPushed(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || + process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" + ); +} + +function publicTreePushSource(manifest) { + return manifest.public_tree_publish && manifest.public_tree_publish.pushed === true + ? "manifest" + : "external-env"; +} + +function publicRepoRemoteForManifest(manifest) { + return ( + manifest.public_repo_url || + manifest.public_repo_remote || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE || + process.env.CLUSTERFLUX_PUBLIC_REPO_URL || + null + ); +} + +function publicTreeCommitForManifest(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.commit) || + process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || + null + ); +} + +function envState(name) { + return process.env[name] ? "set" : "unset"; +} + +function staleEvidence(file, currentSourceCommit) { + if (!fs.existsSync(file)) { + return { file, status: "missing", source_commit: null, release_name: null }; + } + const evidence = readJson(file); + if (!evidence.source_commit) { + return { + file, + status: "unversioned", + source_commit: null, + release_name: evidence.release_name || null, + }; + } + return { + file, + status: evidence.source_commit === currentSourceCommit ? "current" : "stale", + source_commit: evidence.source_commit, + release_name: evidence.release_name || null, + }; +} + +assert(fs.existsSync(manifestPath), `missing public release manifest: ${manifestPath}`); +const manifest = readJson(manifestPath); +const currentSourceCommit = expectedSourceCommit(); +const currentTreeStatus = commandOutput("git", ["status", "--short"]) || ""; +assert.strictEqual( + currentTreeStatus, + "", + "public release preflight requires a clean source tree" +); +assert.strictEqual(manifest.kind, "clusterflux-public-release"); +assert.strictEqual( + manifest.source_commit, + currentSourceCommit, + "public release manifest must be regenerated for the current acceptance commit" +); +assert.strictEqual(manifest.source_tree_clean, true, "public release prep must start clean"); +assert.strictEqual( + publicTreeAlreadyPushed(manifest), + true, + "filtered public tree must be pushed to Forgejo before release publication" +); + +const publicRepoRemote = publicRepoRemoteForManifest(manifest); +assert(publicRepoRemote, "manifest must record public repository URL or remote"); +const publicTreeCommit = publicTreeCommitForManifest(manifest); +const remoteMain = remoteHead(publicRepoRemote); +assert(remoteMain, "Forgejo public repository main branch must be readable"); +if (publicTreeCommit) { + assert.strictEqual( + remoteMain, + publicTreeCommit, + "Forgejo public repository main branch must match the prepared public tree commit" + ); +} + +assert(Array.isArray(manifest.assets) && manifest.assets.length > 0, "manifest assets missing"); +const checksumAsset = manifest.assets.find((asset) => asset.name === "SHA256SUMS"); +assert(checksumAsset, "manifest must include SHA256SUMS"); +assert(fs.existsSync(checksumAsset.file), `missing checksum asset: ${checksumAsset.file}`); +const checksums = parseSha256Sums(checksumAsset.file); +const assets = manifest.assets.map((asset) => { + assert(fs.existsSync(asset.file), `missing release asset: ${asset.file}`); + const actual = sha256File(asset.file); + const expected = checksums.get(asset.name); + if (asset.name !== "SHA256SUMS") { + assert.strictEqual(actual, expected, `checksum mismatch for ${asset.name}`); + } + return { + name: asset.name, + file: asset.file, + bytes: fs.statSync(asset.file).size, + sha256: actual, + }; +}); + +const evidence = [ + staleEvidence( + path.join(acceptanceRoot, "public-release-forgejo-release.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-service.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "hosted-client-compat.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "core-coordinator-compat.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-e2e.json"), + currentSourceCommit + ), + staleEvidence( + path.join(acceptanceRoot, "public-release-final.json"), + currentSourceCommit + ), +]; + +const report = { + kind: "clusterflux-public-release-preflight", + source_commit: currentSourceCommit, + release_name: manifest.release_name, + public_tree_commit: publicTreeCommit || remoteMain, + public_tree_push_source: publicTreePushSource(manifest), + public_repo_url: publicRepoRemote, + public_repo_remote_head: remoteMain, + source_tree_clean: currentTreeStatus === "", + local_assets_ready: true, + assets, + evidence, + external_gates: { + forgejo_release_publication: { + status: envState("CLUSTERFLUX_FORGEJO_TOKEN") === "set" ? "ready" : "pending", + env: { + CLUSTERFLUX_FORGEJO_TOKEN: envState("CLUSTERFLUX_FORGEJO_TOKEN"), + }, + }, + live_service_smoke: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR") === "set" && + envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR" + ), + CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" + ), + }, + }, + public_release_e2e: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_E2E") === "set" && + process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E === "1" && + envState("CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND") === "set" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_E2E: + process.env.CLUSTERFLUX_PUBLIC_RELEASE_E2E || "unset", + CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND: envState( + "CLUSTERFLUX_PUBLIC_RELEASE_BROWSER_OPEN_COMMAND" + ), + }, + }, + final_evidence: { + status: + envState("CLUSTERFLUX_PUBLIC_RELEASE_FINAL") === "set" && + process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL === "1" + ? "ready" + : "pending", + env: { + CLUSTERFLUX_PUBLIC_RELEASE_FINAL: + process.env.CLUSTERFLUX_PUBLIC_RELEASE_FINAL || "unset", + }, + }, + }, +}; + +fs.mkdirSync(path.dirname(reportPath), { recursive: true }); +fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); +console.log(JSON.stringify({ report: reportPath, release_name: report.release_name }, null, 2)); diff --git a/scripts/publish-public-release.js b/scripts/publish-public-release.js new file mode 100755 index 0000000..3622737 --- /dev/null +++ b/scripts/publish-public-release.js @@ -0,0 +1,349 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const https = require("https"); +const path = require("path"); +const cp = require("child_process"); + +const repo = path.resolve(__dirname, ".."); +const releaseRoot = path.resolve( + process.env.CLUSTERFLUX_PUBLIC_RELEASE_DIR || + path.join(repo, "target/public-release") +); +const manifestPath = path.join(releaseRoot, "public-release-manifest.json"); +const reportPath = path.join( + repo, + "target/acceptance/public-release-forgejo-release.json" +); +const forgejoUrl = ( + process.env.CLUSTERFLUX_FORGEJO_URL || "https://git.michelpaulissen.com" +).replace(/\/+$/, ""); +const token = process.env.CLUSTERFLUX_FORGEJO_TOKEN; +let owner = process.env.CLUSTERFLUX_PUBLIC_REPO_OWNER; +let repoName = process.env.CLUSTERFLUX_PUBLIC_REPO_NAME; + +function requireEnv(name, value) { + if (!value) { + throw new Error(`${name} is required`); + } +} + +function commandOutput(command, args) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }) + .trim(); + } catch (_) { + return null; + } +} + +function expectedSourceCommit() { + return ( + process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || + commandOutput("git", ["rev-parse", "HEAD"]) || + "unknown" + ); +} + +function parseForgejoRepoIdentity(remote) { + if (!remote) { + return null; + } + + let pathname = remote; + try { + pathname = new URL(remote).pathname; + } catch (_) { + const scpLike = /^[^@/]+@[^:]+:(.+)$/.exec(remote); + if (scpLike) { + pathname = scpLike[1]; + } + } + + const parts = pathname + .replace(/^\/+/, "") + .replace(/\.git$/, "") + .split("/") + .filter(Boolean); + if (parts.length < 2) { + return null; + } + return { + owner: parts[parts.length - 2], + repoName: parts[parts.length - 1], + }; +} + +function resolveRepoIdentity(manifest) { + if (owner && repoName) { + return; + } + + const inferred = parseForgejoRepoIdentity( + manifest.public_repo_url || + manifest.public_repo_remote || + process.env.CLUSTERFLUX_PUBLIC_REPO_REMOTE + ); + owner = owner || (inferred && inferred.owner); + repoName = repoName || (inferred && inferred.repoName); + if (!owner || !repoName) { + throw new Error( + "CLUSTERFLUX_PUBLIC_REPO_OWNER and CLUSTERFLUX_PUBLIC_REPO_NAME are required when the manifest does not contain a parseable Forgejo repository URL" + ); + } +} + +function publicTreeAlreadyPushed(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.pushed === true) || + process.env.CLUSTERFLUX_PUBLIC_TREE_ALREADY_PUSHED === "1" + ); +} + +function publicTreeCommit(manifest) { + return ( + (manifest.public_tree_publish && manifest.public_tree_publish.commit) || + process.env.CLUSTERFLUX_PUBLIC_TREE_COMMIT || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_TARGET || + null + ); +} + +function apiPath(pathname) { + return `/api/v1${pathname}`; +} + +function request(method, pathname, { body, headers = {} } = {}) { + const url = new URL(apiPath(pathname), forgejoUrl); + const payload = + body === undefined + ? null + : Buffer.isBuffer(body) + ? body + : Buffer.from(JSON.stringify(body)); + const requestHeaders = { + Accept: "application/json", + Authorization: `token ${token}`, + ...headers, + }; + if (payload) { + requestHeaders["Content-Length"] = payload.length; + if (!requestHeaders["Content-Type"]) { + requestHeaders["Content-Type"] = "application/json"; + } + } + + return new Promise((resolve, reject) => { + const req = https.request( + url, + { + method, + headers: requestHeaders, + }, + (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + let parsed = null; + if (text.trim()) { + try { + parsed = JSON.parse(text); + } catch (_) { + parsed = text; + } + } + if (res.statusCode < 200 || res.statusCode >= 300) { + reject( + new Error( + `${method} ${url.pathname} failed with ${res.statusCode}: ${text}` + ) + ); + return; + } + resolve({ status: res.statusCode, body: parsed }); + }); + } + ); + req.on("error", reject); + if (payload) req.write(payload); + req.end(); + }); +} + +function multipartFile(fieldName, file) { + const boundary = `clusterflux-${Date.now()}-${Math.random().toString(16).slice(2)}`; + const name = path.basename(file); + const header = Buffer.from( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="${fieldName}"; filename="${name}"\r\n` + + "Content-Type: application/octet-stream\r\n\r\n" + ); + const footer = Buffer.from(`\r\n--${boundary}--\r\n`); + return { + body: Buffer.concat([header, fs.readFileSync(file), footer]), + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + +async function existingRelease(tagName) { + const releases = await request( + "GET", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases?limit=100` + ); + return (releases.body || []).find((release) => release.tag_name === tagName) || null; +} + +async function createOrReuseRelease(manifest) { + const tagName = manifest.release_name; + const found = await existingRelease(tagName); + if (found) { + return { release: found, created: false }; + } + const targetCommitish = + publicTreeCommit(manifest) || + "main"; + const body = [ + "Clusterflux public release.", + "", + `Default hosted coordinator endpoint: ${manifest.default_hosted_coordinator_endpoint}`, + `DNS publication state: ${manifest.dns_publication_state}`, + `Resolver override: ${manifest.resolver_override}`, + `Public tree identity: ${manifest.public_tree_identity}`, + `Source commit: ${manifest.source_commit}`, + ].join("\n"); + const created = await request( + "POST", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases`, + { + body: { + tag_name: tagName, + target_commitish: targetCommitish, + name: tagName, + body, + draft: false, + prerelease: false, + }, + } + ); + return { release: created.body, created: true }; +} + +async function loadRelease(releaseId) { + const response = await request( + "GET", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${releaseId}` + ); + return response.body; +} + +async function uploadAsset(release, asset) { + const { body, contentType } = multipartFile("attachment", asset.file); + const response = await request( + "POST", + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/releases/${release.id}/assets?name=${encodeURIComponent(asset.name)}`, + { + body, + headers: { + "Content-Type": contentType, + }, + } + ); + return response.body; +} + +function existingAssetByName(release, name) { + return Array.isArray(release.assets) + ? release.assets.find((asset) => asset.name === name) || null + : null; +} + +async function main() { + requireEnv("CLUSTERFLUX_FORGEJO_TOKEN", token); + if (!fs.existsSync(manifestPath)) { + throw new Error(`missing public release manifest: ${manifestPath}`); + } + + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + resolveRepoIdentity(manifest); + if (manifest.kind !== "clusterflux-public-release") { + throw new Error(`unexpected public release manifest kind: ${manifest.kind}`); + } + if (manifest.source_commit !== expectedSourceCommit()) { + throw new Error( + "public release manifest is stale; regenerate it for the current acceptance commit before publishing the Forgejo Release" + ); + } + if ( + !publicTreeAlreadyPushed(manifest) + ) { + throw new Error( + "public tree must be pushed before publishing the Forgejo Release; run prepare-public-release.js with CLUSTERFLUX_PUBLISH_PUBLIC_TREE=1" + ); + } + if (!Array.isArray(manifest.assets) || manifest.assets.length === 0) { + throw new Error("public release manifest has no assets"); + } + + const { release: releaseResult, created } = await createOrReuseRelease(manifest); + const release = await loadRelease(releaseResult.id); + const uploaded = []; + const reused = []; + for (const asset of manifest.assets) { + if (!fs.existsSync(asset.file)) { + throw new Error(`missing release asset: ${asset.file}`); + } + const existing = existingAssetByName(release, asset.name); + if (existing) { + reused.push(existing); + continue; + } + uploaded.push(await uploadAsset(release, asset)); + } + + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + const report = { + kind: "clusterflux-public-release-forgejo-release", + forgejo_url: forgejoUrl, + owner, + repo: repoName, + release_id: release.id, + release_name: release.name || manifest.release_name, + tag_name: release.tag_name || manifest.release_name, + release_created: created, + default_hosted_coordinator_endpoint: manifest.default_hosted_coordinator_endpoint, + public_tree_identity: manifest.public_tree_identity, + public_tree_commit: publicTreeCommit(manifest), + source_commit: manifest.source_commit, + uploaded_assets: uploaded.map((asset) => ({ + id: asset.id, + name: asset.name, + size: asset.size, + browser_download_url: asset.browser_download_url, + })), + reused_assets: reused.map((asset) => ({ + id: asset.id, + name: asset.name, + size: asset.size, + browser_download_url: asset.browser_download_url, + })), + }; + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log( + JSON.stringify( + { report: reportPath, uploaded: uploaded.length, reused: reused.length }, + null, + 2 + ) + ); +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/quic-smoke.js b/scripts/quic-smoke.js new file mode 100755 index 0000000..5ad4634 --- /dev/null +++ b/scripts/quic-smoke.js @@ -0,0 +1,154 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function rendezvousRequest(overrides = {}) { + return { + type: "request_rendezvous", + scope: { + tenant: "tenant", + project: "project", + process: "vp-quic", + object: { Artifact: "quic-artifact" }, + authorization_subject: "node-a-to-node-b", + }, + source: { + node: "node-a", + advertised_addr: "node-a.mesh.invalid:4433", + public_key_fingerprint: "sha256:node-a-public-key", + }, + destination: { + node: "node-b", + advertised_addr: "node-b.mesh.invalid:4433", + public_key_fingerprint: "sha256:node-b-public-key", + }, + direct_connectivity: true, + failure_reason: "", + ...overrides, + }; +} + +(async () => { + const output = cp.execFileSync( + "cargo", + ["run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-quic-smoke"], + { cwd: repo, encoding: "utf8" } + ); + const report = JSON.parse(output.trim().split("\n").at(-1)); + + assert.strictEqual(report.kind, "clusterflux_quic_smoke"); + assert.strictEqual(report.transport, "NativeQuic"); + assert.strictEqual(report.rust_native_quic, true); + assert.strictEqual(report.authenticated_direct_connection, true); + assert.strictEqual(report.coordinator_assisted_rendezvous, true); + assert.strictEqual(report.coordinator_bulk_relay_allowed, false); + assert.strictEqual(report.source_node, "node-a"); + assert.strictEqual(report.destination_node, "node-b"); + assert.strictEqual(report.scope.tenant, "tenant"); + assert.strictEqual(report.scope.project, "project"); + assert.strictEqual(report.scope.process, "vp-quic"); + assert.deepStrictEqual(report.scope.object, { Artifact: "quic-artifact" }); + assert.ok(report.authorization_digest.startsWith("sha256:")); + assert.ok(report.request_bytes > 0); + assert.strictEqual(report.server_received_request_bytes, report.request_bytes); + assert.ok(report.payload_bytes > 0); + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback", + ], + { cwd: repo } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + + const plan = await send(addr, rendezvousRequest()); + assert.strictEqual(plan.type, "rendezvous_plan"); + assert.strictEqual(plan.charged_rendezvous_attempts, 1); + assert.strictEqual(plan.plan.transport, "NativeQuic"); + assert.strictEqual(plan.plan.scope.tenant, "tenant"); + assert.strictEqual(plan.plan.scope.project, "project"); + assert.strictEqual(plan.plan.source.node, "node-a"); + assert.strictEqual(plan.plan.destination.node, "node-b"); + assert.strictEqual(plan.plan.coordinator_assisted_rendezvous, true); + assert.strictEqual(plan.plan.coordinator_bulk_relay_allowed, false); + assert.ok(plan.plan.authorization_digest.startsWith("sha256:")); + + const failed = await send( + addr, + rendezvousRequest({ + direct_connectivity: false, + failure_reason: "nat traversal failed", + }) + ); + assert.strictEqual(failed.type, "error"); + assert.match(failed.message, /nat traversal failed/); + assert.match(failed.message, /coordinator bulk relay is disabled/); + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("QUIC smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/real-flagship-harness.js b/scripts/real-flagship-harness.js new file mode 100644 index 0000000..dcbee77 --- /dev/null +++ b/scripts/real-flagship-harness.js @@ -0,0 +1,283 @@ +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { configurePodmanTestEnvironment } = require("./podman-test-env"); + +const repo = path.resolve(__dirname, ".."); +const project = path.join(repo, "examples/launch-build-demo"); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.stderr?.on("data", (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 4096) stderr = stderr.slice(-4096); + }); + child.once("exit", (code) => { + const detail = stderr.trim(); + reject(new Error( + `process exited before JSON line with code ${code}${detail ? `: ${detail}` : ""}` + )); + }); + }); +} + +function waitForNodeStatus(child, expectedStatus) { + return new Promise((resolve, reject) => { + let buffer = ""; + let stderr = ""; + const cleanup = () => { + child.stdout.off("data", onData); + child.stderr?.off("data", onStderr); + child.off("exit", onExit); + }; + const onData = (chunk) => { + buffer += chunk.toString(); + while (buffer.includes("\n")) { + const newline = buffer.indexOf("\n"); + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + let event; + try { + event = JSON.parse(line); + } catch (error) { + cleanup(); + reject(error); + return; + } + if (event.node_status === expectedStatus) { + cleanup(); + resolve(event); + return; + } + } + }; + const onStderr = (chunk) => { + stderr += chunk.toString(); + if (stderr.length > 4096) stderr = stderr.slice(-4096); + }; + const onExit = (code) => { + cleanup(); + const detail = stderr.trim(); + reject( + new Error( + `worker exited before node status ${expectedStatus} with code ${code}${ + detail ? `: ${detail}` : "" + }` + ) + ); + }; + child.stdout.on("data", onData); + child.stderr?.on("data", onStderr); + child.once("exit", onExit); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function configureContainerPolicyHome() { + configurePodmanTestEnvironment(repo); +} + +function commandWithPodman(program, args) { + if (cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status === 0) { + return { program, args }; + } + if (cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0) { + return { + program: "nix", + args: ["shell", "nixpkgs#podman", "--command", program, ...args], + }; + } + throw new Error( + "real flagship smoke requires rootless Podman (or Nix to provide it)" + ); +} + +function ensureRootlessPodman() { + configureContainerPolicyHome(); + const invocation = commandWithPodman("podman", [ + "info", + "--format", + "{{.Host.Security.Rootless}}", + ]); + const rootless = cp.execFileSync(invocation.program, invocation.args, { + cwd: repo, + env: process.env, + encoding: "utf8", + }).trim(); + assert.strictEqual(rootless, "true", "flagship worker must use rootless Podman"); +} + +async function runFlagshipWorker(addr, node, identity) { + const enrollment = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "user", + ttl_seconds: 60, + }); + assert.strictEqual(enrollment.type, "node_enrollment_grant_created"); + const cargoArgs = [ + "run", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node", "--", + "--coordinator", `${addr.host}:${addr.port}`, + "--tenant", "tenant", + "--project-id", "project", + "--node", node, + "--enrollment-grant", enrollment.grant, + "--worker", + "--emit-ready", + "--project-root", project, + "--assignment-poll-ms", "25", + ]; + const invocation = commandWithPodman("cargo", cargoArgs); + const child = cp.spawn(invocation.program, invocation.args, { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey, + }, + }); + return { child, ready: waitForJsonLine(child) }; +} + +const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForTaskEvent(addr, process, predicate, description) { + for (let attempt = 0; attempt < 2400; attempt += 1) { + const response = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process, + }); + assert.strictEqual(response.type, "task_events", JSON.stringify(response)); + const event = response.events.find(predicate); + if (event) return event; + await delay(25); + } + throw new Error(`timed out waiting for real Wasm task ${description}`); +} + +function startFlagship(addr) { + const report = JSON.parse( + cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "run", "build", + "--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`, + "--project", project, + "--json", + ], + { cwd: repo, env: process.env, encoding: "utf8" } + ) + ); + assert.strictEqual(report.command, "run"); + assert.strictEqual(report.status, "main_launched", JSON.stringify(report)); + assert.strictEqual(report.entry, "build"); + assert.strictEqual(report.task_launch.type, "main_launched"); + assert.strictEqual(report.task_launch.task_instance, report.task_instance); + assert.strictEqual(report.task_launch.task_definition, report.task_definition); + assert.strictEqual(report.worker_placement_requested, true); + assert.match(report.bundle_digest, /^sha256:[0-9a-f]{64}$/); + assert.match(report.entry_export, /^clusterflux_entry_v1_/); + const virtualProcess = report.process; + return { report, process: virtualProcess }; +} + +async function launchFlagship(addr) { + const { report, process: virtualProcess } = startFlagship(addr); + const compileEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task_definition === "compile_linux", + "compile_linux" + ); + const packageEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task_definition === "package_release", + "package_release" + ); + const buildEvent = await waitForTaskEvent( + addr, + virtualProcess, + (event) => event.task === report.task_instance && event.executor === "coordinator_main", + "coordinator build main" + ); + for (const event of [compileEvent, packageEvent, buildEvent]) { + assert.strictEqual(event.terminal_state, "completed", JSON.stringify(event)); + } + return { + report, + process: virtualProcess, + compileEvent, + packageEvent, + buildEvent, + }; +} + +function flagshipNodeCapabilities() { + return { + os: "Linux", + arch: process.arch, + capabilities: [ + "Command", + "Containers", + "RootlessPodman", + "SourceFilesystem", + "VfsArtifacts", + ], + environment_backends: ["Container"], + source_providers: ["filesystem"], + }; +} + +module.exports = { + ensureRootlessPodman, + flagshipNodeCapabilities, + launchFlagship, + project, + repo, + runFlagshipWorker, + send, + startFlagship, + waitForTaskEvent, + waitForJsonLine, + waitForNodeStatus, +}; diff --git a/scripts/release-source-scan.sh b/scripts/release-source-scan.sh new file mode 100755 index 0000000..d8bf013 --- /dev/null +++ b/scripts/release-source-scan.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +release_paths=( + Cargo.toml + Cargo.lock + README.md + crates + examples + private + scripts + vscode-extension +) + +existing_release_paths=() +for path in "${release_paths[@]}"; do + if [[ -e "$path" ]]; then + existing_release_paths+=("$path") + fi +done + +prose_scan_paths=( + README.md + crates + examples + private + scripts + vscode-extension +) + +existing_prose_scan_paths=() +for path in "${prose_scan_paths[@]}"; do + if [[ -e "$path" ]]; then + existing_prose_scan_paths+=("$path") + fi +done + +scan_globs=( + --glob '!**/target/**' + --glob '!**/node_modules/**' + --glob '!scripts/release-source-scan.sh' + --glob '!scripts/rename-to-clusterflux.sh' + --glob '!scripts/check-docs.js' +) + +placeholder_pattern='debugger-gate|experiments/debugger-gate|CLUSTERFLUX-DEMO|device-code-placeholder|artifact://demo|vp-local-demo' +if rg -n "${scan_globs[@]}" "$placeholder_pattern" "${existing_release_paths[@]}"; then + echo "release source scan failed: stale experiment/demo placeholder reference found" >&2 + exit 1 +fi + +demo_setup_pattern='undocumented manual state|hidden setup|demo-only credentials?|hard-coded local paths?' +if rg -n "${scan_globs[@]}" "$demo_setup_pattern" "${existing_prose_scan_paths[@]}"; then + echo "release source scan failed: demo requires hidden setup or demo-only state" >&2 + exit 1 +fi + +hidden_local_pattern='file://|/home/[[:alnum:]_.-]+/|/Users/[[:alnum:]_.-]+/|C:\\Users\\|https?://(localhost|127\.0\.0\.1)[^[:space:]]*/artifacts/' +if rg -n "${scan_globs[@]}" "$hidden_local_pattern" "${existing_release_paths[@]}"; then + echo "release source scan failed: hidden local path or local artifact URL found" >&2 + exit 1 +fi + +public_wording_pattern='reddit|hacker news|lobsters|launch forum|traffic source|free tier' +if rg -n "${scan_globs[@]}" "$public_wording_pattern" "${existing_prose_scan_paths[@]}"; then + echo "release source scan failed: public-facing launch-forum/free-tier wording found" >&2 + exit 1 +fi + +echo "Release source scan passed" diff --git a/scripts/resource-metering-contract-smoke.js b/scripts/resource-metering-contract-smoke.js new file mode 100644 index 0000000..62e593d --- /dev/null +++ b/scripts/resource-metering-contract-smoke.js @@ -0,0 +1,230 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing resource metering evidence: ${name}`); +} + +const coreLimits = read("crates/clusterflux-core/src/limits.rs"); +// Phase 3 keeps the protocol dispatch in service.rs and the metered operation +// implementations in focused service modules. Read the complete relevant +// boundary so this contract follows the refactor instead of one mega-file. +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/processes.rs"), + read("crates/clusterflux-coordinator/src/service/process_launch.rs"), + read("crates/clusterflux-coordinator/src/service/artifacts.rs"), +].join("\n"); +const coordinatorQuota = read("crates/clusterflux-coordinator/src/service/quota.rs"); +const coordinatorLogs = read("crates/clusterflux-coordinator/src/service/logs.rs"); +const coordinatorDebug = read("crates/clusterflux-coordinator/src/service/debug.rs"); +const coordinatorTests = read("crates/clusterflux-coordinator/src/service/tests.rs"); +const wasmRuntime = read("crates/clusterflux-wasm-runtime/src/lib.rs"); +const wasmRuntimeTests = read("crates/clusterflux-wasm-runtime/src/tests.rs"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const quicSmoke = read("scripts/quic-smoke.js"); + +for (const [name, pattern] of [ + ["API call limit kind", /\bApiCall,/], + ["spawn limit kind", /\bSpawn,/], + ["log bytes limit kind", /\bLogBytes,/], + ["metadata bytes limit kind", /\bMetadataBytes,/], + ["debug read bytes limit kind", /\bDebugReadBytes,/], + ["UI event limit kind", /\bUiEvent,/], + ["rendezvous attempt limit kind", /\bRendezvousAttempt,/], + ["artifact download bytes limit kind", /\bArtifactDownloadBytes,/], + [ + "preflight can check without consuming", + /pub fn can_charge\([\s\S]*used\.saturating_add\(amount\) > limit/, + ], + [ + "charge goes through preflight", + /pub fn charge\([\s\S]*self\.can_charge\(limits, kind, amount\)\?/, + ], +]) { + expect(coreLimits, name, pattern); +} + +for (const [name, pattern] of [ + [ + "Wasm stores enforce a concrete memory limit", + /StoreLimitsBuilder::new\(\)[\s\S]*memory_size\(runtime\.memory_bytes\)/, + ], + [ + "Wasm compute uses a refillable fuel token bucket", + /struct FuelTokenBucket[\s\S]*fractional_fuel_numerator[\s\S]*fn refill_after/, + ], + ["Wasmtime fuel consumption is enabled", /config\.consume_fuel\(true\)/], + ["Wasmtime epoch interruption is enabled", /config\.epoch_interruption\(true\)/], +]) { + expect(wasmRuntime, name, pattern); +} + +for (const [name, pattern] of [ + [ + "fuel refill preserves fractional credit", + /frequent_refills_preserve_fractional_credit/, + ], + [ + "linear memory growth is bounded per store", + /wasm_linear_memory_growth_is_bounded_per_store/, + ], + [ + "CPU-bound Wasm is interrupted without a host call", + /epoch_interruption_aborts_cpu_bound_wasm_without_a_host_call/, + ], +]) { + expect(`${wasmRuntime}\n${wasmRuntimeTests}`, name, pattern); +} + +for (const [name, pattern] of [ + [ + "rendezvous charges before transport planning", + /handle_request_rendezvous[\s\S]*charge_rendezvous_attempt\([\s\S]*&scope\.tenant[\s\S]*&scope\.project[\s\S]*now_epoch_seconds[\s\S]*plan_authenticated_direct_bulk_transfer/, + ], + [ + "artifact link creation preflights downloadable bytes before link creation", + /handle_create_artifact_download_link[\s\S]*downloadable_size[\s\S]*can_charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*downloadable_size[\s\S]*create_download_link/, + ], + [ + "artifact delivery charges scoped bytes before advancing its offset", + /handle_open_artifact_download_stream[\s\S]*stream_download_chunk\([\s\S]*charge_download\([\s\S]*&context\.tenant[\s\S]*&context\.project[\s\S]*streamed_bytes[\s\S]*delivered_offset = end/, + ], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, pattern] of [ + [ + "quota keys include tenant/project resource kind and window", + /struct ProjectQuotaScope[\s\S]*tenant: TenantId[\s\S]*project: ProjectId[\s\S]*struct MeterKey[\s\S]*kind: LimitKind[\s\S]*window: u64/, + ], + [ + "quota module discards expired windows for an accessed scope and kind", + /fn meter_mut[\s\S]*self\.meters\.retain[\s\S]*existing\.scope != key\.scope[\s\S]*existing\.kind != kind[\s\S]*existing\.window == key\.window/, + ], + [ + "quota module charges rendezvous attempts through the scoped window meter", + /fn charge_rendezvous_attempt[\s\S]*self\.charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::RendezvousAttempt/, + ], + [ + "quota module charges authenticated API calls through the scoped window meter", + /fn charge_api_call[\s\S]*self\.charge\(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds\)/, + ], + [ + "quota module preflights and charges log bytes through the scoped window meter", + /fn can_charge_log_bytes[\s\S]*LimitKind::LogBytes[\s\S]*fn charge_log_bytes[\s\S]*LimitKind::LogBytes/, + ], + [ + "quota module preflights artifact download bytes through the scoped meter", + /fn can_charge_download[\s\S]*self\.can_charge\([\s\S]*tenant[\s\S]*project[\s\S]*LimitKind::ArtifactDownloadBytes/, + ], + [ + "quota status reports current scoped window usage", + /fn project_status[\s\S]*for kind in LimitKind::ALL[\s\S]*self\.used\(tenant, project, kind, now_epoch_seconds\)/, + ], +]) { + expect(coordinatorQuota, name, pattern); +} + +for (const [source, name, pattern] of [ + [ + coordinatorService, + "authenticated API calls are charged after session authorization and before dispatch", + /authenticate_cli_session[\s\S]*authorize_authenticated_user_operation[\s\S]*charge_api_call[\s\S]*match request/, + ], + [ + coordinatorLogs, + "signed node log ingestion preflights and charges bytes before accepting the report", + /handle_report_task_log[\s\S]*authorize_node_for_process_or_termination[\s\S]*can_charge_log_bytes[\s\S]*charge_log_bytes[\s\S]*TaskLogRecorded/, + ], + [ + coordinatorDebug, + "debug reads charge the scoped debug-read budget before audit state is recorded", + /record_debug_audit_event[\s\S]*charge_debug_read[\s\S]*DebugAuditEvent/, + ], + [ + coordinatorTests, + "tests prove API-call and log-byte quota enforcement and project isolation", + /authenticated_api_calls_are_metered_per_tenant_and_project_before_dispatch[\s\S]*project-a[\s\S]*project-b[\s\S]*signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes[\s\S]*LogBytes/, + ], +]) { + expect(source, name, pattern); +} + +for (const [name, source, patterns] of [ + [ + "rendezvous smoke", + quicSmoke, + [/charged_rendezvous_attempts, 1/, /coordinator bulk relay is disabled/], + ], + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /downloaded\.response\.charged_download_bytes,[\s\S]*packageEvent\.artifact_size_bytes/, + /retaining_node_reverse_stream/, + /revoked/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [ + /type: "submit_panel_event"/, + /max_events: 1/, + /used_events, 1/, + /rate limit/i, + /max_download_bytes: 1/, + /exceeds download limit/, + ], + ], +]) { + for (const pattern of patterns) { + expect(source, name, pattern); + } +} + +const privateHostedLibSource = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); +const privateHostedTests = maybeRead(["private", "hosted-policy", "src", "tests.rs"]); +const privateHostedLib = privateHostedLibSource + ? [privateHostedLibSource, privateHostedTests].filter(Boolean).join("\n") + : null; +if (privateHostedLib) { + for (const [name, pattern] of [ + [ + "private hosted configuration owns exact control-plane limits and quota windows", + /community_tier_resource_limits[\s\S]*LimitKind::ApiCall[\s\S]*LimitKind::ArtifactDownloadBytes[\s\S]*community_tier_quota_configuration[\s\S]*CoordinatorQuotaConfiguration::new/, + ], + [ + "hosted zero-capability policy rejects native execution capabilities", + /hosted_zero_capability_wasm_rejects_filesystem_and_command_capabilities[\s\S]*Capability::Command[\s\S]*Capability::Network[\s\S]*Capability::ArbitrarySyscalls/, + ], + ]) { + expect(privateHostedLib, name, pattern); + } + assert.doesNotMatch( + privateHostedLib, + /HostedFuel|HostedMemoryBytes|HostedWallClockMs|HostedStateBytes/, + "decorative hosted Wasm quota kinds must not return", + ); +} + +console.log("Resource metering contract smoke passed"); diff --git a/scripts/scheduler-placement-smoke.js b/scripts/scheduler-placement-smoke.js new file mode 100755 index 0000000..ffe672f --- /dev/null +++ b/scripts/scheduler-placement-smoke.js @@ -0,0 +1,397 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const digest = (value) => + `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +const environmentDigest = digest("scheduler-linux-container"); +const dependencyDigest = digest("scheduler-toolchain-dependencies"); +const sourceDigest = digest("scheduler-source-tree"); + +function buildFlagshipBundle() { + const output = cp.execFileSync( + "cargo", + [ + "run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", + "build", "--project", "examples/launch-build-demo", "--json", + ], + { cwd: repo, encoding: "utf8" } + ); + const report = JSON.parse(output); + const directory = path.resolve(repo, report.bundle_artifact.directory); + const manifest = JSON.parse(fs.readFileSync(path.join(directory, "manifest.json"), "utf8")); + const entrypoints = JSON.parse( + fs.readFileSync(path.join(directory, manifest.entrypoints), "utf8") + ); + const entrypoint = entrypoints.find((candidate) => candidate.name === "build"); + assert(entrypoint, "flagship bundle omitted build entrypoint"); + const taskDescriptors = JSON.parse( + fs.readFileSync(path.join(directory, manifest.task_descriptors), "utf8") + ); + const prepareSource = taskDescriptors.find( + (candidate) => candidate.name === "prepare_source" + ); + assert(prepareSource, "flagship bundle omitted prepare_source task"); + return { + digest: manifest.bundle_digest, + taskExport: prepareSource.export, + wasmModuleBase64: fs.readFileSync(path.join(directory, "module.wasm")).toString("base64"), + }; +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function linuxCapabilities() { + return { + os: "Linux", + arch: "x86_64", + capabilities: [ + "Command", + "Containers", + "RootlessPodman", + "SourceFilesystem", + "VfsArtifacts" + ], + environment_backends: ["Container"], + source_providers: ["filesystem"] + }; +} + +function gitCapabilities() { + const capabilities = linuxCapabilities(); + capabilities.capabilities = [...capabilities.capabilities, "SourceGit"].sort(); + capabilities.source_providers = [...capabilities.source_providers, "git"].sort(); + return capabilities; +} + +async function attachNode(addr, node) { + const identity = nodeIdentity("scheduler-placement-smoke", node); + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node, + public_key: identity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + assert.strictEqual(attached.node, node); + return identity; +} + +async function reportNode(addr, node, identity, locality) { + const recorded = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node, + capabilities: locality.capabilities || linuxCapabilities(), + cached_environment_digests: locality.cached_environment_digests, + dependency_cache_digests: locality.dependency_cache_digests, + source_snapshots: locality.source_snapshots, + artifact_locations: locality.artifact_locations, + direct_connectivity: locality.direct_connectivity !== false, + online: true + })); + assert.strictEqual( + recorded.type, + "node_capabilities_recorded", + JSON.stringify(recorded) + ); + assert.strictEqual(recorded.node, node); + return recorded; +} + +(async () => { + const bundle = buildFlagshipBundle(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const coldNode = await attachNode(addr, "cold-node"); + const warmNode = await attachNode(addr, "warm-node"); + + const cold = await reportNode(addr, "cold-node", coldNode, { + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + assert.strictEqual(cold.node_descriptors, 1); + + const warm = await reportNode(addr, "warm-node", warmNode, { + cached_environment_digests: [environmentDigest], + dependency_cache_digests: [dependencyDigest], + source_snapshots: [sourceDigest], + artifact_locations: ["toolchain-cache"] + }); + assert.strictEqual(warm.node_descriptors, 2); + const reportedNodes = new Set([cold.node, warm.node]); + assert.strictEqual(reportedNodes.size, 2); + assert(reportedNodes.has("cold-node")); + assert(reportedNodes.has("warm-node")); + + const inspected = await send(addr, { + type: "list_node_descriptors", + tenant: "tenant", + project: "project", + actor_user: "operator" + }); + assert.strictEqual(inspected.type, "node_descriptors"); + assert.strictEqual(inspected.actor, "operator"); + assert.strictEqual(inspected.descriptors.length, 2); + const warmDescriptor = inspected.descriptors.find( + (descriptor) => descriptor.id === "warm-node" + ); + assert(warmDescriptor, "warm node descriptor must be visible to inspector state"); + assert(warmDescriptor.capabilities.capabilities.includes("Command")); + assert(warmDescriptor.capabilities.capabilities.includes("RootlessPodman")); + assert(warmDescriptor.cached_environments.includes(environmentDigest)); + assert(warmDescriptor.dependency_caches.includes(dependencyDigest)); + assert(warmDescriptor.source_snapshots.includes(sourceDigest)); + assert(warmDescriptor.artifact_locations.includes("toolchain-cache")); + + const crossScopeInspection = await send(addr, { + type: "list_node_descriptors", + tenant: "other-tenant", + project: "project", + actor_user: "operator" + }); + assert.strictEqual(crossScopeInspection.type, "node_descriptors"); + assert.strictEqual(crossScopeInspection.descriptors.length, 0); + + const crossTenantReport = await send(addr, signedNodeRequest("warm-node", warmNode, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "other-tenant", + project: "project", + node: "warm-node", + capabilities: linuxCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(crossTenantReport.type, "error"); + assert.match(crossTenantReport.message, /tenant\/project scope/); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Linux", + arch: null, + capabilities: ["Containers", "RootlessPodman"] + }, + environment_digest: environmentDigest, + required_capabilities: ["Command"], + dependency_cache: dependencyDigest, + source_snapshot: sourceDigest, + required_artifacts: ["toolchain-cache"], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "warm-node"); + assert.ok(placement.placement.score > 0); + assert.ok(placement.placement.reasons.includes("warm environment cache")); + assert.ok(placement.placement.reasons.includes("warm dependency cache")); + assert.ok(placement.placement.reasons.includes("source snapshot already local")); + assert.ok( + placement.placement.reasons.includes("1 required artifact(s) already local") + ); + + const impossible = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["WindowsCommandDev"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(impossible.type, "error"); + assert.match(impossible.message, /WindowsCommandDev/); + + const started = await send(addr, { + type: "start_process", + tenant: "tenant", + project: "project", + actor_user: "operator", + process: "vp-wait-for-git" + }); + assert.strictEqual(started.type, "process_started"); + + const queued = await send(addr, { + type: "launch_task", + tenant: "tenant", + project: "project", + actor_user: "operator", + task_spec: { + tenant: "tenant", + project: "project", + process: "vp-wait-for-git", + task_definition: "prepare_source", + task_instance: "prepare_source-1", + dispatch: { + kind: "coordinator_node_wasm", + export: bundle.taskExport, + abi: "task_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: ["SourceFilesystem", "SourceGit"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: started.epoch, + bundle_digest: bundle.digest, + }, + wait_for_node: true, + artifact_path: "/vfs/artifacts/git-status.txt", + wasm_module_base64: bundle.wasmModuleBase64, + }); + assert.strictEqual(queued.type, "error"); + assert.match(queued.message, /external callers may launch only EntrypointV1/); + + const gitNode = await attachNode(addr, "git-node"); + const gitRecorded = await reportNode(addr, "git-node", gitNode, { + capabilities: gitCapabilities(), + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + assert.strictEqual(gitRecorded.type, "node_capabilities_recorded"); + + const pendingAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { + type: "poll_task_assignment", + tenant: "tenant", + project: "project", + node: "git-node" + })); + assert.strictEqual(pendingAssignment.type, "task_assignment"); + assert.strictEqual(pendingAssignment.assignment, null); + + const emptyAssignment = await send(addr, signedNodeRequest("git-node", gitNode, "poll_task_assignment", { + type: "poll_task_assignment", + tenant: "tenant", + project: "project", + node: "git-node" + })); + assert.strictEqual(emptyAssignment.type, "task_assignment"); + assert.strictEqual(emptyAssignment.assignment, null); + + await reportNode(addr, "warm-node", warmNode, { + cached_environment_digests: [], + dependency_cache_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: false + }); + const disconnectedTransfer = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["Command"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: ["toolchain-cache"], + prefer_node: null + }); + assert.strictEqual(disconnectedTransfer.type, "error"); + assert.match(disconnectedTransfer.message, /source snapshot unavailable/); + assert.match(disconnectedTransfer.message, /required artifact\(s\) unavailable/); + assert.match(disconnectedTransfer.message, /direct connectivity unavailable/); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Scheduler placement smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/sdk-spawn-runtime-smoke.js b/scripts/sdk-spawn-runtime-smoke.js new file mode 100755 index 0000000..48fa978 --- /dev/null +++ b/scripts/sdk-spawn-runtime-smoke.js @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const sdk = fs.readFileSync(path.join(repo, "crates/clusterflux-sdk/src/lib.rs"), "utf8"); +const productRuntime = fs.readFileSync( + path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), + "utf8" +); + +assert.match(sdk, /pub struct RuntimeSpawnEvent/); +assert.match(sdk, /pub debugger_visible: bool/); +assert.match(sdk, /fn register_runtime_thread/); +assert.match(sdk, /register_runtime_thread\(id, self\.name, self\.env\)/); +assert.match(sdk, /pub fn debugger_visible\(&self\) -> bool/); +assert.match(sdk, /ProductRuntimeConfig::from_env\(\)/); +assert.match(sdk, /start_remote_task\(\s*config,\s*task_id/); +assert.match(sdk, /start_guest_host_task/); +assert.match(sdk, /join_remote_task\(remote\)/); +assert.doesNotMatch(productRuntime, /"type": "launch_task"/); +assert.match(productRuntime, /native SDK task spawning requires coordinator EntrypointV1 execution/); +assert.match(productRuntime, /task_start_v1/); +assert.match(productRuntime, /task_join_v1/); +assert.match(productRuntime, /command_run_v1/); +assert.match(productRuntime, /remote_completion_observed/); +assert.match(productRuntime, /TaskJoinState::Pending/); +assert.doesNotMatch( + productRuntime, + /entry\s*\(/, + "product runtime must not invoke the submitted local Rust closure" +); + +cp.execFileSync( + "cargo", + [ + "test", + "-p", + "clusterflux-sdk", + "spawn_task_start_registers_debugger_visible_runtime_thread", + ], + { cwd: repo, stdio: "inherit" } +); + +console.log("SDK spawn runtime smoke passed"); diff --git a/scripts/self-hosted-coordinator-smoke.js b/scripts/self-hosted-coordinator-smoke.js new file mode 100644 index 0000000..ca35649 --- /dev/null +++ b/scripts/self-hosted-coordinator-smoke.js @@ -0,0 +1,662 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); + +const repo = path.resolve(__dirname, ".."); +const teamArtifactDigest = `sha256:${"a".repeat(64)}`; +const teamEnvironmentDigest = `sha256:${crypto + .createHash("sha256") + .update("env-team-linux") + .digest("hex")}`; +const teamDependencyCacheDigest = `sha256:${crypto + .createHash("sha256") + .update("cargo-cache") + .digest("hex")}`; +const teamSourceDigest = `sha256:${crypto + .createHash("sha256") + .update("source-team") + .digest("hex")}`; +const coordinatorTaskModule = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); +const coordinatorTaskBundleDigest = `sha256:${crypto + .createHash("sha256") + .update(coordinatorTaskModule) + .digest("hex")}`; +const adminToken = "self-hosted-smoke-admin-token"; +const clientSessionSecret = "self-hosted-smoke-client-session-secret"; +const releaseManifestPath = + process.env.CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST || + path.join(repo, "target/public-release/public-release-manifest.json"); + +function sha256(value) { + return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +} + +function digestFromParts(parts) { + const hash = crypto.createHash("sha256"); + for (const value of parts) { + const bytes = Buffer.from(String(value)); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(bytes.length)); + hash.update(length); + hash.update(bytes); + } + return `sha256:${hash.digest("hex")}`; +} + +function adminRequest(token, operation, tenant, actorUser, targetTenant, nonce) { + const issuedAtEpochSeconds = Math.floor(Date.now() / 1000); + return { + type: operation, + tenant, + actor_user: actorUser, + ...(operation === "suspend_tenant" ? { target_tenant: targetTenant } : {}), + admin_proof: digestFromParts([ + "clusterflux-admin-request-proof:v1", + sha256(token), + operation, + tenant, + actorUser, + targetTenant, + nonce, + issuedAtEpochSeconds, + ]), + admin_nonce: nonce, + issued_at_epoch_seconds: issuedAtEpochSeconds, + }; +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function authenticated(request, sessionSecret = clientSessionSecret) { + return { + type: "authenticated", + session_secret: sessionSecret, + request, + }; +} + +function linuxNodeCapabilities() { + return { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "RootlessPodman", "VfsArtifacts", "QuicDirect"], + environment_backends: ["Container"], + source_providers: ["filesystem", "git"] + }; +} + +function nodeIdentity(node) { + void node; + const { privateKey: privateKeyObject, publicKey } = + crypto.generateKeyPairSync("ed25519"); + const publicDer = publicKey.export({ + format: "der", + type: "spki", + }); + return { + publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`, + privateKeyObject, + }; +} + +function signedRequestPayloadDigest(request) { + const canonicalize = (value, topLevel = true) => { + if (Array.isArray(value)) return value.map((entry) => canonicalize(entry, false)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter( + ([key, entry]) => + entry !== null && + (!topLevel || !["agent_signature", "node_signature"].includes(key)) + ) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalize(entry, false)]) + ); + } + return value; + }; + return `sha256:${crypto + .createHash("sha256") + .update(JSON.stringify(canonicalize(request))) + .digest("hex")}`; +} + +function nodeSignatureMessage( + node, + requestKind, + payloadDigest, + nonce, + issuedAtEpochSeconds +) { + const parts = [ + "clusterflux-node-request-signature:v2", + node, + requestKind, + payloadDigest, + nonce, + String(issuedAtEpochSeconds), + ]; + return Buffer.concat( + parts.flatMap((part) => [ + Buffer.from(`${Buffer.byteLength(part)}:`), + Buffer.from(part), + Buffer.from("\n"), + ]) + ); +} + +function signedNodeHeartbeat(node, identity) { + const request = { type: "node_heartbeat", node }; + const nonce = `self-hosted-heartbeat-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + "node_heartbeat", + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function signedNodeRequest(node, identity, requestKind, request) { + return { + type: "signed_node", + node, + node_signature: signedNodeHeartbeatForKind(node, identity, requestKind, request), + request, + }; +} + +function signedNodeHeartbeatForKind(node, identity, requestKind, request) { + const nonce = `${requestKind}-${process.pid}-${Date.now()}`; + const issuedAt = Math.floor(Date.now() / 1000); + const signature = crypto.sign( + null, + nodeSignatureMessage( + node, + requestKind, + signedRequestPayloadDigest(request), + nonce, + issuedAt + ), + identity.privateKeyObject + ); + return { + nonce, + issued_at_epoch_seconds: issuedAt, + signature: `ed25519:${signature.toString("base64")}`, + }; +} + +function commandOutput(command, args) { + try { + return cp + .execFileSync(command, args, { + cwd: repo, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }) + .trim(); + } catch (_) { + return null; + } +} + +function runJson(command, args, options = {}) { + return JSON.parse( + cp.execFileSync(command, args, { + cwd: options.cwd || repo, + encoding: "utf8", + input: options.input, + }) + ); +} + +function expectedSourceCommit() { + return ( + process.env.CLUSTERFLUX_ACCEPTANCE_COMMIT || + commandOutput("git", ["rev-parse", "HEAD"]) + ); +} + +function readReleaseManifest() { + if (!fs.existsSync(releaseManifestPath)) return null; + const manifest = JSON.parse(fs.readFileSync(releaseManifestPath, "utf8")); + if (manifest.kind !== "clusterflux-public-release") return null; + const expectedCommit = expectedSourceCommit(); + if (expectedCommit && manifest.source_commit !== expectedCommit) return null; + return manifest; +} + +function releaseIdentity() { + const manifest = readReleaseManifest(); + return { + sourceCommit: manifest ? manifest.source_commit : expectedSourceCommit(), + releaseName: + (manifest && manifest.release_name) || + process.env.CLUSTERFLUX_PUBLIC_RELEASE_NAME || + null, + }; +} + +async function attachTrustedNode(addr, node, capabilities = linuxNodeCapabilities()) { + const identity = nodeIdentity(node); + const grant = await send(addr, authenticated({ + type: "create_node_enrollment_grant", + ttl_seconds: 900, + })); + assert.strictEqual(grant.type, "node_enrollment_grant_created"); + + const attached = await send(addr, { + type: "exchange_node_enrollment_grant", + tenant: "team", + project: "self-hosted", + node, + public_key: identity.publicKey, + enrollment_grant: grant.grant, + }); + assert.strictEqual(attached.type, "node_enrollment_exchanged"); + + const heartbeat = await send(addr, { + type: "node_heartbeat", + node, + node_signature: signedNodeHeartbeat(node, identity) + }); + assert.strictEqual(heartbeat.type, "node_heartbeat"); + + const reported = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "team", + project: "self-hosted", + node, + capabilities, + cached_environment_digests: [teamEnvironmentDigest], + dependency_cache_digests: [teamDependencyCacheDigest], + source_snapshots: [teamSourceDigest], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(reported.type, "node_capabilities_recorded"); + return identity; +} + +(async () => { + const release = releaseIdentity(); + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_ADMIN_TOKEN: adminToken, + CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET: clientSessionSecret, + CLUSTERFLUX_SELF_HOSTED_TENANT: "team", + CLUSTERFLUX_SELF_HOSTED_PROJECT: "self-hosted", + CLUSTERFLUX_SELF_HOSTED_USER: "developer", + }, + } + ); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual(ready.client_authority, "strict"); + assert.strictEqual(ready.self_hosted_session_bootstrapped, true); + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const forgedBodyAuthority = await send(addr, { + type: "start_process", + tenant: "victim-tenant", + project: "victim-project", + actor_user: "forged-user", + process: "vp-forged", + }); + assert.strictEqual(forgedBodyAuthority.type, "error"); + assert.match(forgedBodyAuthority.message, /body.*identity.*not authority/i); + const wrongSession = await send( + addr, + authenticated({ type: "auth_status" }, "wrong-session-secret") + ); + assert.strictEqual(wrongSession.type, "error"); + assert.match(wrongSession.message, /session.*not recognized|not recognized.*session/i); + const authStatus = await send(addr, authenticated({ type: "auth_status" })); + assert.strictEqual(authStatus.type, "auth_status"); + assert.strictEqual(authStatus.tenant, "team"); + assert.strictEqual(authStatus.project, "self-hosted"); + assert.strictEqual(authStatus.actor, "developer"); + + const missingAdminCredential = await send( + addr, + adminRequest("", "admin_status", "team", "forged-admin", "team", "admin-empty") + ); + assert.strictEqual(missingAdminCredential.type, "error"); + assert.match(missingAdminCredential.message, /admin.*proof.*invalid/i); + const wrongAdminCredential = await send( + addr, + adminRequest( + "wrong-token", + "admin_status", + "team", + "forged-admin", + "team", + "admin-wrong" + ) + ); + assert.strictEqual(wrongAdminCredential.type, "error"); + assert.match(wrongAdminCredential.message, /admin.*proof.*invalid/i); + const replayableAdminRequest = adminRequest( + adminToken, + "admin_status", + "team", + "self-hosted-admin", + "team", + "admin-replay" + ); + const directAdminStatus = await send(addr, replayableAdminRequest); + assert.strictEqual(directAdminStatus.type, "admin_status"); + const replayedAdminStatus = await send(addr, replayableAdminRequest); + assert.strictEqual(replayedAdminStatus.type, "error"); + assert.match(replayedAdminStatus.message, /nonce was already used/i); + const cargoTargetDir = path.resolve( + process.env.CARGO_TARGET_DIR || path.join(repo, "target") + ); + const cliBin = path.join( + cargoTargetDir, + "debug", + process.platform === "win32" ? "clusterflux.exe" : "clusterflux" + ); + const selfHostedCliProject = path.join( + repo, + "target/acceptance/self-hosted-cli-project" + ); + fs.rmSync(selfHostedCliProject, { recursive: true, force: true }); + fs.mkdirSync(selfHostedCliProject, { recursive: true }); + const cliConnect = runJson( + cliBin, + [ + "auth", + "connect-self-hosted", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--project-id", + "self-hosted", + "--user", + "developer", + "--session-secret-stdin", + "--json", + ], + { cwd: selfHostedCliProject, input: `${clientSessionSecret}\n` } + ); + assert.strictEqual(cliConnect.status, "connected"); + assert.strictEqual(cliConnect.session_secret_read_from_stdin, true); + assert.strictEqual(cliConnect.session_secret_exposed_in_report, false); + assert.strictEqual(cliConnect.coordinator_response.type, "auth_status"); + const sessionPath = path.join( + selfHostedCliProject, + ".clusterflux/session.json" + ); + const storedSession = JSON.parse(fs.readFileSync(sessionPath, "utf8")); + assert.strictEqual(storedSession.kind, "self_hosted"); + assert.strictEqual(storedSession.session_secret, clientSessionSecret); + assert.strictEqual(storedSession.provider_tokens_exposed_to_cli, false); + assert.strictEqual(storedSession.provider_tokens_sent_to_nodes, false); + if (process.platform !== "win32") { + assert.strictEqual(fs.statSync(sessionPath).mode & 0o777, 0o600); + } + const cliAuthStatus = runJson( + cliBin, + ["auth", "status", "--json"], + { cwd: selfHostedCliProject } + ); + assert.strictEqual(cliAuthStatus.active_coordinator, ready.listen); + assert.strictEqual( + cliAuthStatus.coordinator_account_status.used_cli_session_credential, + true + ); + assert.strictEqual( + cliAuthStatus.coordinator_account_status.coordinator_response_type, + "auth_status" + ); + const cliAdminStatus = runJson(cliBin, [ + "admin", + "status", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--user", + "self-hosted-admin", + "--admin-token", + adminToken, + "--json", + ]); + assert.strictEqual(cliAdminStatus.response.type, "admin_status"); + assert.strictEqual(cliAdminStatus.suspended, false); + const cliAdminSuspend = runJson(cliBin, [ + "admin", + "suspend-tenant", + "--coordinator", + ready.listen, + "--tenant", + "team", + "--user", + "self-hosted-admin", + "--target-tenant", + "admin-probe-tenant", + "--admin-token", + adminToken, + "--yes", + "--json", + ]); + assert.strictEqual(cliAdminSuspend.response.type, "tenant_suspended"); + assert.strictEqual(cliAdminSuspend.suspended, true); + const cliAdminProbeStatus = runJson(cliBin, [ + "admin", + "status", + "--coordinator", + ready.listen, + "--tenant", + "admin-probe-tenant", + "--user", + "self-hosted-admin", + "--admin-token", + adminToken, + "--json", + ]); + assert.strictEqual(cliAdminProbeStatus.suspended, true); + + const teamLinuxA = await attachTrustedNode(addr, "team-linux-a"); + const teamLinuxBCapabilities = linuxNodeCapabilities(); + teamLinuxBCapabilities.capabilities = teamLinuxBCapabilities.capabilities.filter( + (capability) => capability !== "VfsArtifacts" + ); + await attachTrustedNode(addr, "team-linux-b", teamLinuxBCapabilities); + + const placement = await send(addr, authenticated({ + type: "schedule_task", + environment: { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "QuicDirect"] + }, + environment_digest: teamEnvironmentDigest, + required_capabilities: ["VfsArtifacts"], + source_snapshot: teamSourceDigest, + required_artifacts: [], + prefer_node: "team-linux-a" + })); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "team-linux-a"); + assert(placement.placement.reasons.includes("preferred node")); + assert(placement.placement.reasons.includes("warm environment cache")); + assert(placement.placement.reasons.includes("source snapshot already local")); + + const started = await send(addr, authenticated({ + type: "start_process", + process: "vp-team-build", + restart: false, + })); + assert.strictEqual(started.type, "process_started"); + + const reconnected = await send(addr, signedNodeRequest("team-linux-a", teamLinuxA, "reconnect_node", { + type: "reconnect_node", + node: "team-linux-a", + process: "vp-team-build", + epoch: started.epoch + })); + assert.strictEqual(reconnected.type, "node_reconnected"); + + const launched = await send(addr, authenticated({ + type: "launch_task", + task_spec: { + tenant: "team", + project: "self-hosted", + process: "vp-team-build", + task_definition: "compile-linux", + task_instance: "compile-linux", + dispatch: { + kind: "coordinator_node_wasm", + export: "compile_linux", + abi: "task_v1", + }, + environment_id: null, + environment: null, + environment_digest: null, + required_capabilities: ["VfsArtifacts"], + dependency_cache: null, + source_snapshot: null, + required_artifacts: [], + args: [], + vfs_epoch: started.epoch, + bundle_digest: coordinatorTaskBundleDigest, + }, + wait_for_node: false, + artifact_path: "/vfs/artifacts/team-output.txt", + wasm_module_base64: coordinatorTaskModule.toString("base64"), + })); + assert.strictEqual(launched.type, "error", JSON.stringify(launched)); + assert.match(launched.message, /external callers may launch only EntrypointV1/); + + const reportPath = path.join(repo, "target/acceptance/core-coordinator-compat.json"); + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + fs.writeFileSync( + reportPath, + `${JSON.stringify( + { + kind: "clusterflux-core-coordinator-compatibility", + source_commit: release.sourceCommit, + release_name: release.releaseName, + coordinator_implementation: "standalone-core-coordinator", + coordinator_addr: ready.listen, + client_authority: ready.client_authority, + authenticated_session: authStatus.authenticated, + forged_body_authority_denied: forgedBodyAuthority.type, + wrong_session_denied: wrongSession.type, + ping: "pong", + nodes: ["team-linux-a", "team-linux-b"], + task_placement: placement.type, + process_started: started.type, + external_task_v1_denied: launched.type, + self_hosted_admin: { + missing_credential_denied: missingAdminCredential.type, + wrong_credential_denied: wrongAdminCredential.type, + nonce_bound_proof_succeeded: directAdminStatus.type, + replay_denied: replayedAdminStatus.type, + cli_status: cliAdminStatus.response.type, + cli_suspend: cliAdminSuspend.response.type, + suspended_state_observed: cliAdminProbeStatus.suspended, + }, + self_hosted_cli: { + connected: cliConnect.status, + secret_read_from_stdin: cliConnect.session_secret_read_from_stdin, + secret_exposed_in_report: cliConnect.session_secret_exposed_in_report, + session_file_mode: + process.platform === "win32" + ? "windows-best-effort" + : (fs.statSync(sessionPath).mode & 0o777).toString(8), + authenticated_status: + cliAuthStatus.coordinator_account_status.coordinator_response_type, + }, + }, + null, + 2 + )}\n` + ); + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Self-hosted coordinator smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/source-preparation-smoke.js b/scripts/source-preparation-smoke.js new file mode 100755 index 0000000..9f9acd8 --- /dev/null +++ b/scripts/source-preparation-smoke.js @@ -0,0 +1,218 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function sourceCapableNode(sourceProviders = ["git"]) { + return { + os: "Linux", + arch: "x86_64", + capabilities: ["Command", "SourceFilesystem", "SourceGit"], + environment_backends: [], + source_providers: sourceProviders + }; +} + +(async () => { + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const pending = await send(addr, { + type: "request_source_preparation", + tenant: "tenant", + project: "project", + provider: "Git" + }); + assert.strictEqual(pending.type, "source_preparation"); + assert.strictEqual(pending.status.preparation.tenant, "tenant"); + assert.strictEqual(pending.status.preparation.project, "project"); + assert.strictEqual(pending.status.preparation.provider, "Git"); + assert.strictEqual( + pending.status.preparation.coordinator_requires_checkout_access, + false + ); + assert.deepStrictEqual(pending.status.preparation.required_capabilities, [ + "SourceGit" + ]); + assert.match(pending.status.disposition.Pending.reason, /waiting|node/i); + + const nodeIdentities = new Map(); + for (const node of ["source-cold", "source-ready"]) { + const identity = nodeIdentity("source-preparation-smoke", node); + nodeIdentities.set(node, identity); + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node, + public_key: identity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + } + + const cold = await send(addr, signedNodeRequest("source-cold", nodeIdentities.get("source-cold"), "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: "source-cold", + capabilities: sourceCapableNode(), + cached_environment_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(cold.type, "node_capabilities_recorded"); + + const readyReport = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: "source-ready", + capabilities: sourceCapableNode(), + cached_environment_digests: [], + source_snapshots: [], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(readyReport.type, "node_capabilities_recorded"); + + const assigned = await send(addr, { + type: "request_source_preparation", + tenant: "tenant", + project: "project", + provider: "Git" + }); + assert.strictEqual(assigned.type, "source_preparation"); + assert(["source-cold", "source-ready"].includes(assigned.status.disposition.Assigned.node)); + assert.strictEqual( + assigned.status.preparation.coordinator_requires_checkout_access, + false + ); + + const crossTenantCompletion = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { + type: "complete_source_preparation", + tenant: "other", + project: "project", + node: "source-ready", + provider: "Git", + source_snapshot: "sha256:source-prepared" + })); + assert.strictEqual(crossTenantCompletion.type, "error"); + assert.match(crossTenantCompletion.message, /tenant\/project scope/i); + + const completed = await send(addr, signedNodeRequest("source-ready", nodeIdentities.get("source-ready"), "complete_source_preparation", { + type: "complete_source_preparation", + tenant: "tenant", + project: "project", + node: "source-ready", + provider: "Git", + source_snapshot: "sha256:source-prepared" + })); + assert.strictEqual(completed.type, "source_preparation_completed"); + assert.strictEqual(completed.node, "source-ready"); + assert.strictEqual(completed.provider, "Git"); + assert.strictEqual(completed.source_snapshot, "sha256:source-prepared"); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: null, + environment_digest: null, + required_capabilities: ["SourceGit"], + source_snapshot: "sha256:source-prepared", + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "source-ready"); + assert( + placement.placement.reasons.includes("source snapshot already local"), + "completed source preparation must update node source locality" + ); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Source preparation smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/tenant-isolation-contract-smoke.js b/scripts/tenant-isolation-contract-smoke.js new file mode 100644 index 0000000..fb93e49 --- /dev/null +++ b/scripts/tenant-isolation-contract-smoke.js @@ -0,0 +1,181 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(relativePath) { + return fs.readFileSync(path.join(repo, relativePath), "utf8"); +} + +function maybeRead(segments) { + const fullPath = path.join(repo, ...segments); + if (!fs.existsSync(fullPath)) return null; + return fs.readFileSync(fullPath, "utf8"); +} + +function expect(source, name, pattern) { + assert.match(source, pattern, `missing tenant-isolation evidence: ${name}`); +} + +const auth = read("crates/clusterflux-core/src/auth.rs"); +const artifact = read("crates/clusterflux-core/src/artifact.rs"); +const operatorPanel = read("crates/clusterflux-core/src/operator_panel.rs"); +const source = read("crates/clusterflux-core/src/source.rs"); +const coordinatorService = [ + read("crates/clusterflux-coordinator/src/service.rs"), + read("crates/clusterflux-coordinator/src/service/routing.rs"), + read("crates/clusterflux-coordinator/src/service/nodes.rs"), + read("crates/clusterflux-coordinator/src/service/keys.rs"), + read("crates/clusterflux-coordinator/src/service/artifacts.rs"), + read("crates/clusterflux-coordinator/src/service/processes.rs"), + read("crates/clusterflux-coordinator/src/service/process_launch.rs"), + read("crates/clusterflux-coordinator/src/service/tests.rs"), +].join("\n"); +const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js"); +const operatorPanelSmoke = read("scripts/operator-panel-smoke.js"); +const schedulerSmoke = read("scripts/scheduler-placement-smoke.js"); +const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js"); + +for (const [name, pattern] of [ + ["auth contexts carry tenant and project", /pub struct AuthContext[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["enrollment grants carry tenant and project", /pub struct EnrollmentGrant[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["node credentials carry tenant and project", /pub struct NodeCredential[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["same tenant/project denies tenant mismatch", /pub fn same_tenant_project[\s\S]*tenant mismatch/], + ["same tenant/project denies project mismatch", /pub fn same_tenant_project[\s\S]*project mismatch/], + ["auth unit denies cross-tenant access", /fn tenant_project_scope_denies_cross_tenant_access\(\)/], +]) { + expect(auth, name, pattern); +} + +for (const [name, pattern] of [ + ["artifact metadata carries tenant and project", /pub struct ArtifactMetadata[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/], + ["download links carry tenant project process and actor", /pub struct DownloadLink[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId[\s\S]*pub actor: Actor/], + ["downloads authorize against scoped metadata", /let authz = same_tenant_project\(context, &scope\)/], + ["artifact test denies cross-tenant download", /fn cross_tenant_download_is_denied_even_with_known_artifact_id\(\)/], + ["artifact test denies cross-project download", /fn cross_project_download_is_denied_even_with_known_artifact_id\(\)/], +]) { + expect(artifact, name, pattern); +} + +for (const [name, pattern] of [ + ["panel state carries tenant project and process", /pub struct PanelState[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], + ["panel events carry tenant project and process", /pub struct PanelEvent[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId[\s\S]*pub process: ProcessId/], + ["panel events reject scope mismatch", /PanelError::ScopeMismatch/], + ["panel scope validation compares tenant project process", /self\.tenant != event\.tenant[\s\S]*self\.project != event\.project[\s\S]*self\.process != event\.process/], +]) { + expect(operatorPanel, name, pattern); +} + +expect( + source, + "source preparation carries tenant and project", + /pub struct SourcePreparation[\s\S]*pub tenant: TenantId[\s\S]*pub project: ProjectId/ +); + +for (const [name, pattern] of [ + ["project creation rejects foreign tenant reuse", /project id is outside the signed-in tenant scope/], + ["project selection rejects foreign tenant", /project is outside the signed-in tenant scope/], + ["project listing uses tenant-scoped context", /CoordinatorRequest::ListProjects[\s\S]*list_projects\(&context\)/], + ["node capability reports check enrollment tenant scope", /node capability report is outside the enrolled tenant\/project scope/], + ["operator panel stop state is tenant/project/process keyed", /type PanelStopKey = \(TenantId, ProjectId, ProcessId\)/], + ["download service tests tenant mismatch", /cross_tenant\.to_string\(\)\.contains\("tenant mismatch"\)/], + ["download service tests project mismatch", /cross_project\.to_string\(\)\.contains\("project mismatch"\)/], + ["node capability test rejects cross-scope report", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/], + ["source preparation test rejects cross-scope completion", /fn service_rejects_source_preparation_completion_outside_node_scope\(\)/], +]) { + expect(coordinatorService, name, pattern); +} + +for (const [name, sourceText, patterns] of [ + [ + "artifact download smoke", + artifactDownloadSmoke, + [ + /const crossTenant = await send/, + /const crossProject = await send/, + /const crossTenantOpen = await send/, + /const crossProjectOpen = await send/, + /tenant mismatch/, + /project mismatch/, + /token is invalid/, + ], + ], + [ + "operator panel smoke", + operatorPanelSmoke, + [/const crossTenant = await send/, /render_operator_panel/, /scope\|tenant\|project/], + ], + [ + "scheduler and capability smoke", + schedulerSmoke, + [ + /const crossScopeInspection = await send/, + /assert\.strictEqual\(crossScopeInspection\.descriptors\.length, 0\)/, + /const crossTenantReport = await send/, + /tenant\\\/project scope/, + ], + ], + [ + "source preparation smoke", + sourcePreparationSmoke, + [/const crossTenantCompletion = await send/, /complete_source_preparation/, /tenant\\\/project scope/i], + ], +]) { + for (const pattern of patterns) { + expect(sourceText, name, pattern); + } +} + +const hostedLibRoot = maybeRead(["private", "hosted-policy", "src", "lib.rs"]); +const hostedServiceSource = maybeRead([ + "private", + "hosted-policy", + "src", + "bin", + "clusterflux-hosted-service.rs", +]); +const hostedLib = hostedLibRoot; +const hostedSmoke = maybeRead([ + "private", + "hosted-policy", + "scripts", + "hosted-client-compat-smoke.js", +]); + +if (hostedLib && hostedServiceSource && hostedSmoke) { + for (const [name, pattern] of [ + ["hosted private layer is a compact identity broker", /pub struct HostedIdentityBroker[\s\S]*cli_session_ttl_seconds/], + ["hosted private layer has no parallel coordinator", /Runtime, persistence, nodes, processes,[\s\S]*remain owned by public CoordinatorService/], + ["hosted service delegates Client state to Core", /core_coordinator: CoordinatorService/], + ["hosted login creates project through Core", /issue_cli_session[\s\S]*AuthenticatedCoordinatorRequest::CreateProject/], + ]) { + expect(`${hostedLib}\n${hostedServiceSource}`, name, pattern); + } + + for (const forbidden of [ + "HostedCommunityControlPlane", + "HostedObservabilitySnapshot", + "agent_public_keys: BTreeMap", + "node_statuses: BTreeMap", + "process_statuses: BTreeMap", + "debug_sessions: BTreeMap", + ]) { + assert( + !hostedLib.includes(forbidden), + `private hosted policy must not reimplement Core state: ${forbidden}` + ); + } + + for (const [name, pattern] of [ + ["client-supplied identity is denied", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/], + ["second OIDC subject receives a different tenant", /assert\.notStrictEqual\(victimLogin\.session\.tenant, session\.tenant\)/], + ["cross-tenant process events are denied", /const crossTenantTaskEventsDenied = await sendHostedControl[\s\S]*vp-victim[\s\S]*scope\|denied\|unauthorized/], + ]) { + expect(hostedSmoke, name, pattern); + } +} + +console.log("Tenant isolation contract smoke passed"); diff --git a/scripts/user-session-token-boundary-smoke.js b/scripts/user-session-token-boundary-smoke.js new file mode 100755 index 0000000..d00a9da --- /dev/null +++ b/scripts/user-session-token-boundary-smoke.js @@ -0,0 +1,120 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); + +function read(file) { + return fs.readFileSync(path.join(repo, file), "utf8"); +} + +function extractBalancedBlock(source, marker) { + const start = source.indexOf(marker); + assert(start >= 0, `missing marker ${marker}`); + const open = source.indexOf("{", start); + assert(open >= 0, `missing opening brace for ${marker}`); + let depth = 0; + for (let index = open; index < source.length; index += 1) { + const char = source[index]; + if (char === "{") depth += 1; + if (char === "}") { + depth -= 1; + if (depth === 0) return source.slice(start, index + 1); + } + } + throw new Error(`missing closing brace for ${marker}`); +} + +function extractEnumVariant(source, variant) { + const marker = ` ${variant} {`; + return extractBalancedBlock(source, marker); +} + +const forbiddenUserSessionCredentials = + /\b(BrowserSession|CliDeviceSession|BrowserLoginFlow|CliLoginFlow|authorization_url|verification_url|device_code|access_token|refresh_token|provider_token|provider_tokens|session_token|user_token|oauth_token)\b/i; + +function assertNoUserSessionCredential(surface, text) { + assert.doesNotMatch( + text, + forbiddenUserSessionCredentials, + `${surface} must not carry user OAuth/browser/session credentials` + ); +} + +const coordinatorService = `${read("crates/clusterflux-coordinator/src/service/protocol.rs")}\n${read("crates/clusterflux-coordinator/src/service/protocol/responses.rs")}`; +for (const variant of [ + "AdminStatus", + "SuspendTenant", + "AttachNode", + "RegisterAgentPublicKey", + "ListAgentPublicKeys", + "RotateAgentPublicKey", + "RevokeAgentPublicKey", + "NodeHeartbeat", + "ReportNodeCapabilities", + "RevokeNodeCredential", + "RequestRendezvous", + "RequestSourcePreparation", + "CompleteSourcePreparation", + "StartProcess", + "ReconnectNode", + "CancelTask", + "CancelProcess", + "PollTaskControl", + "RestartTask", + "DebugAttach", + "TaskCompleted", +]) { + assertNoUserSessionCredential( + `CoordinatorRequest::${variant}`, + extractEnumVariant(coordinatorService, variant) + ); +} + +const nodeRuntime = [ + read("crates/clusterflux-node/src/lib.rs"), + read("crates/clusterflux-node/src/command_runner.rs"), +].join("\n"); +for (const marker of [ + "pub struct LinuxCommandRunPlan", + "pub struct LinuxCommandTaskOutput", + "pub struct CapturedCommandLogs", + "pub struct VirtualThreadCommand", + "pub struct CommandOutput", +]) { + assertNoUserSessionCredential(marker, extractBalancedBlock(nodeRuntime, marker)); +} + +const coreExecution = read("crates/clusterflux-core/src/execution.rs"); +assertNoUserSessionCredential( + "CommandInvocation", + extractBalancedBlock(coreExecution, "pub struct CommandInvocation") +); + +const dapAdapter = read("crates/clusterflux-dap/src/variables.rs"); +assertNoUserSessionCredential( + "DAP variables response", + extractBalancedBlock(dapAdapter, "fn variables_response") +); + +const panel = read("crates/clusterflux-core/src/operator_panel.rs"); +assertNoUserSessionCredential( + "PanelEvent", + extractBalancedBlock(panel, "pub struct PanelEvent") +); + +const auth = read("crates/clusterflux-core/src/auth.rs"); +assert.match( + auth, + /task_credentials_do_not_contain_user_session/, + "core auth must keep the task credential user-session guard" +); +assert.match( + auth, + /CredentialKind::BrowserSession \| CredentialKind::CliDeviceSession/, + "task credential guard must reject browser and CLI sessions" +); + +console.log("User session token boundary smoke passed"); diff --git a/scripts/verify-public-split.sh b/scripts/verify-public-split.sh new file mode 100755 index 0000000..f264477 --- /dev/null +++ b/scripts/verify-public-split.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_commit="$(git -C "$repo_root" rev-parse HEAD)" +export CLUSTERFLUX_ACCEPTANCE_COMMIT="$source_commit" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +tar \ + --exclude='./.git' \ + --exclude='./target' \ + --exclude='./private' \ + --exclude='./internal' \ + --exclude='./experiments' \ + --exclude='./.clusterflux' \ + --exclude='./vscode-extension/node_modules' \ + --exclude='./scripts/containers-home' \ + -C "$repo_root" \ + -cf - . | tar -C "$tmp_dir" -xf - + +if find "$tmp_dir" -path "$tmp_dir/private" -print -quit | grep -q .; then + echo "private directory leaked into public split" >&2 + exit 1 +fi + +if find "$tmp_dir" -path "$tmp_dir/experiments" -print -quit | grep -q .; then + echo "experiments directory leaked into public split" >&2 + exit 1 +fi + +if find "$tmp_dir" -path "$tmp_dir/internal" -print -quit | grep -q .; then + echo "internal directory leaked into public split" >&2 + exit 1 +fi + +(cd "$tmp_dir" && scripts/check-old-name.sh) +(cd "$tmp_dir" && CLUSTERFLUX_FILTERED_PUBLIC_TREE=1 node scripts/check-docs.js) +(cd "$tmp_dir" && scripts/check-code-size.sh) +(cd "$tmp_dir" && node scripts/resource-metering-contract-smoke.js) +(cd "$tmp_dir" && node scripts/hostile-input-contract-smoke.js) +(cd "$tmp_dir" && node scripts/tenant-isolation-contract-smoke.js) +(cd "$tmp_dir" && cargo fmt --all --check) +CARGO_TARGET_DIR="$tmp_dir/target" cargo test \ + --workspace \ + --all-targets \ + --manifest-path "$tmp_dir/Cargo.toml" +CARGO_TARGET_DIR="$tmp_dir/target" cargo build \ + --workspace \ + --all-targets \ + --manifest-path "$tmp_dir/Cargo.toml" +(cd "$tmp_dir" && node scripts/cli-install-smoke.js) +(cd "$tmp_dir" && node scripts/flagship-demo-smoke.js) +(cd "$tmp_dir" && node scripts/cli-local-run-smoke.js) +(cd "$tmp_dir" && node scripts/node-attach-smoke.js) +(cd "$tmp_dir" && node scripts/vscode-extension-smoke.js) +(cd "$tmp_dir" && node scripts/vscode-f5-smoke.js) +(cd "$tmp_dir" && node scripts/artifact-download-smoke.js) +(cd "$tmp_dir" && node scripts/artifact-export-smoke.js) + +public_digest="$( + find "$tmp_dir" \ + -path "$tmp_dir/target" -prune -o \ + -type f -print0 \ + | LC_ALL=C sort -z \ + | xargs -0 sha256sum \ + | sha256sum \ + | cut -d' ' -f1 +)" +printf 'Filtered public tree passed: commit=%s digest=sha256:%s\n' \ + "$source_commit" \ + "$public_digest" diff --git a/scripts/vscode-extension-smoke.js b/scripts/vscode-extension-smoke.js new file mode 100755 index 0000000..7eddcea --- /dev/null +++ b/scripts/vscode-extension-smoke.js @@ -0,0 +1,229 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const assert = require("assert"); + +const extension = require("../vscode-extension/extension"); +const packageJson = require("../vscode-extension/package.json"); +const extensionSource = fs.readFileSync( + path.join(__dirname, "../vscode-extension/extension.js"), + "utf8" +); +const repo = path.resolve(__dirname, ".."); + +assert.strictEqual(packageJson.main, "./extension.js"); +assert(fs.existsSync(path.join(__dirname, "../vscode-extension", packageJson.main))); +assert.deepStrictEqual(packageJson.dependencies || {}, {}); +const root = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-vscode-")); +fs.mkdirSync(path.join(root, "envs/linux"), { recursive: true }); +fs.writeFileSync(path.join(root, "envs/linux/Containerfile"), "FROM alpine\n"); +fs.mkdirSync(path.join(root, ".clusterflux"), { recursive: true }); +fs.writeFileSync( + extension.clusterfluxViewStatePath(root), + JSON.stringify({ + nodes: [{ id: "node-linux", status: "online", capabilities: "Command RootlessPodman" }], + processes: [{ id: "vp-build", status: "running", entry: "build" }], + logs: [{ task: "compile-linux", message: "stdout=12 stderr=0", bytes: 12 }], + artifacts: [{ path: "/vfs/artifacts/app.tar.zst", status: "retained", size: 12 }], + inspector: [{ label: "debug", value: "attached" }] + }) +); + +const envs = extension.discoverEnvironmentNames(root); +assert.deepStrictEqual(envs, ["linux"]); + +const diagnostics = extension.diagnoseEnvReferences( + 'let _ = env!("linux"); let _ = env!("windows");', + envs +); +assert.strictEqual(diagnostics.length, 1); +assert.strictEqual(diagnostics[0].name, "windows"); +assert.match(diagnostics[0].message, /envs\/windows\/Containerfile/); + +const inspectCommand = extension.bundleInspectCommand(root, "/repo"); +assert.strictEqual(inspectCommand.command, "cargo"); +assert.deepStrictEqual(inspectCommand.args.slice(0, 8), [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "bundle" +]); +assert(inspectCommand.args.includes("inspect")); +assert(inspectCommand.args.includes("--project")); +assert(inspectCommand.args.includes(root)); +assert(inspectCommand.args.includes("--json")); + +const refreshed = extension.refreshBundleBeforeLaunch(root, "/repo", (command, args, options) => { + assert.strictEqual(command, "cargo"); + assert(args.includes("bundle")); + assert.strictEqual(options.cwd, "/repo"); + return { + status: 0, + stdout: JSON.stringify({ metadata: { identity: "sha256:abc" } }), + stderr: "" + }; +}); +assert.strictEqual(refreshed.metadata.identity, "sha256:abc"); + +const launch = extension.resolveClusterfluxDebugConfiguration( + { uri: { fsPath: root } }, + {} +); +assert.deepStrictEqual(launch, { + name: "Clusterflux: Launch Virtual Process", + type: "clusterflux", + request: "launch", + entry: "build", + project: root, + runtimeBackend: "local-services" +}); +assert.strictEqual( + extension.clusterfluxProcessId("/workspace/app", "build"), + "vp-e4bd6ef50539" +); +assert.strictEqual( + extension.existingProcessRelationship( + { process: "vp-e4bd6ef50539", state: "running" }, + "vp-e4bd6ef50539" + ), + "same_launch_target" +); +assert.strictEqual( + extension.existingProcessRelationship( + { process: "vp-other", state: "running" }, + "vp-e4bd6ef50539" + ), + "different_launch_target" +); + +const liveProcesses = extension.loadLiveProcesses(root, "/repo", (_command, args) => { + assert.deepStrictEqual(args.slice(-3), ["process", "list", "--json"]); + return { + status: 0, + stdout: JSON.stringify({ + coordinator: "https://clusterflux.michelpaulissen.com", + tenant: "tenant-live", + project: "project-live", + user: "user-live", + processes: [{ process: "vp-live", state: "cancelling" }] + }), + stderr: "" + }; +}); +assert.deepStrictEqual(liveProcesses.processes, [ + { process: "vp-live", state: "cancelling" } +]); +assert.strictEqual(liveProcesses.project, "project-live"); + +const adapter = extension.debugAdapterExecutableSpec(root, repo); +assert.strictEqual(adapter.command, "cargo"); +assert.deepStrictEqual(adapter.args, [ + "run", + "-q", + "-p", + "clusterflux-dap", + "--bin", + "clusterflux-debug-dap" +]); +assert.deepStrictEqual(adapter.options, { cwd: repo }); + +const releasedAdapterPath = path.join(root, process.platform === "win32" ? "clusterflux-debug-dap.exe" : "clusterflux-debug-dap"); +fs.writeFileSync(releasedAdapterPath, ""); +const releasedAdapter = extension.debugAdapterExecutableSpec(root, repo); +assert.strictEqual(releasedAdapter.command, releasedAdapterPath); +assert.deepStrictEqual(releasedAdapter.args, []); +assert.deepStrictEqual(releasedAdapter.options, { cwd: root }); + +assert.throws( + () => + extension.refreshBundleBeforeLaunch(root, "/repo", () => ({ + status: 1, + stdout: "", + stderr: "missing environment linux" + })), + /missing environment linux/ +); + +assert( + packageJson.contributes.viewsContainers.activitybar.some( + (container) => container.id === "clusterflux" && container.title === "Clusterflux" + ), + "package.json must contribute a Clusterflux activity-bar container" +); +assert( + fs.existsSync(path.join(__dirname, "../vscode-extension/resources/clusterflux.svg")), + "Clusterflux activity-bar icon must exist" +); +const packageViewIds = packageJson.contributes.views.clusterflux.map((view) => view.id).sort(); +const descriptorViewIds = extension.clusterfluxViewDescriptors().map((view) => view.id).sort(); +assert.deepStrictEqual(descriptorViewIds, [ + "clusterflux.artifacts", + "clusterflux.inspector", + "clusterflux.logs", + "clusterflux.nodes", + "clusterflux.processes" +]); +assert.deepStrictEqual(packageViewIds, descriptorViewIds); +for (const viewId of descriptorViewIds) { + const items = extension.clusterfluxViewItems( + extension.loadClusterfluxViewState(root), + viewId + ); + assert( + items.length > 0 && !items[0].label.startsWith("No "), + `${viewId} should render state-backed items` + ); +} +assert.deepStrictEqual( + extension.clusterfluxViewItems(extension.loadClusterfluxViewState(root), "clusterflux.nodes")[0], + { + label: "node-linux", + description: "online Command RootlessPodman" + } +); + +assert( + packageJson.contributes.debuggers.some((debuggerContribution) => debuggerContribution.type === "clusterflux"), + "package.json must contribute the clusterflux debugger type" +); +for (const viewId of descriptorViewIds) { + assert( + packageJson.activationEvents.includes(`onView:${viewId}`), + `${viewId} should activate the extension when opened` + ); +} +const launchProperties = + packageJson.contributes.debuggers[0].configurationAttributes.launch.properties; +assert.strictEqual(launchProperties.runtimeBackend.default, "local-services"); +assert(launchProperties.runtimeBackend.enum.includes("live-services")); +assert.strictEqual(launchProperties.coordinatorEndpoint.default, undefined); +assert.strictEqual(launchProperties.tenant, undefined); +assert.strictEqual(launchProperties.projectId, undefined); +assert.strictEqual(launchProperties.actorUser, undefined); +assert( + packageJson.contributes.debuggers[0].configurationAttributes.attach, + "package.json must contribute an attach configuration" +); +for (const command of [ + "clusterflux.refreshProcesses", + "clusterflux.process.attach", + "clusterflux.process.cancel", + "clusterflux.process.abort" +]) { + assert( + packageJson.contributes.commands.some((entry) => entry.command === command), + `${command} must be contributed` + ); +} +assert.match(extensionSource, /registerDebugAdapterDescriptorFactory\("clusterflux"/); +assert.match(extensionSource, /clusterflux-debug-dap/); +assert.match(extensionSource, /\.clusterflux\/views\.json/); + +fs.rmSync(root, { recursive: true, force: true }); +console.log("VS Code extension smoke passed"); diff --git a/scripts/vscode-f5-smoke.js b/scripts/vscode-f5-smoke.js new file mode 100755 index 0000000..97a6654 --- /dev/null +++ b/scripts/vscode-f5-smoke.js @@ -0,0 +1,310 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const extension = require("../vscode-extension/extension"); + +class DapClient { + constructor(spec) { + this.child = cp.spawn(spec.command, spec.args, { + cwd: spec.options && spec.options.cwd, + env: spec.options && spec.options.env, + detached: process.platform !== "win32" + }); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(message)); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error( + `DAP ${command} failed: ${message.message || JSON.stringify(message)}\n${this.stderr}` + ); + } + return message; + } + + terminate() { + if (this.child.exitCode !== null) return; + if (process.platform === "win32") { + this.child.kill("SIGKILL"); + return; + } + try { + process.kill(-this.child.pid, "SIGKILL"); + } catch (_) { + this.child.kill("SIGKILL"); + } + } + + waitFor(predicate, timeoutMs = 240000) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.terminate(); + reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +(async () => { + const repo = path.resolve(__dirname, ".."); + const project = path.join(repo, "examples/launch-build-demo"); + const buildSourceLines = fs + .readFileSync(path.join(project, "src/build.rs"), "utf8") + .split(/\r?\n/); + const sourceLine = (needle) => { + const index = buildSourceLines.findIndex((line) => line.includes(needle)); + assert(index >= 0, `flagship source must contain ${needle}`); + return index + 1; + }; + const buildMainLine = sourceLine("pub async fn build_main()"); + const launchConfig = extension.resolveClusterfluxDebugConfiguration( + { uri: { fsPath: project } }, + {} + ); + + assert.strictEqual(launchConfig.type, "clusterflux"); + assert.strictEqual(launchConfig.request, "launch"); + assert.strictEqual(launchConfig.entry, "build"); + assert.strictEqual(launchConfig.project, project); + assert.strictEqual(launchConfig.runtimeBackend, "local-services"); + + const inspection = extension.refreshBundleBeforeLaunch(project, repo); + assert.match(inspection.metadata.identity, /^sha256:/); + + // Keep the timed attach focused on the runtime boundary. A completely fresh + // public checkout may otherwise spend most of that window compiling the + // coordinator or node after the adapter has already started waiting. + cp.execFileSync( + "cargo", + [ + "build", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-node", + ], + { cwd: repo, stdio: "inherit" } + ); + const executableSuffix = process.platform === "win32" ? ".exe" : ""; + const adapterSpec = extension.debugAdapterExecutableSpec(repo); + adapterSpec.options = { + ...(adapterSpec.options || {}), + env: { + ...process.env, + CLUSTERFLUX_COORDINATOR_BIN: path.join( + repo, + "target", + "debug", + `clusterflux-coordinator${executableSuffix}` + ), + CLUSTERFLUX_NODE_BIN: path.join( + repo, + "target", + "debug", + `clusterflux-node${executableSuffix}` + ), + }, + }; + + const client = new DapClient(adapterSpec); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", launchConfig); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const breakpoints = client.send("setBreakpoints", { + source: { path: path.join(project, "src/build.rs") }, + breakpoints: [{ line: buildMainLine }] + }); + const breakpointResponse = await client.response(breakpoints, "setBreakpoints"); + assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, true); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + const stopped = await client.waitFor( + (message) => message.type === "event" && message.event === "stopped" + ); + assert.strictEqual(stopped.body.allThreadsStopped, true); + assert.strictEqual(stopped.body.reason, "breakpoint"); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body.threads; + const mainThread = threads.find((thread) => thread.name.includes("build coordinator main")); + assert(mainThread, "F5 launch must expose the real coordinator-main entrypoint thread"); + + const stackRequest = client.send("stackTrace", { + threadId: mainThread.id, + startFrame: 0, + levels: 1 + }); + const stack = (await client.response(stackRequest, "stackTrace")).body.stackFrames; + assert.strictEqual(stack[0].line, buildMainLine); + assert.strictEqual(stack[0].source.path, path.join(project, "src/build.rs")); + assert.strictEqual(stack[0].source.sourceReference || 0, 0); + + const sourceRequest = client.send("source", { source: stack[0].source }); + const source = (await client.response(sourceRequest, "source")).body; + assert.match(source.content, /compile_linux/); + + const scopesRequest = client.send("scopes", { frameId: stack[0].id }); + const scopes = (await client.response(scopesRequest, "scopes")).body.scopes; + const localsScope = scopes.find((scope) => scope.name === "Source Locals"); + const argsScope = scopes.find((scope) => scope.name === "Task Args and Handles"); + const runtimeScope = scopes.find((scope) => scope.name === "Clusterflux Runtime"); + const outputScope = scopes.find((scope) => scope.name === "Recent Output"); + assert(localsScope, "F5 launch must expose source locals scope"); + assert(argsScope, "F5 launch must expose task args and handles"); + assert(runtimeScope, "F5 launch must expose Clusterflux runtime state"); + assert(outputScope, "F5 launch must expose recent output state"); + + const localsRequest = client.send("variables", { + variablesReference: localsScope.variablesReference + }); + const locals = (await client.response(localsRequest, "variables")).body.variables; + assert( + locals.some( + (variable) => + variable.name === "unavailable-local-diagnostic" && + String(variable.value).includes("cannot be inspected") + ), + "source locals scope must report unavailable real Rust locals explicitly" + ); + + const runtimeRequest = client.send("variables", { + variablesReference: runtimeScope.variablesReference + }); + const runtime = (await client.response(runtimeRequest, "variables")).body.variables; + assert( + runtime.some( + (variable) => variable.name === "runtime_backend" && variable.value === "LocalServices" + ), + "extension-resolved F5 launch must use the real local-services backend" + ); + assert( + runtime.some( + (variable) => variable.name === "coordinator_task_events" && variable.value === 0 + ), + "a task frozen at its entry probe must not fabricate a terminal task event" + ); + assert( + runtime.some( + (variable) => + variable.name === "command_status" && + String(variable.value).includes( + "frozen through local services at executing Wasm probe" + ) + ) + ); + assert( + runtime.some( + (variable) => variable.name === "state" && variable.value === "Frozen" + ), + "F5 must expose the node-acknowledged frozen participant state" + ); + assert(runtime.some((variable) => variable.name === "command_spec")); + assert(runtime.some((variable) => variable.name === "stdout_tail")); + assert(runtime.some((variable) => variable.name === "stderr_tail")); + + const outputRequest = client.send("variables", { + variablesReference: outputScope.variablesReference + }); + const output = (await client.response(outputRequest, "variables")).body.variables; + assert(output.some((variable) => variable.name === "stdout_tail")); + assert(output.some((variable) => variable.name === "stderr_tail")); + + await client.close(); + } catch (error) { + client.terminate(); + throw error; + } + + console.log("VS Code F5 smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/wasmtime-assignment-smoke.js b/scripts/wasmtime-assignment-smoke.js new file mode 100755 index 0000000..9ec2683 --- /dev/null +++ b/scripts/wasmtime-assignment-smoke.js @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const sdkRuntime = fs.readFileSync( + path.join(repo, "crates/clusterflux-sdk/src/sdk_runtime.rs"), + "utf8" +); + +assert.doesNotMatch( + sdkRuntime, + /"type": "launch_task"/, + "native SDK code must not submit external TaskV1 work" +); +assert.match( + sdkRuntime, + /native SDK task spawning requires coordinator EntrypointV1 execution/ +); + +cp.execFileSync("node", ["scripts/cli-local-run-smoke.js"], { + cwd: repo, + stdio: "inherit", +}); +cp.execFileSync( + "cargo", + ["test", "-p", "clusterflux-node", "wasmtime_runtime_runs_named_task_export"], + { cwd: repo, stdio: "inherit" } +); +cp.execFileSync( + "cargo", + [ + "test", + "-p", + "clusterflux-coordinator", + "signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only", + ], + { cwd: repo, stdio: "inherit" } +); + +console.log("Wasmtime assignment smoke passed"); diff --git a/scripts/wasmtime-node-smoke.js b/scripts/wasmtime-node-smoke.js new file mode 100644 index 0000000..03b8f81 --- /dev/null +++ b/scripts/wasmtime-node-smoke.js @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +const repo = path.resolve(__dirname, ".."); +const wasmTarget = path.join( + repo, + "target", + "wasm32-unknown-unknown", + "release", + "launch_build_demo.wasm" +); + +cp.execFileSync( + "cargo", + [ + "build", + "--release", + "-p", + "launch-build-demo", + "--target", + "wasm32-unknown-unknown", + ], + { + cwd: repo, + stdio: "inherit", + env: { + ...process.env, + CARGO_PROFILE_RELEASE_OPT_LEVEL: "z", + CARGO_PROFILE_RELEASE_LTO: "thin", + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1", + }, + } +); + +assert(fs.existsSync(wasmTarget), `missing compiled Wasm module at ${wasmTarget}`); + +const output = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + wasmTarget, + "task_add_one", + "41", + "42", + ], + { cwd: repo, encoding: "utf8" } +); + +const report = JSON.parse(output); +assert.strictEqual(report.type, "wasmtime_task_smoke"); +assert.strictEqual(report.export, "task_add_one"); +assert.strictEqual(report.arg, 41); +assert.strictEqual(report.result, 42); + +const debugOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--debug-freeze-resume", + wasmTarget, + "task_add_one", + "41", + "42", + ], + { cwd: repo, encoding: "utf8" } +); +const debugReport = JSON.parse(debugOutput); +assert.strictEqual(debugReport.type, "wasmtime_debug_freeze_resume_smoke"); +assert.strictEqual(debugReport.export, "task_add_one"); +assert.strictEqual(debugReport.task, "task_add_one"); +assert.strictEqual(debugReport.frozen_state, "Frozen"); +assert.strictEqual(debugReport.resumed_state, "Running"); +assert(debugReport.stack_frames.some((frame) => String(frame).includes("task_add_one"))); +assert( + debugReport.local_values.some( + ([name, value]) => name === "wasm_local_0" && String(value).includes("41") + ), + "Wasmtime debug snapshot must expose the real i32 argument as a frame local" +); +assert.strictEqual(debugReport.node_runtime_captured_wasm_locals, true); +assert.strictEqual(debugReport.result, 42); +assert.strictEqual(debugReport.node_runtime_reached_wasm_task, true); + +const hostCommandOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--host-command", + wasmTarget, + "compile_linux", + ], + { cwd: repo, encoding: "utf8" } +); +const hostCommandReport = JSON.parse(hostCommandOutput); +assert.strictEqual(hostCommandReport.type, "wasmtime_host_command_smoke"); +assert.strictEqual(hostCommandReport.task, "compile_linux"); +assert.match(hostCommandReport.export, /^clusterflux_task_v1_[0-9a-f]{64}$/); +assert.strictEqual(hostCommandReport.program, "cc"); +assert.deepStrictEqual(hostCommandReport.args, [ + "-Os", + "-static", + "-s", + "fixture/hello-clusterflux.c", + "-o", + "/clusterflux/output/hello-clusterflux", +]); +assert.strictEqual(hostCommandReport.working_directory, "/workspace"); +assert.deepStrictEqual(hostCommandReport.environment_variables, { + SOURCE_DATE_EPOCH: "0", +}); +assert.strictEqual(hostCommandReport.timeout_ms, 180000); +assert.strictEqual(hostCommandReport.network, "disabled"); +assert.strictEqual(hostCommandReport.status_code, 0); +assert.strictEqual(hostCommandReport.stdout, ""); +assert.strictEqual(hostCommandReport.artifact_name, "hello-clusterflux"); +assert.strictEqual(hostCommandReport.artifact_size_bytes, "hello-clusterflux".length); +assert.match(hostCommandReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); +assert.strictEqual(hostCommandReport.node_host_import, "clusterflux.command_run_v1"); +assert.strictEqual(hostCommandReport.artifact_host_import, "clusterflux.vfs_operation_v1"); +assert.strictEqual(hostCommandReport.flagship_linux_build_task, true); +assert.strictEqual(hostCommandReport.node_executed_host_command, true); +assert.strictEqual(hostCommandReport.hosted_control_plane_ran_command, false); + +const artifactOutput = cp.execFileSync( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-wasmtime-smoke", + "--", + "--task-artifact", + wasmTarget, + "package_release", + ], + { cwd: repo, encoding: "utf8" } +); +const artifactReport = JSON.parse(artifactOutput); +assert.strictEqual(artifactReport.type, "wasmtime_task_artifact_smoke"); +assert.strictEqual(artifactReport.task, "package_release"); +assert.strictEqual(artifactReport.artifact_name, "release.tar"); +assert.strictEqual(artifactReport.artifact_size_bytes, "release.tar".length); +assert.match(artifactReport.artifact_digest, /^sha256:[0-9a-f]{64}$/); +assert.strictEqual(artifactReport.host_import, "clusterflux.vfs_operation_v1"); +assert.strictEqual(artifactReport.host_issued_handle_returned, true); +assert.ok(artifactReport.artifact.id.endsWith(artifactReport.artifact_digest.slice("sha256:".length))); + +console.log("Wasmtime node smoke passed"); diff --git a/scripts/windows-best-effort-smoke.js b/scripts/windows-best-effort-smoke.js new file mode 100755 index 0000000..bdb0442 --- /dev/null +++ b/scripts/windows-best-effort-smoke.js @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const digest = (value) => + `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +const environmentDigest = digest("windows-command-dev-environment"); +const sourceDigest = digest("windows-best-effort-source"); +const windowsArtifactBytes = Buffer.from("windows-output-data"); +const windowsArtifactDigest = digest(windowsArtifactBytes); +const emptyWasm = Buffer.from([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); +const emptyWasmDigest = digest(emptyWasm); +const nodeDaemon = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/daemon.rs"), "utf8"); +const taskReports = fs.readFileSync( + path.join(repo, "crates/clusterflux-node/src/task_reports.rs"), + "utf8" +); +const nodeLib = fs.readFileSync(path.join(repo, "crates/clusterflux-node/src/lib.rs"), "utf8"); +const windowsDev = fs.readFileSync( + path.join(repo, "crates/clusterflux-node/src/windows_dev.rs"), + "utf8" +); +const executionCore = fs.readFileSync( + path.join(repo, "crates/clusterflux-core/src/execution.rs"), + "utf8" +); +const dapSource = [ + fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/demo_backend.rs"), "utf8"), + fs.readFileSync(path.join(repo, "crates/clusterflux-dap/src/tests.rs"), "utf8"), +].join("\n"); +const windowsNode = "windows-node"; +const windowsIdentity = nodeIdentity("windows-best-effort-smoke", windowsNode); + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function windowsCapabilities() { + return { + os: "Windows", + arch: "x86_64", + capabilities: ["Command", "VfsArtifacts", "WindowsCommandDev"], + environment_backends: ["WindowsCommandDev"], + source_providers: ["filesystem"] + }; +} + +function assertWindowsBackendBoundary() { + assert.match(nodeDaemon, /CoordinatorSession::connect\(&args\.coordinator\)/); + assert.match(nodeDaemon, /record_completed_task\(/); + assert.match(taskReports, /"type": "task_completed"/); + assert.doesNotMatch(nodeDaemon, /WindowsCommandDev|windows-command-dev|cfg\(windows\)/); + + assert.match(nodeLib, /impl CommandBackend for LinuxRootlessPodmanBackend/); + assert.match(nodeLib, /mod windows_dev/); + assert.match(nodeLib, /pub use windows_dev::\{WindowsCommandDevBackend, WindowsSandboxStubBackend\}/); + assert.match(windowsDev, /impl CommandBackend for WindowsCommandDevBackend/); + assert.match(windowsDev, /impl CommandBackend for WindowsSandboxStubBackend/); + assert.doesNotMatch(windowsDev, /LinuxRootlessPodmanBackend/); + assert.match(executionCore, /\bLinuxRootlessPodman\b/); + assert.match(executionCore, /\bWindowsCommandDev\b/); + assert.match(executionCore, /\bStubbedWindowsSandbox\b/); + + assert.match(dapSource, /thread\(WINDOWS_THREAD, "compile-windows", "compile windows"/); + assert.match(dapSource, /fn launch_threads_include_windows_task_in_same_virtual_process\(\)/); + assert.match(dapSource, /fn windows_thread_runtime_variables_share_virtual_process\(\)/); +} + +(async () => { + assertWindowsBackendBoundary(); + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const attached = await send(addr, { + type: "attach_node", + tenant: "tenant", + project: "project", + node: windowsNode, + public_key: windowsIdentity.publicKey + }); + assert.strictEqual(attached.type, "node_attached"); + assert.strictEqual(attached.node, windowsNode); + + const recorded = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: windowsNode, + capabilities: windowsCapabilities(), + cached_environment_digests: [environmentDigest], + dependency_cache_digests: [], + source_snapshots: [sourceDigest], + artifact_locations: [], + direct_connectivity: true, + online: true + })); + assert.strictEqual(recorded.type, "node_capabilities_recorded"); + assert.strictEqual(recorded.node, windowsNode); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"] + }, + environment_digest: environmentDigest, + required_capabilities: ["Command"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: [], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, windowsNode); + assert.ok(placement.placement.reasons.includes("warm environment cache")); + assert.ok(placement.placement.reasons.includes("source snapshot already local")); + + const started = await send(addr, { + type: "start_process", + tenant: "tenant", + project: "project", + process: "vp-windows" + }); + assert.strictEqual(started.type, "process_started"); + assert.strictEqual(started.process, "vp-windows"); + + const reconnected = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "reconnect_node", { + type: "reconnect_node", + node: windowsNode, + process: "vp-windows", + epoch: started.epoch + })); + assert.strictEqual(reconnected.type, "node_reconnected"); + + const launched = await send(addr, { + type: "launch_task", + tenant: "tenant", + project: "project", + actor_user: "operator", + task_spec: { + tenant: "tenant", + project: "project", + process: "vp-windows", + task_definition: "windows-command-dev", + task_instance: "windows-command-dev", + dispatch: { + kind: "coordinator_node_wasm", + export: "windows_command_dev", + abi: "task_v1", + }, + environment_id: "windows", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"], + }, + environment_digest: environmentDigest, + required_capabilities: ["Command", "WindowsCommandDev"], + dependency_cache: null, + source_snapshot: sourceDigest, + required_artifacts: [], + args: [{ SourceSnapshot: sourceDigest }], + vfs_epoch: started.epoch, + bundle_digest: emptyWasmDigest, + }, + wait_for_node: false, + artifact_path: "/vfs/artifacts/windows-output.txt", + wasm_module_base64: emptyWasm.toString("base64"), + }); + assert.strictEqual(launched.type, "error", JSON.stringify(launched)); + assert.match(launched.message, /external callers may launch only EntrypointV1/); + + const recordedTask = await send(addr, signedNodeRequest(windowsNode, windowsIdentity, "task_completed", { + type: "task_completed", + tenant: "tenant", + project: "project", + process: "vp-windows", + node: windowsNode, + task: "windows-command-dev", + status_code: 0, + stdout_bytes: 18, + stderr_bytes: 0, + stdout_tail: "", + stderr_tail: "", + stdout_truncated: false, + stderr_truncated: false, + artifact_path: "/vfs/artifacts/windows-output.txt", + artifact_digest: windowsArtifactDigest, + artifact_size_bytes: windowsArtifactBytes.length + })); + assert.strictEqual(recordedTask.type, "error"); + assert.match(recordedTask.message, /coordinator-issued task instance|issued active task|active task/); + + const events = await send(addr, { + type: "list_task_events", + tenant: "tenant", + project: "project", + actor_user: "user", + process: "vp-windows" + }); + assert.strictEqual(events.type, "task_events"); + assert.strictEqual(events.events.length, 0); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact: "windows-output.txt", + max_bytes: 1024 + }); + assert.strictEqual(link.type, "error"); + assert.match(link.message, /does not exist|not found|unavailable/); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + console.log("Windows best-effort smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/windows-runner-smoke.js b/scripts/windows-runner-smoke.js new file mode 100755 index 0000000..a80889c --- /dev/null +++ b/scripts/windows-runner-smoke.js @@ -0,0 +1,479 @@ +#!/usr/bin/env node + +const assert = require("assert"); +const cp = require("child_process"); +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); +const path = require("path"); +const { coordinatorWireRequest } = require("./coordinator-wire"); +const { nodeIdentity, signedNodeRequest } = require("./node-signing"); + +const repo = path.resolve(__dirname, ".."); +const forgejoWindowsNode = "forgejo-windows-node"; +const forgejoWindowsIdentity = nodeIdentity("windows-runner-smoke", forgejoWindowsNode); +const windowsEnvironmentDigest = `sha256:${crypto + .createHash("sha256") + .update("clusterflux/windows-command-dev/v1") + .digest("hex")}`; +const sourceSnapshot = `sha256:${crypto + .createHash("sha256") + .update("clusterflux/windows-runner/source/v1") + .digest("hex")}`; + +class DapClient { + constructor() { + this.child = cp.spawn( + "cargo", + ["run", "-q", "-p", "clusterflux-dap", "--bin", "clusterflux-debug-dap"], + { cwd: repo } + ); + this.seq = 1; + this.buffer = Buffer.alloc(0); + this.messages = []; + this.waiters = []; + this.stderr = ""; + + this.child.stdout.on("data", (chunk) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + this.parse(); + }); + this.child.stderr.on("data", (chunk) => { + this.stderr += chunk.toString(); + }); + this.child.on("exit", () => this.flushWaiters()); + } + + send(command, args = {}) { + const seq = this.seq++; + const message = { seq, type: "request", command, arguments: args }; + const payload = Buffer.from(JSON.stringify(coordinatorWireRequest(message))); + this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`); + this.child.stdin.write(payload); + return seq; + } + + async response(seq, command) { + const message = await this.waitFor( + (item) => + item.type === "response" && + item.request_seq === seq && + item.command === command + ); + if (!message.success) { + throw new Error(`DAP ${command} failed: ${message.message || JSON.stringify(coordinatorWireRequest(message))}`); + } + return message; + } + + waitFor(predicate, timeoutMs = 120000) { + const existing = this.messages.find(predicate); + if (existing) return Promise.resolve(existing); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.child.kill("SIGKILL"); + reject(new Error(`timed out waiting for DAP message\n${this.stderr}`)); + }, timeoutMs); + this.waiters.push({ predicate, resolve, timer }); + }); + } + + parse() { + while (true) { + const headerEnd = this.buffer.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const header = this.buffer.slice(0, headerEnd).toString(); + const match = header.match(/Content-Length: (\d+)/i); + if (!match) throw new Error(`bad DAP header: ${header}`); + const length = Number(match[1]); + const start = headerEnd + 4; + const end = start + length; + if (this.buffer.length < end) return; + const payload = this.buffer.slice(start, end).toString(); + this.buffer = this.buffer.slice(end); + this.messages.push(JSON.parse(payload)); + this.flushWaiters(); + } + } + + flushWaiters() { + for (const waiter of [...this.waiters]) { + const message = this.messages.find(waiter.predicate); + if (!message) continue; + clearTimeout(waiter.timer); + this.waiters.splice(this.waiters.indexOf(waiter), 1); + waiter.resolve(message); + } + } + + async close() { + if (this.child.exitCode !== null) return; + const seq = this.send("disconnect"); + await this.response(seq, "disconnect"); + this.child.stdin.end(); + } +} + +function waitForJsonLine(child) { + return new Promise((resolve, reject) => { + let buffer = ""; + child.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + try { + resolve(JSON.parse(buffer.slice(0, newline).trim())); + } catch (error) { + reject(error); + } + }); + child.once("exit", (code) => { + reject(new Error(`process exited before JSON line with code ${code}`)); + }); + }); +} + +function send(addr, message) { + return new Promise((resolve, reject) => { + const socket = net.connect(addr.port, addr.host, () => { + socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`); + }); + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk.toString(); + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + socket.end(); + try { + resolve(JSON.parse(buffer.slice(0, newline))); + } catch (error) { + reject(error); + } + }); + socket.on("error", reject); + }); +} + +function runWindowsAttach(addr, grant) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-cli", + "--bin", + "clusterflux", + "--", + "node", + "attach", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + forgejoWindowsNode, + "--public-key", + forgejoWindowsIdentity.publicKey, + "--enrollment-grant", + grant, + "--cap", + "windows-command-dev", + "--json" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Windows node attach failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (error) { + reject( + new Error(`Windows node attach output was not JSON: ${stdout}\n${error.stack || error.message}`) + ); + } + }); + }); +} + +function runWindowsNode(addr, grant) { + return new Promise((resolve, reject) => { + const child = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-node", + "--bin", + "clusterflux-node", + "--", + "--coordinator", + `${addr.host}:${addr.port}`, + "--tenant", + "tenant", + "--project-id", + "project", + "--node", + forgejoWindowsNode, + "--public-key", + forgejoWindowsIdentity.publicKey, + "--enrollment-grant", + grant, + "--process", + "vp-forgejo-windows", + "--task", + "windows-command-dev", + "--command", + "cmd", + "--arg", + "/C", + "--arg", + "echo clusterflux-windows-runner", + "--artifact", + "/vfs/artifacts/windows-runner-output.txt" + ], + { + cwd: repo, + env: { + ...process.env, + CLUSTERFLUX_NODE_PRIVATE_KEY: forgejoWindowsIdentity.privateKey, + }, + } + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Windows node smoke failed with code ${code}\n${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout.trim().split(/\r?\n/).at(-1))); + } catch (error) { + reject(new Error(`node output was not JSON: ${stdout}\n${error.stack || error.message}`)); + } + }); + }); +} + +async function assertDebuggerShowsWindowsThread() { + const client = new DapClient(); + try { + const initialize = client.send("initialize", { + adapterID: "clusterflux", + linesStartAt1: true, + columnsStartAt1: true + }); + await client.response(initialize, "initialize"); + + const launch = client.send("launch", { + entry: "build", + project: path.join(repo, "examples/launch-build-demo"), + runtimeBackend: "simulated" + }); + await client.response(launch, "launch"); + await client.waitFor( + (message) => message.type === "event" && message.event === "initialized" + ); + + const configurationDone = client.send("configurationDone"); + await client.response(configurationDone, "configurationDone"); + + const threadsRequest = client.send("threads"); + const threads = (await client.response(threadsRequest, "threads")).body.threads; + assert( + threads.some((thread) => thread.name.includes("compile windows")), + "debugger must represent the Windows task as a virtual thread" + ); + + await client.close(); + } catch (error) { + client.child.kill("SIGKILL"); + throw error; + } +} + +(async () => { + if (process.platform !== "win32") { + throw new Error( + `Windows runner validation requires win32; current platform is ${process.platform}` + ); + } + + const coordinator = cp.spawn( + "cargo", + [ + "run", + "-q", + "-p", + "clusterflux-coordinator", + "--bin", + "clusterflux-coordinator", + "--", + "--listen", + "127.0.0.1:0", + "--allow-local-trusted-loopback" + ], + { cwd: repo } + ); + let coordinatorStderr = ""; + coordinator.stderr.on("data", (chunk) => { + coordinatorStderr += chunk.toString(); + }); + + try { + const ready = await waitForJsonLine(coordinator); + const [host, portText] = ready.listen.split(":"); + const addr = { host, port: Number(portText) }; + assert.strictEqual((await send(addr, { type: "ping" })).type, "pong"); + + const attachGrant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(attachGrant.type, "node_enrollment_grant_created"); + assert.strictEqual(attachGrant.scope, "node:attach"); + + const attach = await runWindowsAttach(addr, attachGrant.grant); + assert.strictEqual(attach.plan.node, forgejoWindowsNode); + assert.strictEqual(attach.boundary.cli_contacted_coordinator, true); + assert.strictEqual(attach.boundary.used_enrollment_exchange, true); + assert.strictEqual(attach.coordinator_response.type, "node_enrollment_exchanged"); + assert.strictEqual(attach.coordinator_response.credential.scope, "node:attach"); + assert(attach.plan.capabilities.capabilities.includes("WindowsCommandDev")); + + const runtimeGrant = await send(addr, { + type: "create_node_enrollment_grant", + tenant: "tenant", + project: "project", + actor_user: "operator", + ttl_seconds: 900 + }); + assert.strictEqual(runtimeGrant.type, "node_enrollment_grant_created"); + + const report = await runWindowsNode(addr, runtimeGrant.grant); + assert.strictEqual(report.node_status, "completed"); + assert.strictEqual(report.virtual_thread, "windows-command-dev"); + assert.strictEqual(report.status_code, 0); + assert.strictEqual(report.large_bytes_uploaded, false); + assert.strictEqual( + report.staged_artifact.path, + "/vfs/artifacts/windows-runner-output.txt" + ); + assert.strictEqual(report.registration_response.type, "node_enrollment_exchanged"); + assert.strictEqual(report.registration_response.credential.scope, "node:attach"); + assert.strictEqual(report.coordinator_response.type, "task_recorded"); + + const recorded = await send(addr, signedNodeRequest(forgejoWindowsNode, forgejoWindowsIdentity, "report_node_capabilities", { + type: "report_node_capabilities", + tenant: "tenant", + project: "project", + node: forgejoWindowsNode, + capabilities: { + os: "Windows", + arch: "x86_64", + capabilities: ["Command", "WindowsCommandDev", "VfsArtifacts"], + environment_backends: ["WindowsCommandDev"], + source_providers: ["filesystem"] + }, + cached_environment_digests: [windowsEnvironmentDigest], + source_snapshots: [sourceSnapshot], + artifact_locations: ["windows-runner-output.txt"], + direct_connectivity: true, + online: true + })); + assert.strictEqual(recorded.type, "node_capabilities_recorded"); + + const placement = await send(addr, { + type: "schedule_task", + tenant: "tenant", + project: "project", + environment: { + os: "Windows", + arch: null, + capabilities: ["WindowsCommandDev"] + }, + environment_digest: windowsEnvironmentDigest, + required_capabilities: ["Command"], + source_snapshot: sourceSnapshot, + required_artifacts: ["windows-runner-output.txt"], + prefer_node: null + }); + assert.strictEqual(placement.type, "task_placement"); + assert.strictEqual(placement.placement.node, "forgejo-windows-node"); + + const link = await send(addr, { + type: "create_artifact_download_link", + tenant: "tenant", + project: "project", + actor_user: "user", + artifact: "windows-runner-output.txt", + max_bytes: 1024 + }); + assert.strictEqual(link.type, "artifact_download_link"); + assert.deepStrictEqual(link.link.source, { + RetainedNode: "forgejo-windows-node" + }); + } catch (error) { + if (coordinatorStderr) { + error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`; + } + throw error; + } finally { + coordinator.kill("SIGTERM"); + } + + await assertDebuggerShowsWindowsThread(); + + const outDir = path.join(repo, "target", "acceptance"); + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync( + path.join(outDir, "windows-runner.json"), + `${JSON.stringify( + { + kind: "clusterflux_windows_runner_validation", + platform: process.platform, + runner: process.env.FORGEJO_RUNNER_NAME || process.env.RUNNER_NAME || null, + validated: true + }, + null, + 2 + )}\n` + ); + + console.log("Windows runner smoke passed"); +})().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json deleted file mode 100644 index d6aebd7..0000000 --- a/vscode-extension/package-lock.json +++ /dev/null @@ -1,3810 +0,0 @@ -{ - "name": "clusterflux-vscode", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "clusterflux-vscode", - "version": "0.1.0", - "devDependencies": { - "@vscode/vsce": "3.9.2" - }, - "engines": { - "vscode": "^1.80.0" - } - }, - "node_modules/@azu/format-text": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz", - "integrity": "sha512-Swi4N7Edy1Eqq82GxgEECXSSLyn6GOb5htRFPzBDdUkECGXtlf12ynO5oJSpWKPwCaUssOu7NfhDcCWpIC6Ywg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@azu/style-format": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@azu/style-format/-/style-format-1.0.1.tgz", - "integrity": "sha512-AHcTojlNBdD/3/KxIKlg8sxIWHfOtQszLvOpagLTO+bjC3u7SAszu1lf//u7JJC50aUSH+BVWDD/KvaA6Gfn5g==", - "dev": true, - "license": "WTFPL", - "dependencies": { - "@azu/format-text": "^1.0.1" - } - }, - "node_modules/@azure/abort-controller": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", - "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", - "dev": true, - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-auth": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", - "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-util": "^1.13.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-client": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.0.tgz", - "integrity": "sha512-JjQWO6akOck45PH/XBrxzsQGAiKrfFl4m5iggJ0ItMIz5omRufOXWpqCPpdjKN3vKDzlSUvFjaMb7Zwf0gvAdA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-rest-pipeline": "^1.22.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.25.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", - "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "@typespec/ts-http-runtime": "^0.3.4", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-tracing": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", - "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/core-util": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", - "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/identity": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.1.tgz", - "integrity": "sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.9.0", - "@azure/core-client": "^1.9.2", - "@azure/core-rest-pipeline": "^1.17.0", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.0.0", - "@azure/msal-browser": "^5.5.0", - "@azure/msal-node": "^5.1.0", - "open": "^10.1.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/logger": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", - "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@azure/msal-browser": { - "version": "5.17.1", - "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-5.17.1.tgz", - "integrity": "sha512-zBhRGzABKSI7hfWh5EaZmril5ybZ7imBN1qEZl5sDTaelr+l8SnPjZO50Q4dnKnm347YPIlBMSnXKZyh3Yu5DQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/msal-common": "16.11.2" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-common": { - "version": "16.11.2", - "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.11.2.tgz", - "integrity": "sha512-yDhtBOGDCdK9ipQ9g3+wmlMEPnZx2pXaDicDd9jYyR1L+7lEbvEohTDmF5qejZDutZY3m9pWPxeYxzNC701A2w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-node": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-5.4.1.tgz", - "integrity": "sha512-yqgoyOIMCH7TNaSLMBTP+4LUlbMMf1zgC8nzOFG95lmW82CmsAEtUT0J93e4BdqDcnX5qle/9X+yb7A8Mw9M0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/msal-common": "16.11.2", - "jsonwebtoken": "^9.0.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", - "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.29.7", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", - "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@secretlint/config-creator": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/config-creator/-/config-creator-10.2.2.tgz", - "integrity": "sha512-BynOBe7Hn3LJjb3CqCHZjeNB09s/vgf0baBaHVw67w7gHF0d25c3ZsZ5+vv8TgwSchRdUCRrbbcq5i2B1fJ2QQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/config-loader": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/config-loader/-/config-loader-10.2.2.tgz", - "integrity": "sha512-ndjjQNgLg4DIcMJp4iaRD6xb9ijWQZVbd9694Ol2IszBIbGPPkwZHzJYKICbTBmh6AH/pLr0CiCaWdGJU7RbpQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/profiler": "^10.2.2", - "@secretlint/resolver": "^10.2.2", - "@secretlint/types": "^10.2.2", - "ajv": "^8.17.1", - "debug": "^4.4.1", - "rc-config-loader": "^4.1.3" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/core": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/core/-/core-10.2.2.tgz", - "integrity": "sha512-6rdwBwLP9+TO3rRjMVW1tX+lQeo5gBbxl1I5F8nh8bgGtKwdlCMhMKsBWzWg1ostxx/tIG7OjZI0/BxsP8bUgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/profiler": "^10.2.2", - "@secretlint/types": "^10.2.2", - "debug": "^4.4.1", - "structured-source": "^4.0.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/formatter": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/formatter/-/formatter-10.2.2.tgz", - "integrity": "sha512-10f/eKV+8YdGKNQmoDUD1QnYL7TzhI2kzyx95vsJKbEa8akzLAR5ZrWIZ3LbcMmBLzxlSQMMccRmi05yDQ5YDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/resolver": "^10.2.2", - "@secretlint/types": "^10.2.2", - "@textlint/linter-formatter": "^15.2.0", - "@textlint/module-interop": "^15.2.0", - "@textlint/types": "^15.2.0", - "chalk": "^5.4.1", - "debug": "^4.4.1", - "pluralize": "^8.0.0", - "strip-ansi": "^7.1.0", - "table": "^6.9.0", - "terminal-link": "^4.0.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/formatter/node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/@secretlint/node": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/node/-/node-10.2.2.tgz", - "integrity": "sha512-eZGJQgcg/3WRBwX1bRnss7RmHHK/YlP/l7zOQsrjexYt6l+JJa5YhUmHbuGXS94yW0++3YkEJp0kQGYhiw1DMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/config-loader": "^10.2.2", - "@secretlint/core": "^10.2.2", - "@secretlint/formatter": "^10.2.2", - "@secretlint/profiler": "^10.2.2", - "@secretlint/source-creator": "^10.2.2", - "@secretlint/types": "^10.2.2", - "debug": "^4.4.1", - "p-map": "^7.0.3" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/profiler": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/profiler/-/profiler-10.2.2.tgz", - "integrity": "sha512-qm9rWfkh/o8OvzMIfY8a5bCmgIniSpltbVlUVl983zDG1bUuQNd1/5lUEeWx5o/WJ99bXxS7yNI4/KIXfHexig==", - "dev": true, - "license": "MIT" - }, - "node_modules/@secretlint/resolver": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/resolver/-/resolver-10.2.2.tgz", - "integrity": "sha512-3md0cp12e+Ae5V+crPQYGd6aaO7ahw95s28OlULGyclyyUtf861UoRGS2prnUrKh7MZb23kdDOyGCYb9br5e4w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@secretlint/secretlint-formatter-sarif": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-formatter-sarif/-/secretlint-formatter-sarif-10.2.2.tgz", - "integrity": "sha512-ojiF9TGRKJJw308DnYBucHxkpNovDNu1XvPh7IfUp0A12gzTtxuWDqdpuVezL7/IP8Ua7mp5/VkDMN9OLp1doQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "node-sarif-builder": "^3.2.0" - } - }, - "node_modules/@secretlint/secretlint-rule-no-dotenv": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-no-dotenv/-/secretlint-rule-no-dotenv-10.2.2.tgz", - "integrity": "sha512-KJRbIShA9DVc5Va3yArtJ6QDzGjg3PRa1uYp9As4RsyKtKSSZjI64jVca57FZ8gbuk4em0/0Jq+uy6485wxIdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/secretlint-rule-preset-recommend": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-preset-recommend/-/secretlint-rule-preset-recommend-10.2.2.tgz", - "integrity": "sha512-K3jPqjva8bQndDKJqctnGfwuAxU2n9XNCPtbXVI5JvC7FnQiNg/yWlQPbMUlBXtBoBGFYp08A94m6fvtc9v+zA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/source-creator": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/source-creator/-/source-creator-10.2.2.tgz", - "integrity": "sha512-h6I87xJfwfUTgQ7irWq7UTdq/Bm1RuQ/fYhA3dtTIAop5BwSFmZyrchph4WcoEvbN460BWKmk4RYSvPElIIvxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/types": "^10.2.2", - "istextorbinary": "^9.5.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@secretlint/types": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/@secretlint/types/-/types-10.2.2.tgz", - "integrity": "sha512-Nqc90v4lWCXyakD6xNyNACBJNJ0tNCwj2WNk/7ivyacYHxiITVgmLUFXTBOeCdy79iz6HtN9Y31uw/jbLrdOAg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@sindresorhus/merge-streams": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", - "integrity": "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@textlint/ast-node-types": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-15.7.1.tgz", - "integrity": "sha512-Wii5UgUKFEh9Uv6wbq1zr4/Kf+dtjiUuzPrrXzKp8H+ifkvKNzi23V4Nz+6wVyHQn5T28AFuc8VH8OtzvGYecA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/linter-formatter": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/linter-formatter/-/linter-formatter-15.7.1.tgz", - "integrity": "sha512-TdwZ/debWYFD05K3CcoHtwvnCrza29wZxD+BjDTk/V5N7iRqkK1dTTHSD4A8AIgROLiDkHJmIKQbasbmsg8AvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azu/format-text": "^1.0.2", - "@azu/style-format": "^1.0.1", - "@textlint/module-interop": "15.7.1", - "@textlint/resolver": "15.7.1", - "@textlint/types": "15.7.1", - "chalk": "^4.1.2", - "debug": "^4.4.3", - "js-yaml": "^4.1.1", - "lodash": "^4.18.1", - "pluralize": "^2.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1", - "table": "^6.9.0", - "text-table": "^0.2.0" - } - }, - "node_modules/@textlint/linter-formatter/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@textlint/linter-formatter/node_modules/pluralize": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-2.0.0.tgz", - "integrity": "sha512-TqNZzQCD4S42De9IfnnBvILN7HAW7riLqsCyp8lgjXeysyPlX5HhqKAcJHHHb9XskE4/a+7VGC9zzx8Ls0jOAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/linter-formatter/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@textlint/module-interop": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/module-interop/-/module-interop-15.7.1.tgz", - "integrity": "sha512-Jg+sQW2L/cRJypk59wtcMUVVpt8vmit5ZMT3gUnFwevP3A6Qp1HfOtUy9ObT4hBX3lOSGT/ekcCDxR1pL7uH1g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/resolver": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/resolver/-/resolver-15.7.1.tgz", - "integrity": "sha512-8XnO0pgF6mXnm41VvWmBbEIdGPhiCUt31uLZkOis1ECeg/1SoUcIT6Mx/F0e1rukq8l0UlOSeY9a31CsvRMK0g==", - "dev": true, - "license": "MIT" - }, - "node_modules/@textlint/types": { - "version": "15.7.1", - "resolved": "https://registry.npmjs.org/@textlint/types/-/types-15.7.1.tgz", - "integrity": "sha512-Vye/GmFNBTgVzZFtIFJTmLB+s2A7oIADxNG6r9UhfPuY+Czv0z5G3xeyFZZudPlfxURsKUyPIU5XsjOFqVp33A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@textlint/ast-node-types": "15.7.1" - } - }, - "node_modules/@types/normalize-package-data": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", - "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/sarif": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@types/sarif/-/sarif-2.1.7.tgz", - "integrity": "sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@typespec/ts-http-runtime": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.7.tgz", - "integrity": "sha512-JVUD8X2tfDMWjcjLs4yVxxVrS8yR5vnh386GAXT9Qj79nBxxXSaHFQZg5FweLmT8HlPQ3kii6noUB+Z9RN7DvQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@vscode/vsce": { - "version": "3.9.2", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.9.2.tgz", - "integrity": "sha512-XSxMosEEDO6vLxELAHVkwmhC0qe0ijZni2jB9Rcs8kQsW4lhTDQ/wMzmwFs/buotAWSnpmUp/dRWD2ufG3UYKA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@azure/identity": "^4.1.0", - "@secretlint/node": "^10.1.2", - "@secretlint/secretlint-formatter-sarif": "^10.1.2", - "@secretlint/secretlint-rule-no-dotenv": "^10.1.2", - "@secretlint/secretlint-rule-preset-recommend": "^10.1.2", - "@vscode/vsce-sign": "^2.0.0", - "azure-devops-node-api": "^12.5.0", - "chalk": "^4.1.2", - "cheerio": "^1.0.0-rc.9", - "cockatiel": "^3.1.2", - "commander": "^12.1.0", - "form-data": "^4.0.0", - "glob": "^13.0.6", - "hosted-git-info": "^4.0.2", - "jsonc-parser": "^3.2.0", - "leven": "^3.1.0", - "markdown-it": "^14.1.0", - "mime": "^1.3.4", - "minimatch": "^10.2.2", - "parse-semver": "^1.1.1", - "read": "^1.0.7", - "secretlint": "^10.1.2", - "semver": "^7.5.2", - "tmp": "^0.2.3", - "typed-rest-client": "^1.8.4", - "url-join": "^4.0.1", - "xml2js": "^0.5.0", - "yauzl": "^3.2.1", - "yazl": "^2.2.2" - }, - "bin": { - "vsce": "vsce" - }, - "engines": { - "node": ">= 20" - }, - "optionalDependencies": { - "keytar": "^7.7.0" - } - }, - "node_modules/@vscode/vsce-sign": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign/-/vsce-sign-2.0.9.tgz", - "integrity": "sha512-8IvaRvtFyzUnGGl3f5+1Cnor3LqaUWvhaUjAYO8Y39OUYlOf3cRd+dowuQYLpZcP3uwSG+mURwjEBOSq4SOJ0g==", - "dev": true, - "hasInstallScript": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optionalDependencies": { - "@vscode/vsce-sign-alpine-arm64": "2.0.6", - "@vscode/vsce-sign-alpine-x64": "2.0.6", - "@vscode/vsce-sign-darwin-arm64": "2.0.6", - "@vscode/vsce-sign-darwin-x64": "2.0.6", - "@vscode/vsce-sign-linux-arm": "2.0.6", - "@vscode/vsce-sign-linux-arm64": "2.0.6", - "@vscode/vsce-sign-linux-x64": "2.0.6", - "@vscode/vsce-sign-win32-arm64": "2.0.6", - "@vscode/vsce-sign-win32-x64": "2.0.6" - } - }, - "node_modules/@vscode/vsce-sign-alpine-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-arm64/-/vsce-sign-alpine-arm64-2.0.6.tgz", - "integrity": "sha512-wKkJBsvKF+f0GfsUuGT0tSW0kZL87QggEiqNqK6/8hvqsXvpx8OsTEc3mnE1kejkh5r+qUyQ7PtF8jZYN0mo8Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "alpine" - ] - }, - "node_modules/@vscode/vsce-sign-alpine-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-alpine-x64/-/vsce-sign-alpine-x64-2.0.6.tgz", - "integrity": "sha512-YoAGlmdK39vKi9jA18i4ufBbd95OqGJxRvF3n6ZbCyziwy3O+JgOpIUPxv5tjeO6gQfx29qBivQ8ZZTUF2Ba0w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "alpine" - ] - }, - "node_modules/@vscode/vsce-sign-darwin-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-arm64/-/vsce-sign-darwin-arm64-2.0.6.tgz", - "integrity": "sha512-5HMHaJRIQuozm/XQIiJiA0W9uhdblwwl2ZNDSSAeXGO9YhB9MH5C4KIHOmvyjUnKy4UCuiP43VKpIxW1VWP4tQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@vscode/vsce-sign-darwin-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-darwin-x64/-/vsce-sign-darwin-x64-2.0.6.tgz", - "integrity": "sha512-25GsUbTAiNfHSuRItoQafXOIpxlYj+IXb4/qarrXu7kmbH94jlm5sdWSCKrrREs8+GsXF1b+l3OB7VJy5jsykw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@vscode/vsce-sign-linux-arm": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm/-/vsce-sign-linux-arm-2.0.6.tgz", - "integrity": "sha512-UndEc2Xlq4HsuMPnwu7420uqceXjs4yb5W8E2/UkaHBB9OWCwMd3/bRe/1eLe3D8kPpxzcaeTyXiK3RdzS/1CA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-linux-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-arm64/-/vsce-sign-linux-arm64-2.0.6.tgz", - "integrity": "sha512-cfb1qK7lygtMa4NUl2582nP7aliLYuDEVpAbXJMkDq1qE+olIw/es+C8j1LJwvcRq1I2yWGtSn3EkDp9Dq5FdA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-linux-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-linux-x64/-/vsce-sign-linux-x64-2.0.6.tgz", - "integrity": "sha512-/olerl1A4sOqdP+hjvJ1sbQjKN07Y3DVnxO4gnbn/ahtQvFrdhUi0G1VsZXDNjfqmXw57DmPi5ASnj/8PGZhAA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@vscode/vsce-sign-win32-arm64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-arm64/-/vsce-sign-win32-arm64-2.0.6.tgz", - "integrity": "sha512-ivM/MiGIY0PJNZBoGtlRBM/xDpwbdlCWomUWuLmIxbi1Cxe/1nooYrEQoaHD8ojVRgzdQEUzMsRbyF5cJJgYOg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@vscode/vsce-sign-win32-x64": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@vscode/vsce-sign-win32-x64/-/vsce-sign-win32-x64-2.0.6.tgz", - "integrity": "sha512-mgth9Kvze+u8CruYMmhHw6Zgy3GRX2S+Ed5oSokDEK5vPEwGGKnmuXua9tmFhomeAnhgJnL4DCna3TiNuGrBTQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "SEE LICENSE IN LICENSE.txt", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-escapes": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", - "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", - "dev": true, - "license": "MIT", - "dependencies": { - "environment": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/azure-devops-node-api": { - "version": "12.5.0", - "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-12.5.0.tgz", - "integrity": "sha512-R5eFskGvOm3U/GzeAuxRkUsAl0hrAwGgWn6zAd2KrZmrEhWZVqLew4OOupbQlXUuojUzpGtq62SmdhJ06N88og==", - "dev": true, - "license": "MIT", - "dependencies": { - "tunnel": "0.0.6", - "typed-rest-client": "^1.8.4" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/binaryextensions": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-6.11.0.tgz", - "integrity": "sha512-sXnYK/Ij80TO3lcqZVV2YgfKN5QjUWIRk/XSm2J/4bd/lPko3lvk0O4ZppH6m+6hB2/GTu+ptNwVFe1xh+QLQw==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "editions": "^6.21.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "dev": true, - "license": "ISC" - }, - "node_modules/boundary": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz", - "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/bundle-name": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", - "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "run-applescript": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, - "node_modules/cheerio-select": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", - "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-select": "^5.1.0", - "css-what": "^6.1.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/cockatiel": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/cockatiel/-/cockatiel-3.2.1.tgz", - "integrity": "sha512-gfrHV6ZPkquExvMh9IOkKsBzNDk6sDuZ6DdBGUBkvFnTCqCxzpuq48RySgP0AnaqQkw2zynOFj9yly6T1Q2G5Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/css-select": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", - "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-what": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", - "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/default-browser": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", - "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "bundle-name": "^4.1.0", - "default-browser-id": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", - "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", - "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "BSD-2-Clause" - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", - "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/editions": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/editions/-/editions-6.22.0.tgz", - "integrity": "sha512-UgGlf8IW75je7HZjNDpJdCv4cGJWIi6yumFdZ0R7A8/CIhQiWUjyGLCxdHpd8bmyD1gnkfUNK0oeOXqUS2cpfQ==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "version-range": "^4.15.0" - }, - "engines": { - "ecmascript": ">= es5", - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/encoding-sniffer": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", - "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", - "dev": true, - "license": "MIT", - "dependencies": { - "iconv-lite": "^0.6.3", - "whatwg-encoding": "^3.1.1" - }, - "funding": { - "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/environment": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", - "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", - "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "dev": true, - "license": "(MIT OR WTFPL)", - "optional": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-glob": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.8" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/fastq": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", - "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, - "license": "ISC", - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/fs-extra": { - "version": "11.3.6", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.6.tgz", - "integrity": "sha512-w8ZNZr2mKIc7qeNaQ9AVPT1+iFaI+Avd4xudVOvdDJ8VytREi1Ft5Ih7hd9jjehod8vAM5GMsfQ/TpPf4EyoEA==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-14.1.0.tgz", - "integrity": "sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sindresorhus/merge-streams": "^2.1.0", - "fast-glob": "^3.3.3", - "ignore": "^7.0.3", - "path-type": "^6.0.0", - "slash": "^5.1.0", - "unicorn-magic": "^0.3.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hosted-git-info": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", - "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", - "dev": true, - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, - "node_modules/htmlparser2/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/ignore": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", - "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/index-to-position": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", - "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/is-docker": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", - "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "dev": true, - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", - "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-wsl": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", - "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-inside-container": "^1.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/istextorbinary": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-9.5.0.tgz", - "integrity": "sha512-5mbUj3SiZXCuRf9fT3ibzbSSEWiy63gFfksmGfdOzujPjW3k+z8WvIBxcJHBoQNlaZaiyB25deviif2+osLmLw==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "binaryextensions": "^6.11.0", - "editions": "^6.21.0", - "textextensions": "^6.11.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsonc-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/jsonfile": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", - "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/jsonwebtoken": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", - "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", - "dev": true, - "license": "MIT", - "dependencies": { - "jws": "^4.0.1", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=12", - "npm": ">=6" - } - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", - "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jwa": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/keytar": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/keytar/-/keytar-7.9.0.tgz", - "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-addon-api": "^4.3.0", - "prebuild-install": "^7.0.1" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/linkify-it": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz", - "integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "uc.micro": "^2.0.0" - } - }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.truncate": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", - "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", - "dev": true, - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/markdown-it": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.3.0.tgz", - "integrity": "sha512-RCEsPjR+sr0x+AuYp601tKTkgFG4YEPLCzHST3cQ/fhlJkqAkz1L2/Qbp1j9qw5SBwQHFBoW8+hoN5xssOF0Tw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/markdown-it" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.5.0", - "linkify-it": "^5.0.2", - "mdurl": "^2.0.0", - "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" - }, - "bin": { - "markdown-it": "bin/markdown-it.mjs" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "dev": true, - "license": "MIT" - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true, - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "license": "MIT", - "optional": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "dev": true, - "license": "ISC" - }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/node-abi": { - "version": "3.94.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", - "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-addon-api": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", - "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/node-sarif-builder": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/node-sarif-builder/-/node-sarif-builder-3.4.0.tgz", - "integrity": "sha512-tGnJW6OKRii9u/b2WiUViTJS+h7Apxx17qsMUjsUeNDiMMX5ZFf8F8Fcz7PAQ6omvOxHZtvDTmOYKJQwmfpjeg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/sarif": "^2.1.7", - "fs-extra": "^11.1.1" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/normalize-package-data": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz", - "integrity": "sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "hosted-git-info": "^7.0.0", - "semver": "^7.3.5", - "validate-npm-package-license": "^3.0.4" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/normalize-package-data/node_modules/hosted-git-info": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", - "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==", - "dev": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^10.0.1" - }, - "engines": { - "node": "^16.14.0 || >=18.0.0" - } - }, - "node_modules/normalize-package-data/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "optional": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", - "dev": true, - "license": "MIT", - "dependencies": { - "default-browser": "^5.2.1", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-map": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.5.tgz", - "integrity": "sha512-e8vJF4XdVkzqqSHguEMz41mQO1wKwxKm5ENrUJQUu9kLDCtn83cxbyHZcszr4QC5zEA7WffRRC4gsTecC7J9oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parse-json": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", - "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.26.2", - "index-to-position": "^1.1.0", - "type-fest": "^4.39.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parse-semver": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/parse-semver/-/parse-semver-1.1.1.tgz", - "integrity": "sha512-Eg1OuNntBMH0ojvEKSrvDSnwLmvVuUOSdylH/pSCPNMIspLlweJyIWXCE+k/5hm3cj/EBUYwmWkjhBALNP4LXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^5.1.0" - } - }, - "node_modules/parse-semver/node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/parse5": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", - "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "entities": "^6.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", - "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "domhandler": "^5.0.3", - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-parser-stream": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", - "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", - "dev": true, - "license": "MIT", - "dependencies": { - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "11.5.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", - "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/path-type": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-6.0.0.tgz", - "integrity": "sha512-Vj7sf++t5pBD637NSfkxpHSMfWaeig5+DKWLhcqIYx6mWQz5hdJTGDVMQiJcw1ZYkhs7AazKDGpRVji1LJCZUQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pluralize": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", - "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pump": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", - "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "es-define-property": "^1.0.1", - "side-channel": "^1.1.1" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "optional": true, - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc-config-loader": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/rc-config-loader/-/rc-config-loader-4.1.4.tgz", - "integrity": "sha512-3GiwEzklkbXTDp52UR5nT8iXgYAx1V9ZG/kDZT7p60u2GCv2XTwQq4NzinMoMpNtXhmt3WkhYXcj6HH8HdwCEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "js-yaml": "^4.1.1", - "json5": "^2.2.3", - "require-from-string": "^2.0.2" - } - }, - "node_modules/read": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", - "integrity": "sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "mute-stream": "~0.0.4" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/read-pkg": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-9.0.1.tgz", - "integrity": "sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/normalize-package-data": "^2.4.3", - "normalize-package-data": "^6.0.0", - "parse-json": "^8.0.0", - "type-fest": "^4.6.0", - "unicorn-magic": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/read-pkg/node_modules/unicorn-magic": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz", - "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/reusify": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", - "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/run-applescript": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", - "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, - "license": "MIT" - }, - "node_modules/sax": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", - "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=11.0.0" - } - }, - "node_modules/secretlint": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/secretlint/-/secretlint-10.2.2.tgz", - "integrity": "sha512-xVpkeHV/aoWe4vP4TansF622nBEImzCY73y/0042DuJ29iKIaqgoJ8fGxre3rVSHHbxar4FdJobmTnLp9AU0eg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@secretlint/config-creator": "^10.2.2", - "@secretlint/formatter": "^10.2.2", - "@secretlint/node": "^10.2.2", - "@secretlint/profiler": "^10.2.2", - "debug": "^4.4.1", - "globby": "^14.1.0", - "read-pkg": "^9.0.1" - }, - "bin": { - "secretlint": "bin/secretlint.js" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/side-channel": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", - "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4", - "side-channel-list": "^1.0.1", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/slash": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", - "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/spdx-correct": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", - "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", - "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", - "dev": true, - "license": "CC-BY-3.0" - }, - "node_modules/spdx-expression-parse": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", - "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.23", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", - "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", - "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.2.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/structured-source": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz", - "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "boundary": "^2.0.0" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-hyperlinks": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.2.0.tgz", - "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=14.18" - }, - "funding": { - "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" - } - }, - "node_modules/table": { - "version": "6.9.0", - "resolved": "https://registry.npmjs.org/table/-/table-6.9.0.tgz", - "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/table/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/table/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tar-fs": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", - "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/terminal-link": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-4.0.0.tgz", - "integrity": "sha512-lk+vH+MccxNqgVqSnkMVKx4VLJfnLjDBGzH16JVZjKE2DoxP57s6/vt6JmXV5I3jBcfGrxNrYtC+mPtU7WJztA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-escapes": "^7.0.0", - "supports-hyperlinks": "^3.2.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true, - "license": "MIT" - }, - "node_modules/textextensions": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-6.11.0.tgz", - "integrity": "sha512-tXJwSr9355kFJI3lbCkPpUH5cP8/M0GGy2xLO34aZCjMXBaK3SoPnZwr/oWmo1FdCnELcs4npdCIOFtq9W3ruQ==", - "dev": true, - "license": "Artistic-2.0", - "dependencies": { - "editions": "^6.21.0" - }, - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/tmp": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", - "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.14" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD" - }, - "node_modules/tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6.11 <=0.7.0 || >=0.7.3" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/type-fest": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typed-rest-client": { - "version": "1.8.11", - "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.11.tgz", - "integrity": "sha512-5UvfMpd1oelmUPRbbaVnq+rHP7ng2cE4qoQkQeAqxRL6PklkxsM0g32/HL0yfvruK6ojQ5x8EE+HF4YV6DtuCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "qs": "^6.9.1", - "tunnel": "0.0.6", - "underscore": "^1.12.1" - } - }, - "node_modules/uc.micro": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "dev": true, - "license": "MIT" - }, - "node_modules/underscore": { - "version": "1.13.8", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", - "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/unicorn-magic": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", - "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/url-join": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", - "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", - "dev": true, - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, - "license": "MIT", - "optional": true - }, - "node_modules/validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, - "node_modules/version-range": { - "version": "4.15.0", - "resolved": "https://registry.npmjs.org/version-range/-/version-range-4.15.0.tgz", - "integrity": "sha512-Ck0EJbAGxHwprkzFO966t4/5QkRuzh+/I1RxhLgUKKwEn+Cd8NwM60mE3AqBZg5gYODoXW0EFsQvbZjRlvdqbg==", - "dev": true, - "license": "Artistic-2.0", - "engines": { - "node": ">=4" - }, - "funding": { - "url": "https://bevry.me/fund" - } - }, - "node_modules/whatwg-encoding": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", - "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", - "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", - "dev": true, - "license": "MIT", - "dependencies": { - "iconv-lite": "0.6.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/whatwg-mimetype": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", - "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC", - "optional": true - }, - "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-wsl": "^3.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, - "license": "ISC" - }, - "node_modules/yauzl": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", - "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yazl": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/yazl/-/yazl-2.5.1.tgz", - "integrity": "sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3" - } - } - } -} diff --git a/vscode-extension/package.json b/vscode-extension/package.json index 2099a95..8f616bc 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -7,12 +7,6 @@ "engines": { "vscode": "^1.80.0" }, - "scripts": { - "package:vsix": "vsce package --allow-missing-repository --skip-license" - }, - "devDependencies": { - "@vscode/vsce": "3.9.2" - }, "categories": [ "Debuggers", "Other" @@ -84,7 +78,7 @@ }, "sourcePath": { "type": "string", - "description": "Source file shown in the Clusterflux virtual stack. Defaults to src/lib.rs or src/main.rs." + "description": "Source file shown in the Clusterflux virtual stack. Defaults to src/build.rs or src/main.rs." }, "runtimeBackend": { "type": "string",